Compare commits
47
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
69c5e6d1a1 | ||
|
|
0cd37aa618 | ||
|
|
4d894379df | ||
|
|
44fc0090c7 | ||
|
|
38c7fb44e1 | ||
|
|
c0222a55e1 | ||
|
|
ea6636f9d9 | ||
|
|
8c172ec9dd | ||
|
|
514b72cddb | ||
|
|
6983925b2e | ||
|
|
d897124e94 | ||
|
|
543534a058 | ||
|
|
234491f1cc | ||
|
|
cb0cb04534 | ||
|
|
d4dc695e1d | ||
|
|
f4b9d4cf38 | ||
|
|
fd41f56381 | ||
|
|
aef670c3bc | ||
|
|
3a43537e67 | ||
|
|
bc2b815391 | ||
|
|
7af0cf241e | ||
|
|
209fb50f16 | ||
|
|
a5a0fad86e | ||
|
|
9c6964e1a1 | ||
|
|
901285b225 | ||
|
|
620cd2746b | ||
|
|
355da1a4fd | ||
|
|
8deb18273d | ||
|
|
a8227723bf | ||
|
|
28ef3ac3c8 | ||
|
|
6d77e5a135 | ||
|
|
09204687fd | ||
|
|
1fb09efc9b | ||
|
|
064963f45f | ||
|
|
214171ac90 | ||
|
|
31b2716089 | ||
|
|
6564ee235a | ||
|
|
c16d876007 | ||
|
|
266b4453ca | ||
|
|
3ba4373e72 | ||
|
|
0163df3683 | ||
|
|
356c014f41 | ||
|
|
17c4f19db0 | ||
|
|
807384f4af | ||
|
|
c5daad0dcb | ||
|
|
1490b6318d | ||
|
|
1e36d42bcc |
@@ -1,7 +1,7 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## Project Structure & Module Organization
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/` as paired numbered `.up.sql` and `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/sql/` as Goose SQL migrations. Legacy converted migrations intentionally keep their original numeric versions so existing `schema_versions` rows can bootstrap cleanly into Goose. New migrations must use timestamped filenames created with `make migrate-create NAME=add_thing`; do not run `goose fix` or create paired `.up.sql` / `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
|
||||
When creating or editing `docs/superpowers/specs/` or `docs/superpowers/plans/`, never include local absolute filesystem paths or transient worktree IDs. Use repository-relative paths and wording like "Commands assume the repository root is the cwd."
|
||||
|
||||
@@ -53,6 +53,7 @@ Use the checked-in `Makefile` for the common paths:
|
||||
- `make dev-frontend`: start the Vite dev server with HMR
|
||||
- `make dev-proxy` / `make dev-transcode`: run standalone worker modes
|
||||
- `make lint`: run `golangci-lint` and frontend ESLint
|
||||
- `make migrate-status` / `make migrate-up`: inspect or apply Goose migrations through Silo's legacy-safe bootstrapping runner
|
||||
|
||||
Run before opening a merge request:
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks migrate-create migrate-validate migrate-status migrate-up
|
||||
|
||||
GIT_COMMON_DIR := $(strip $(shell git rev-parse --git-common-dir 2>/dev/null))
|
||||
MAIN_CHECKOUT_ROOT := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/..))
|
||||
SHARED_MAKEFILE_LOCAL := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/../Makefile.local))
|
||||
DEFAULT_PLUGIN_SDK_DIR := $(abspath ../silo-plugin-sdk)
|
||||
SHARED_PLUGIN_SDK_DIR := $(if $(MAIN_CHECKOUT_ROOT),$(abspath $(MAIN_CHECKOUT_ROOT)/../silo-plugin-sdk))
|
||||
GOOSE := go run github.com/pressly/goose/v3/cmd/goose@v3.27.1
|
||||
GOOSE_DIR := migrations/sql
|
||||
ENV_FILE ?= .env
|
||||
|
||||
ifneq ($(wildcard $(DEFAULT_PLUGIN_SDK_DIR)),)
|
||||
DEV_PLUGIN_SDK_DIR ?= $(DEFAULT_PLUGIN_SDK_DIR)
|
||||
@@ -54,6 +57,23 @@ lint:
|
||||
verify-local-paths:
|
||||
scripts/check-local-path-leaks.sh
|
||||
|
||||
# Create a timestamped Goose SQL migration. Usage: make migrate-create NAME=add_thing
|
||||
migrate-create:
|
||||
@if [ -z "$(NAME)" ]; then echo "usage: make migrate-create NAME=add_thing"; exit 1; fi
|
||||
$(GOOSE) -dir $(GOOSE_DIR) create "$(NAME)" sql
|
||||
|
||||
# Validate Goose migration annotations and SQL parsing without touching a database.
|
||||
migrate-validate:
|
||||
$(GOOSE) -dir $(GOOSE_DIR) validate
|
||||
|
||||
# Show Goose migration status through Silo's bootstrapping runner.
|
||||
migrate-status:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-status
|
||||
|
||||
# Apply pending Goose migrations through Silo's bootstrapping runner.
|
||||
migrate-up:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-only
|
||||
|
||||
# Install repo-local git hooks for this checkout/worktree.
|
||||
install-hooks:
|
||||
@existing="$$(git config --local core.hooksPath 2>/dev/null || true)"; \
|
||||
|
||||
@@ -178,8 +178,36 @@ Technical notes:
|
||||
| `make dev-backend` | Run Go backend (integrated mode) |
|
||||
| `make dev-proxy` | Run a standalone proxy node |
|
||||
| `make dev-transcode` | Run a standalone transcode node |
|
||||
| `make migrate-create NAME=add_thing` | Create a timestamped Goose SQL migration |
|
||||
| `make migrate-validate` | Validate Goose migration files without touching a database |
|
||||
| `make migrate-status` | Show Goose migration status using Silo's bootstrapping runner |
|
||||
| `make migrate-up` | Apply pending Goose migrations using Silo's bootstrapping runner |
|
||||
| `make clean` | Remove build artifacts |
|
||||
|
||||
### Database Migrations
|
||||
|
||||
PostgreSQL schema migrations are managed by Goose. Migration SQL files live in
|
||||
`migrations/sql/` and use Goose annotations. Converted legacy migrations keep
|
||||
their original numeric versions so existing `schema_versions` rows can bootstrap
|
||||
cleanly into Goose without replaying old SQL. New migrations should be created
|
||||
with timestamped filenames:
|
||||
|
||||
```sh
|
||||
make migrate-create NAME=add_thing
|
||||
make migrate-validate
|
||||
```
|
||||
|
||||
Do not run `goose fix`; timestamped migrations are the repository policy because
|
||||
they avoid version collisions across parallel PRs. The existing `001`-style
|
||||
files are historical compatibility records, not the naming pattern for new work.
|
||||
Runtime migrations are applied by the integrated/API server only. Proxy and
|
||||
transcode modes never mutate schema.
|
||||
For existing installs, use `make migrate-status` and `make migrate-up` rather
|
||||
than invoking the Goose CLI directly; those targets copy legacy
|
||||
`schema_versions` rows into `public.goose_db_version` under the migration lock
|
||||
before reading or applying migrations. Set `ENV_FILE=path/to/.env` when the
|
||||
database URL should be read from a non-default env file.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```sh
|
||||
@@ -285,5 +313,5 @@ internal/
|
||||
scanner/ Media file discovery and FFProbe
|
||||
worker/ Background jobs (scan, match, reconcile)
|
||||
web/ React + TypeScript frontend (Vite, Tailwind, shadcn/ui)
|
||||
migrations/ PostgreSQL schema migrations
|
||||
migrations/sql/ Goose-managed PostgreSQL schema migrations
|
||||
```
|
||||
|
||||
+296
-30
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -23,15 +23,16 @@ import (
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
sdkcapability "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/activitylog"
|
||||
"github.com/Silo-Server/silo-server/internal/adminjob"
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks"
|
||||
"github.com/Silo-Server/silo-server/internal/api"
|
||||
"github.com/Silo-Server/silo-server/internal/api/handlers"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
@@ -52,7 +53,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/logfilter"
|
||||
"github.com/Silo-Server/silo-server/internal/logstream"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/markers/introdb"
|
||||
"github.com/Silo-Server/silo-server/internal/mdblist"
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -251,6 +251,8 @@ func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxCon
|
||||
|
||||
func main() {
|
||||
envFile := flag.String("env", ".env", "path to .env bootstrap file")
|
||||
migrateOnly := flag.Bool("migrate-only", false, "apply database migrations and exit")
|
||||
migrateStatus := flag.Bool("migrate-status", false, "show database migration status and exit")
|
||||
flag.Parse()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -270,12 +272,48 @@ func main() {
|
||||
defer pool.Close()
|
||||
slog.Info("connected to PostgreSQL")
|
||||
|
||||
if *migrateStatus {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
statuses, statusErr := database.MigrationStatuses(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if statusErr != nil {
|
||||
log.Fatalf("failed to read migration status: %v", statusErr)
|
||||
}
|
||||
|
||||
fmt.Printf("%-8s %8s %-25s %s\n", "STATE", "VERSION", "APPLIED_AT", "MIGRATION")
|
||||
for _, status := range statuses {
|
||||
appliedAt := "-"
|
||||
if !status.AppliedAt.IsZero() {
|
||||
appliedAt = status.AppliedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
source := status.Source
|
||||
if source != "" {
|
||||
source = filepath.Base(source)
|
||||
} else {
|
||||
source = "-"
|
||||
}
|
||||
fmt.Printf("%-8s %8d %-25s %s\n", status.State, status.Version, appliedAt, source)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if *migrateOnly {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if migErr != nil {
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
slog.Info("database migrations applied")
|
||||
return
|
||||
}
|
||||
|
||||
// Run migrations only for integrated/api modes. Proxy and transcode nodes
|
||||
// should never alter the schema — they may scale independently and would
|
||||
// race or apply migrations before the primary node is deliberately upgraded.
|
||||
if bc.Mode == "integrated" || bc.Mode == "api" || bc.Mode == "" {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "."); migErr != nil {
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql"); migErr != nil {
|
||||
migCancel()
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
@@ -497,36 +535,34 @@ func main() {
|
||||
}
|
||||
if deps.DB != nil {
|
||||
markerRegistry := markers.NewRegistry(slog.Default())
|
||||
introdbAPIKey, err := settingsRepo.Get(appCtx, "introdb.api_key")
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
slog.Warn("load introdb.api_key from settings failed; provider will run with no key",
|
||||
markerProviderConfig := markers.NewProviderConfigStore(deps.DB)
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("load marker provider config failed; falling back to registration-order fetch",
|
||||
"error", err)
|
||||
introdbAPIKey = ""
|
||||
}
|
||||
introdbClient := introdb.NewClient(introdbAPIKey)
|
||||
if err := markerRegistry.Register(introdb.NewProvider(introdbClient)); err != nil {
|
||||
log.Fatalf("register introdb marker provider: %v", err)
|
||||
}
|
||||
deps.OnServerSettingUpdated = func(_ context.Context, key, value string) {
|
||||
if key == "introdb.api_key" {
|
||||
introdbClient.SetAPIKey(value)
|
||||
} else {
|
||||
markerRegistry.UseConfigStore(markerProviderConfig)
|
||||
if deps.EventBus != nil {
|
||||
if err := deps.EventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventMarkerProviderConfigChanged {
|
||||
return
|
||||
}
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("reload marker provider config failed", "provider", event.Payload, "error", err)
|
||||
}
|
||||
}); err != nil {
|
||||
slog.Warn("subscribe marker provider config reload failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if eventBus != nil {
|
||||
_ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventSettingsChanged || event.Payload != "introdb.api_key" {
|
||||
return
|
||||
}
|
||||
value, loadErr := settingsRepo.Get(context.Background(), "introdb.api_key")
|
||||
if loadErr != nil {
|
||||
slog.Warn("introdb api key reload failed", "error", loadErr)
|
||||
return
|
||||
}
|
||||
introdbClient.SetAPIKey(value)
|
||||
})
|
||||
}
|
||||
deps.MarkerProviderConfig = markerProviderConfig
|
||||
deps.MarkerRegistry = markerRegistry
|
||||
deps.MarkerResolver = markers.NewDBExternalIDResolver(deps.DB)
|
||||
markerResolver := markers.NewDBExternalIDResolver(deps.DB)
|
||||
deps.MarkerResolver = markerResolver
|
||||
markerContributionStore := markers.NewContributionStore(deps.DB)
|
||||
deps.MarkerContributionStore = markerContributionStore
|
||||
deps.MarkerContributionService = markers.NewContributionService(
|
||||
markerRegistry, markerResolver, markerProviderConfig, markerContributionStore, slog.Default(),
|
||||
)
|
||||
}
|
||||
var watchProviderService *watchsync.Service
|
||||
if deps.DB != nil {
|
||||
@@ -738,6 +774,22 @@ func main() {
|
||||
installer,
|
||||
plugins.NewHostAdapter(pluginHost),
|
||||
)
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
markerPluginResolver := markers.NewPluginResolverAdapter(pluginService)
|
||||
pluginService.AddLifecycleHook(func(ctx context.Context) {
|
||||
if err := reloadMarkerPluginProviders(
|
||||
ctx,
|
||||
deps.MarkerRegistry,
|
||||
deps.MarkerProviderConfig,
|
||||
installationStore,
|
||||
runtimeConfigStore,
|
||||
settingsRepo,
|
||||
markerPluginResolver,
|
||||
); err != nil {
|
||||
slog.Warn("reload marker plugin providers failed", "error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := pluginService.PreloadEnabled(appCtx); err != nil {
|
||||
log.Fatalf("preload enabled plugins: %v", err)
|
||||
}
|
||||
@@ -1356,6 +1408,11 @@ func main() {
|
||||
if deps.IntroAnalyzer != nil {
|
||||
taskMgr.Register(tasks.NewDetectIntroMarkersTask(deps.IntroAnalyzer, settingsRepo))
|
||||
}
|
||||
if deps.MarkerContributionService != nil && deps.MarkerProviderConfig != nil && deps.MarkerContributionStore != nil && deps.FileRepo != nil {
|
||||
taskMgr.Register(tasks.NewContributeMarkersTask(
|
||||
deps.MarkerContributionService, deps.MarkerProviderConfig, deps.MarkerContributionStore, deps.FileRepo,
|
||||
))
|
||||
}
|
||||
if chapterBackfiller, ok := deps.ChapterThumbnailQueuer.(*chapterthumbs.Service); ok {
|
||||
taskMgr.Register(tasks.NewChapterThumbnailBackfillTask(chapterBackfiller, 25))
|
||||
}
|
||||
@@ -1447,6 +1504,12 @@ func main() {
|
||||
slog.Info("task manager started")
|
||||
}
|
||||
|
||||
// Scaffold audiobooks service (sub-plan 1: kill-switch reader only).
|
||||
// audiobooksSettingsAdapter (defined at package level below) bridges
|
||||
// catalog.ServerSettingsRepo.Get to audiobooks.SettingsReader.GetString.
|
||||
audiobooksService := audiobooks.New(&audiobooksSettingsAdapter{repo: settingsRepo})
|
||||
_ = audiobooksService // sub-plan 2+ will register routes / scheduled tasks
|
||||
|
||||
if deps.DB != nil && pluginInstallationStore != nil && pluginRuntimeConfigStore != nil && deps.PluginService != nil {
|
||||
userRepo := auth.NewUserRepository(deps.DB)
|
||||
sessionRepo := auth.NewSessionRepository(deps.DB)
|
||||
@@ -2003,6 +2066,198 @@ func configureS3Clients(cfg *config.Config, deps *api.Dependencies) {
|
||||
}
|
||||
}
|
||||
|
||||
type markerPluginCapabilityStore interface {
|
||||
ListEnabled(ctx context.Context) ([]*plugins.Installation, error)
|
||||
ListCapabilities(ctx context.Context, installationID int) ([]*plugins.Capability, error)
|
||||
}
|
||||
|
||||
type markerPluginRuntimeConfigStore interface {
|
||||
ListGlobalConfigs(ctx context.Context, installationID int) ([]*plugins.RuntimeConfig, error)
|
||||
PutGlobalConfig(ctx context.Context, installationID int, key string, value map[string]any) error
|
||||
}
|
||||
|
||||
type markerLegacySettingsStore interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
func reloadMarkerPluginProviders(
|
||||
ctx context.Context,
|
||||
registry *markers.Registry,
|
||||
configStore *markers.ProviderConfigStore,
|
||||
store markerPluginCapabilityStore,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
resolver *markers.PluginResolverAdapter,
|
||||
) error {
|
||||
if registry == nil {
|
||||
return nil
|
||||
}
|
||||
var providers []markers.Provider
|
||||
if store == nil || resolver == nil {
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
installations, err := store.ListEnabled(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list enabled plugin installations: %w", err)
|
||||
}
|
||||
sort.Slice(installations, func(i, j int) bool {
|
||||
if installations[i] == nil {
|
||||
return false
|
||||
}
|
||||
if installations[j] == nil {
|
||||
return true
|
||||
}
|
||||
return installations[i].ID < installations[j].ID
|
||||
})
|
||||
|
||||
nextPriority := 1000
|
||||
for _, installation := range installations {
|
||||
if installation == nil {
|
||||
continue
|
||||
}
|
||||
capabilities, err := store.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err)
|
||||
}
|
||||
sort.Slice(capabilities, func(i, j int) bool {
|
||||
if capabilities[i] == nil {
|
||||
return false
|
||||
}
|
||||
if capabilities[j] == nil {
|
||||
return true
|
||||
}
|
||||
return capabilities[i].ID < capabilities[j].ID
|
||||
})
|
||||
for _, capability := range capabilities {
|
||||
if capability == nil || capability.Type != sdkcapability.MarkerProvider {
|
||||
continue
|
||||
}
|
||||
descriptor, err := plugins.DecodeCapability(capability)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode marker provider capability %d/%s: %w", installation.ID, capability.ID, err)
|
||||
}
|
||||
metadataMap := markerCapabilityMetadata(descriptor)
|
||||
provider, err := markers.NewPluginProvider(markers.PluginProviderOptions{
|
||||
InstallationID: installation.ID,
|
||||
CapabilityID: capability.ID,
|
||||
DisplayName: firstNonEmptyMarkerText(descriptor.GetDisplayName(), capability.ID),
|
||||
PluginID: installation.PluginID,
|
||||
RequiredExternalIDs: markers.PluginRequiredExternalIDsFromMetadata(metadataMap),
|
||||
}, resolver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
providers = append(providers, provider)
|
||||
|
||||
priority := nextPriority
|
||||
nextPriority++
|
||||
if configuredPriority, ok := markers.PluginDefaultFetchPriorityFromMetadata(metadataMap); ok {
|
||||
priority = configuredPriority
|
||||
}
|
||||
if configStore != nil {
|
||||
defaultConfig := markers.ProviderConfig{
|
||||
Provider: provider.ID(),
|
||||
FetchEnabled: true,
|
||||
FetchPriority: priority,
|
||||
ContributeEnabled: false,
|
||||
ContributeAutoLocal: false,
|
||||
ContributeMinConfidence: 0.95,
|
||||
}
|
||||
if legacy, ok := legacyIntroDBProviderConfig(configStore, installation, capability, provider.ID()); ok {
|
||||
defaultConfig = legacy
|
||||
}
|
||||
if err := configStore.Ensure(ctx, defaultConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := copyLegacyIntroDBPluginConfig(ctx, runtimeConfigs, legacySettings, installation, capability); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
func legacyIntroDBProviderConfig(
|
||||
configStore *markers.ProviderConfigStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
providerID string,
|
||||
) (markers.ProviderConfig, bool) {
|
||||
if configStore == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
if _, exists := configStore.Get(providerID); exists {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy, ok := configStore.Get("introdb")
|
||||
if !ok {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy.Provider = providerID
|
||||
return legacy, true
|
||||
}
|
||||
|
||||
func copyLegacyIntroDBPluginConfig(
|
||||
ctx context.Context,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
) error {
|
||||
if runtimeConfigs == nil ||
|
||||
legacySettings == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return nil
|
||||
}
|
||||
configs, err := runtimeConfigs.ListGlobalConfigs(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list TheIntroDB plugin config: %w", err)
|
||||
}
|
||||
for _, config := range configs {
|
||||
if config != nil && config.Key == "account" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
apiKey, err := legacySettings.Get(ctx, "introdb.api_key")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load legacy introdb.api_key: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(apiKey) == "" {
|
||||
return nil
|
||||
}
|
||||
if err := runtimeConfigs.PutGlobalConfig(ctx, installation.ID, "account", map[string]any{
|
||||
"api_key": strings.TrimSpace(apiKey),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("copy legacy introdb.api_key to plugin config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markerCapabilityMetadata(descriptor *pluginv1.CapabilityDescriptor) map[string]any {
|
||||
if descriptor == nil || descriptor.GetMetadata() == nil {
|
||||
return nil
|
||||
}
|
||||
return descriptor.GetMetadata().AsMap()
|
||||
}
|
||||
|
||||
func firstNonEmptyMarkerText(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mapFolderTypeToMediaType maps silo's MediaFolder.Type values
|
||||
// ("movies", "series", "mixed") to the SDK's MediaType values
|
||||
// ("movie", "tv", "mixed"). Unknown values map to "mixed".
|
||||
@@ -2016,3 +2271,14 @@ func mapFolderTypeToMediaType(t string) string {
|
||||
return "mixed"
|
||||
}
|
||||
}
|
||||
|
||||
// audiobooksSettingsAdapter bridges catalog.ServerSettingsRepo (which
|
||||
// exposes Get) to the audiobooks.SettingsReader interface (which
|
||||
// requires GetString). The two signatures are identical modulo name.
|
||||
type audiobooksSettingsAdapter struct {
|
||||
repo *catalog.ServerSettingsRepo
|
||||
}
|
||||
|
||||
func (a *audiobooksSettingsAdapter) GetString(ctx context.Context, key string) (string, error) {
|
||||
return a.repo.Get(ctx, key)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
type fakeMarkerRuntimeConfigs struct {
|
||||
configs map[int][]*plugins.RuntimeConfig
|
||||
puts []plugins.RuntimeConfig
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) ListGlobalConfigs(_ context.Context, installationID int) ([]*plugins.RuntimeConfig, error) {
|
||||
return append([]*plugins.RuntimeConfig(nil), f.configs[installationID]...), nil
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) PutGlobalConfig(_ context.Context, installationID int, key string, value map[string]any) error {
|
||||
f.puts = append(f.puts, plugins.RuntimeConfig{InstallationID: installationID, Key: key, Value: value})
|
||||
f.configs[installationID] = append(f.configs[installationID], &plugins.RuntimeConfig{
|
||||
InstallationID: installationID,
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeMarkerLegacySettings map[string]string
|
||||
|
||||
func (f fakeMarkerLegacySettings) Get(_ context.Context, key string) (string, error) {
|
||||
return f[key], nil
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigCopiesAPIKeyOnce(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
installation := &plugins.Installation{ID: 42, PluginID: "silo.theintrodb"}
|
||||
capability := &plugins.Capability{ID: "introdb"}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": " legacy-key "},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
got := runtimeConfigs.puts[0]
|
||||
if got.InstallationID != 42 || got.Key != "account" || got.Value["api_key"] != "legacy-key" {
|
||||
t.Fatalf("put = %+v, want account api_key copy", got)
|
||||
}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "new-key"},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("second copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("second copy overwrote config; puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigIgnoresOtherPlugins(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "legacy-key"},
|
||||
&plugins.Installation{ID: 7, PluginID: "silo.other"},
|
||||
&plugins.Capability{ID: "introdb"},
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 0 {
|
||||
t.Fatalf("puts = %d, want 0 for non-TheIntroDB plugin", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,916 @@
|
||||
# Audiobooks Absorption — Sub-plan 1: Discovery + Schema
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Resolve open data-model questions from the audiobooks spec, then land the foundational schema (two new tables, optional one column add, one feature-flag setting) and a compiling-but-empty `internal/audiobooks/` package scaffold. No user-visible behavior change after this sub-plan; silo continues to operate exactly as it does today.
|
||||
|
||||
**Architecture:** Six tasks total. Task 1 audits silo-server's existing schema/code and writes a findings doc that locks the remaining decisions. Tasks 2–5 land additive SQL migrations (idempotent up + symmetric down). Task 6 creates a Go package skeleton with one no-op service constructor wired into the dependency-injection point in `cmd/silo`, which proves the new package compiles inside the rest of the binary without changing any runtime behavior.
|
||||
|
||||
**Tech Stack:** Go 1.26 + pgx; PostgreSQL 18 (pgvector image); plain `.up.sql` / `.down.sql` migrations applied at startup by `internal/database/migrate.go`. Frontend out of scope for this sub-plan.
|
||||
|
||||
**Source spec:** `docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md`
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
| Path | Created/Modified | Purpose |
|
||||
|---|---|---|
|
||||
| `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` | Create | Findings doc produced by Task 1. Locks the remaining schema/code decisions. |
|
||||
| `migrations/147_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/147_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/157_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/157_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/159_media_folders_kind_noop.up.sql` | Create (conditional — see Task 4 step 1) | Document `media_folders.type='audiobooks'`; no column change needed. |
|
||||
| `migrations/159_media_folders_kind_noop.down.sql` | Create (same condition) | No-op rollback. |
|
||||
| `migrations/160_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/160_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `internal/audiobooks/doc.go` | Create | Package comment + intent. |
|
||||
| `internal/audiobooks/service.go` | Create | Empty `Service` struct, `New` constructor, `Enabled()` reader. |
|
||||
| `internal/audiobooks/service_test.go` | Create | One unit test proving the package compiles and `Enabled()` returns the setting value. |
|
||||
| `cmd/silo/main.go` | Modify | Construct `audiobooks.New(...)` once during startup so the scaffold is referenced. No routes are mounted, no tasks registered. |
|
||||
|
||||
Migrations and the scaffold compile and run independently from the audiobooks ports that arrive in later sub-plans.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Discovery Audit
|
||||
|
||||
Audits silo-server's existing schema and code to resolve the spec's open
|
||||
Risk questions. Produces one findings document and one commit. No code
|
||||
changes elsewhere.
|
||||
|
||||
**Files:**
|
||||
- Create: `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md`
|
||||
|
||||
### Step 1.1: Create the findings doc skeleton
|
||||
|
||||
- [ ] **Create** `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` with this exact content:
|
||||
|
||||
```markdown
|
||||
# Audiobooks Absorption — Discovery Findings
|
||||
|
||||
Produced by sub-plan 1, Task 1. Locks data-model and integration
|
||||
decisions for migrations 139–142 and downstream sub-plans.
|
||||
|
||||
## D1 — Next migration number
|
||||
|
||||
(Filled in step 1.2)
|
||||
|
||||
## D2 — `media_libraries` kind/type column
|
||||
|
||||
(Filled in step 1.3)
|
||||
|
||||
## D3 — `media_files.chapters` JSONB shape
|
||||
|
||||
(Filled in step 1.4)
|
||||
|
||||
## D4 — `user_watch_progress` scoping (profile vs user)
|
||||
|
||||
(Filled in step 1.5)
|
||||
|
||||
## D5 — `user_playback_sessions` audiobook fit
|
||||
|
||||
(Filled in step 1.6)
|
||||
|
||||
## D6 — `people` / `item_people` role conventions
|
||||
|
||||
(Filled in step 1.7)
|
||||
|
||||
## D7 — Catalog FTS handling of `type='audiobook'`
|
||||
|
||||
(Filled in step 1.8)
|
||||
|
||||
## D8 — First-party scheduled-task registration
|
||||
|
||||
(Filled in step 1.9)
|
||||
```
|
||||
|
||||
### Step 1.2: D1 — Migration number
|
||||
|
||||
- [ ] **Run:** `ls migrations/ | grep -E '^[0-9]+_' | sort -t_ -k1 -n | tail -3`
|
||||
|
||||
Expected output is three filenames whose numeric prefixes are consecutive.
|
||||
Whatever the highest prefix is, the next available number is **(highest + 1)**.
|
||||
|
||||
- [ ] **Append to the findings doc** under the D1 heading:
|
||||
|
||||
```text
|
||||
Highest existing migration: <NN>_<name>.up.sql
|
||||
Next available number: <NN+1>
|
||||
Sub-plan 1 will use migrations <NN+1> through <NN+4>.
|
||||
```
|
||||
|
||||
(Replace the placeholders with the observed values.) This locks the
|
||||
numbering used by Tasks 2–5. If the next number is not 139, update the
|
||||
filenames in Tasks 2–5 accordingly when you reach them.
|
||||
|
||||
### Step 1.3: D2 — `media_libraries` kind/type column
|
||||
|
||||
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.media_libraries"`
|
||||
|
||||
Inspect the column list. Look for any column whose name or comment
|
||||
suggests a per-library *kind* / *type* / *category* discriminator
|
||||
(candidate names: `kind`, `library_type`, `category`, `media_type`,
|
||||
`content_type`).
|
||||
|
||||
- [ ] **Append to the findings doc** under the D2 heading exactly one of:
|
||||
|
||||
- `No existing kind/type column on media_libraries. Task 4 adds 'kind' as planned.`
|
||||
- `Existing column '<NAME>' (<TYPE>) discriminates library content. Task 4 becomes a no-op migration; the audiobook scanner branch will read '<NAME>' instead of 'kind'.`
|
||||
|
||||
### Step 1.4: D3 — `media_files.chapters` JSONB shape
|
||||
|
||||
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT chapters FROM media_files WHERE chapters IS NOT NULL AND jsonb_array_length(chapters) > 0 LIMIT 1;"`
|
||||
|
||||
This returns one sample chapter array used by silo's existing video
|
||||
chapter feature (added in migration 066).
|
||||
|
||||
- [ ] **Append to the findings doc** under the D3 heading:
|
||||
|
||||
```text
|
||||
Sample chapter JSON (live data):
|
||||
<paste the JSON exactly as returned, no edits>
|
||||
|
||||
Sub-plan 2 (scanner) MUST emit objects with the same keys when writing
|
||||
audiobook chapters so the existing player and serialization code accept
|
||||
them without changes.
|
||||
```
|
||||
|
||||
### Step 1.5: D4 — `user_watch_progress` scoping
|
||||
|
||||
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.user_watch_progress"`
|
||||
|
||||
- [ ] **Append to the findings doc** under the D4 heading exactly one of:
|
||||
|
||||
- `user_watch_progress is profile-scoped: column '<profile_id>' (FK to user_profiles). Audiobook progress slots in directly.`
|
||||
- `user_watch_progress is user-scoped only (no profile column). Audiobook progress will share user state across all household profiles unless Sub-plan 3 adds a profile_id column. RAISE THIS AS A RISK in sub-plan 2 brainstorming.`
|
||||
|
||||
### Step 1.6: D5 — `user_playback_sessions` fit
|
||||
|
||||
- [ ] **Run:** `sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.user_playback_sessions"`
|
||||
|
||||
- [ ] **Append to the findings doc** under the D5 heading:
|
||||
|
||||
```text
|
||||
Column list:
|
||||
<paste the \d output verbatim>
|
||||
|
||||
Columns required by audiobook sessions: media_item_id (or equivalent),
|
||||
profile/user FK, started_at, current_position_seconds (or equivalent),
|
||||
status. Mark any required column as MISSING and surface in sub-plan 3.
|
||||
```
|
||||
|
||||
### Step 1.7: D6 — `people` / `item_people` role conventions
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.item_people"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT DISTINCT role FROM item_people ORDER BY role;"
|
||||
```
|
||||
|
||||
- [ ] **Append to the findings doc** under the D6 heading:
|
||||
|
||||
```text
|
||||
item_people.role storage: <TEXT | enum named '<name>' | CHECK constraint>
|
||||
Existing role values in use: <comma-separated list from the SELECT>
|
||||
|
||||
Sub-plan 2 will UPSERT 'author' and 'narrator' into item_people for
|
||||
audiobook items. If the role column is constrained by an enum or CHECK,
|
||||
Sub-plan 2 must add 'author' and 'narrator' to the constraint as part of
|
||||
its first migration.
|
||||
```
|
||||
|
||||
### Step 1.8: D7 — FTS handling of new `type` values
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
grep -lE "type\s*=\s*'(movie|series|episode)'" migrations/*.up.sql | head -5
|
||||
```
|
||||
|
||||
These migrations build/maintain silo's title search. Open each one and
|
||||
scan for whether the type filter is hardcoded (`type IN ('movie','series')`)
|
||||
or generic (no type filter, indexes everything).
|
||||
|
||||
- [ ] **Append to the findings doc** under the D7 heading:
|
||||
|
||||
```text
|
||||
Indexes / generated columns that filter by media_items.type:
|
||||
<bulleted list of (migration_file, what it filters)>
|
||||
|
||||
Verdict: audiobooks WILL / WILL NOT be FTS-searchable out of the box.
|
||||
|
||||
If WILL NOT: Sub-plan 3 must include an extra migration that extends the
|
||||
type filter to include 'audiobook' and 'podcast'.
|
||||
```
|
||||
|
||||
### Step 1.9: D8 — Scheduled-task registration
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
grep -rEn "RegisterTask|TaskManager|RegisterScheduled|task\.Register" cmd/silo/main.go internal --include='*.go' | head -10
|
||||
```
|
||||
|
||||
- [ ] **Append to the findings doc** under the D8 heading:
|
||||
|
||||
```text
|
||||
First-party scheduled tasks register at: <file:line>
|
||||
Registration call shape: <one-line copy of the call as used by an
|
||||
existing first-party task such as smart-count refresh>
|
||||
|
||||
Sub-plan 5 (podcasts) will register podcastfeed.Refresher at the same
|
||||
call site using the same pattern.
|
||||
```
|
||||
|
||||
### Step 1.10: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md
|
||||
git commit -m "$(cat <<'EOF'
|
||||
docs(audiobooks): discovery findings for absorption sub-plan 1
|
||||
|
||||
Locks schema/code decisions for migrations 139-142 and downstream
|
||||
sub-plans. Resolves open Risk questions from the absorption design spec.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migration 147 — `abs_sessions`
|
||||
|
||||
Parallel of `jellycompat_sessions` for Audiobookshelf clients. Lets ABS
|
||||
mobile/desktop apps reconnect without re-authenticating against silo's
|
||||
main `auth_sessions`.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/147_abs_sessions.up.sql`
|
||||
- Create: `migrations/147_abs_sessions.down.sql`
|
||||
|
||||
> **Note:** If Task 1, Step 1.2 found the next number is not 139, rename
|
||||
> both filenames to match. All other tasks downstream of 139 shift by the
|
||||
> same delta.
|
||||
|
||||
### Step 2.1: Write the failing assertion
|
||||
|
||||
- [ ] **Run** (this should error — the table does not exist yet):
|
||||
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
|
||||
```
|
||||
|
||||
Expected: `Did not find any relation named "public.abs_sessions".`
|
||||
|
||||
### Step 2.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/147_abs_sessions.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Audiobookshelf-compatible client sessions. Parallel to
|
||||
-- jellycompat_sessions: each row identifies an ABS mobile/desktop
|
||||
-- client by its device + token so it can reconnect without
|
||||
-- re-authenticating against silo's main auth_sessions.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.abs_sessions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
device_name TEXT,
|
||||
client_name TEXT,
|
||||
client_version TEXT,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
||||
last_seen_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
||||
revoked_at TIMESTAMP WITH TIME ZONE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token_hash
|
||||
ON public.abs_sessions (token_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_abs_sessions_user_device
|
||||
ON public.abs_sessions (user_id, device_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_abs_sessions_last_seen
|
||||
ON public.abs_sessions (last_seen_at);
|
||||
```
|
||||
|
||||
### Step 2.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/147_abs_sessions.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_last_seen;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_user_device;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token_hash;
|
||||
DROP TABLE IF EXISTS public.abs_sessions;
|
||||
```
|
||||
|
||||
### Step 2.4: Apply the migration by restarting silo
|
||||
|
||||
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
|
||||
|
||||
- [ ] **Wait until healthy:**
|
||||
```bash
|
||||
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
|
||||
```
|
||||
|
||||
- [ ] **Verify silo applied the migration:**
|
||||
```bash
|
||||
sudo docker logs --since 2m silo-prod-silo-1 2>&1 | grep -i 'database migrations applied'
|
||||
```
|
||||
Expected: one line containing `database migrations applied`.
|
||||
|
||||
### Step 2.5: Verify the table exists and is usable
|
||||
|
||||
- [ ] **Run** (each command should now succeed):
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 147;"
|
||||
```
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=147`.
|
||||
|
||||
- [ ] **Run** a sample insert/delete to prove constraints work:
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
INSERT INTO abs_sessions (user_id, token_hash, device_id, device_name, client_name, client_version)
|
||||
VALUES ((SELECT id FROM users ORDER BY id LIMIT 1), 'test-token-hash-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token_hash = 'test-token-hash-001';
|
||||
"
|
||||
```
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (At least one user must exist.)
|
||||
|
||||
### Step 2.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/147_abs_sessions.up.sql migrations/147_abs_sessions.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 147 add abs_sessions table
|
||||
|
||||
Parallel of jellycompat_sessions for Audiobookshelf-compatible clients.
|
||||
Lets ABS mobile/desktop apps maintain a device-bound session that
|
||||
silo's audiobooks/abs handlers will validate.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Migration 157 — `podcast_feeds`
|
||||
|
||||
One row per subscribed podcast. The podcast itself lives in `media_items`
|
||||
(`type='podcast'`); this side table carries the RSS-specific metadata the
|
||||
feed refresher needs.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/157_podcast_feeds.up.sql`
|
||||
- Create: `migrations/157_podcast_feeds.down.sql`
|
||||
|
||||
### Step 3.1: Write the failing assertion
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
|
||||
```
|
||||
Expected: `Did not find any relation named "public.podcast_feeds".`
|
||||
|
||||
### Step 3.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- RSS-feed metadata for subscribed podcasts. One row per podcast
|
||||
-- media_items row; the feed refresher polls feed_url every
|
||||
-- refresh_interval_seconds and upserts new episodes into the existing
|
||||
-- episodes table.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.podcast_feeds (
|
||||
media_item_id TEXT PRIMARY KEY
|
||||
REFERENCES public.media_items(content_id) ON DELETE CASCADE,
|
||||
feed_url TEXT NOT NULL,
|
||||
etag TEXT,
|
||||
last_modified TEXT,
|
||||
last_refreshed_at TIMESTAMP WITH TIME ZONE,
|
||||
last_refresh_error TEXT,
|
||||
refresh_interval_seconds INTEGER NOT NULL DEFAULT 600,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_podcast_feeds_feed_url
|
||||
ON public.podcast_feeds (feed_url);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_podcast_feeds_due_for_refresh
|
||||
ON public.podcast_feeds (last_refreshed_at);
|
||||
```
|
||||
|
||||
> **Note:** `media_items.content_id` is TEXT (silo uses snowflake-style
|
||||
> string IDs, not BIGSERIAL — confirmed by the FK targets in other
|
||||
> migrations). If Task 1 Step 1.4 sampled a numeric `content_id`,
|
||||
> change the column type to match. Otherwise leave as TEXT.
|
||||
|
||||
### Step 3.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_podcast_feeds_due_for_refresh;
|
||||
DROP INDEX IF EXISTS public.idx_podcast_feeds_feed_url;
|
||||
DROP TABLE IF EXISTS public.podcast_feeds;
|
||||
```
|
||||
|
||||
### Step 3.4: Apply the migration
|
||||
|
||||
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
|
||||
|
||||
- [ ] **Wait until healthy:**
|
||||
```bash
|
||||
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
|
||||
```
|
||||
|
||||
### Step 3.5: Verify
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 157;"
|
||||
```
|
||||
Expected: column list printed; one row with `version=157`.
|
||||
|
||||
### Step 3.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/157_podcast_feeds.up.sql migrations/157_podcast_feeds.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 157 add podcast_feeds table
|
||||
|
||||
Side table on media_items for RSS-subscribed podcasts. Holds feed URL,
|
||||
ETag/Last-Modified for conditional fetches, last-refresh timestamp, and
|
||||
the per-feed refresh interval consumed by the upcoming
|
||||
podcastfeed.Refresher scheduled task.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Migration 159 — `media_libraries.kind` audiobook value (no-op)
|
||||
|
||||
Adds a discriminator column so the scanner can branch on
|
||||
`audiobooks`/`podcasts` libraries without filename heuristics.
|
||||
|
||||
> **CONDITIONAL:** If Task 1 Step 1.3 (D2) recorded *"Existing column
|
||||
> '<NAME>' discriminates library content"*, swap this whole task for a
|
||||
> documented no-op: create both migration files as no-op SQL (`-- intentionally empty: existing column '<NAME>' covers this need`)
|
||||
> and adjust downstream sub-plans to read `'<NAME>'` instead of `kind`.
|
||||
> Otherwise, proceed with the steps below.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/159_media_folders_kind_noop.up.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.down.sql`
|
||||
|
||||
### Step 4.1: Write the failing assertion
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT kind FROM media_libraries LIMIT 1;"
|
||||
```
|
||||
Expected: `column "kind" does not exist`.
|
||||
|
||||
### Step 4.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Per-library content discriminator. Lets the scanner choose the right
|
||||
-- parser (movie/tv extensions vs audiobook/podcast extensions) without
|
||||
-- filename heuristics. Existing rows default to 'movies' to preserve
|
||||
-- current behavior; operators set 'audiobooks' or 'podcasts' on the
|
||||
-- libraries they want the new flavor for.
|
||||
|
||||
ALTER TABLE public.media_libraries
|
||||
ADD COLUMN IF NOT EXISTS kind TEXT NOT NULL DEFAULT 'movies';
|
||||
|
||||
ALTER TABLE public.media_libraries
|
||||
ADD CONSTRAINT media_libraries_kind_check
|
||||
CHECK (kind IN ('movies', 'tv', 'audiobooks', 'podcasts'));
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_media_libraries_kind
|
||||
ON public.media_libraries (kind);
|
||||
```
|
||||
|
||||
### Step 4.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_media_libraries_kind;
|
||||
ALTER TABLE public.media_libraries
|
||||
DROP CONSTRAINT IF EXISTS media_libraries_kind_check;
|
||||
ALTER TABLE public.media_libraries
|
||||
DROP COLUMN IF EXISTS kind;
|
||||
```
|
||||
|
||||
### Step 4.4: Apply the migration
|
||||
|
||||
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
|
||||
- [ ] **Wait until healthy** (same command as Task 2 Step 2.4).
|
||||
|
||||
### Step 4.5: Verify
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
SELECT id, name, kind FROM media_libraries ORDER BY id;
|
||||
"
|
||||
```
|
||||
Expected: every existing library row has `kind='movies'`.
|
||||
|
||||
- [ ] **Run** to confirm the CHECK rejects bad values:
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
INSERT INTO media_libraries (id, name, kind) VALUES (-9999, 'bad-test', 'garbage');
|
||||
"
|
||||
```
|
||||
Expected: `ERROR: new row for relation "media_libraries" violates check constraint "media_libraries_kind_check"`.
|
||||
|
||||
- [ ] **Run** to clean up any test row that snuck through (no-op if the previous step rejected as expected):
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "DELETE FROM media_libraries WHERE id = -9999;"
|
||||
```
|
||||
|
||||
### Step 4.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/159_media_folders_kind_noop.up.sql migrations/159_media_folders_kind_noop.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 159 document audiobooks media folder type
|
||||
|
||||
Per-library discriminator that lets the scanner pick the right parser
|
||||
('movies', 'tv', 'audiobooks', 'podcasts') instead of inferring from
|
||||
filename. Existing libraries default to 'movies'; operators set the new
|
||||
values on the libraries they want indexed with audiobook/podcast logic.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Migration 160 — `audiobooks.enabled` feature flag
|
||||
|
||||
Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
"audiobooks compiled in but turned off" vs "fully live". Default is
|
||||
`false` so this sub-plan's landing is a strict no-op for users.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/160_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.down.sql`
|
||||
|
||||
### Step 5.1: Write the failing assertion
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
SELECT value FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
"
|
||||
```
|
||||
Expected: zero rows.
|
||||
|
||||
### Step 5.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Master kill-switch for the absorbed audiobooks feature. Defaults to
|
||||
-- 'false' so landing migrations 147/157/159/160 is a no-op for users; operators
|
||||
-- flip this to 'true' at cutover.
|
||||
|
||||
INSERT INTO server_settings (key, value) VALUES ('audiobooks.enabled', 'false')
|
||||
ON CONFLICT (key) DO NOTHING;
|
||||
```
|
||||
|
||||
### Step 5.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.down.sql` with:
|
||||
|
||||
```sql
|
||||
DELETE FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
```
|
||||
|
||||
### Step 5.4: Apply and verify
|
||||
|
||||
- [ ] **Run:** `sudo docker compose -p silo-prod restart silo`
|
||||
- [ ] **Wait until healthy** (same command).
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
SELECT key, value FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
"
|
||||
```
|
||||
Expected: one row, `value='false'`.
|
||||
|
||||
### Step 5.5: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/160_audiobooks_feature_flag.up.sql migrations/160_audiobooks_feature_flag.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 160 add audiobooks.enabled flag
|
||||
|
||||
Server-settings row that gates the absorbed audiobooks feature.
|
||||
Defaults to 'false' so sub-plan 1 lands as a strict no-op; subsequent
|
||||
sub-plans branch on this flag and operators flip it to 'true' at
|
||||
cutover.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 6: Scaffold `internal/audiobooks/` package
|
||||
|
||||
Creates an empty but compiling Go package, wires its constructor into
|
||||
`cmd/silo/main.go` so the package is referenced from the binary, and
|
||||
ships a unit test that proves the `Enabled()` reader returns the flag
|
||||
value seeded by migration 160.
|
||||
|
||||
This task proves the package compiles inside the rest of silo without
|
||||
changing any runtime behavior. No routes are mounted. No scheduled
|
||||
tasks are registered. No DB writes happen.
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/audiobooks/doc.go`
|
||||
- Create: `internal/audiobooks/service.go`
|
||||
- Create: `internal/audiobooks/service_test.go`
|
||||
- Modify: `cmd/silo/main.go` (one new constructor call near the other service constructions)
|
||||
|
||||
### Step 6.1: Write the failing test
|
||||
|
||||
- [ ] **Create** `internal/audiobooks/service_test.go` with:
|
||||
|
||||
```go
|
||||
package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeSettingsReader struct {
|
||||
value string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSettingsReader) GetString(_ context.Context, key string) (string, error) {
|
||||
if key != "audiobooks.enabled" {
|
||||
return "", errors.New("unexpected key: " + key)
|
||||
}
|
||||
return f.value, f.err
|
||||
}
|
||||
|
||||
func TestServiceEnabledReadsFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
stored string
|
||||
want bool
|
||||
}{
|
||||
{"flag true", "true", true},
|
||||
{"flag false", "false", false},
|
||||
{"flag empty defaults false", "", false},
|
||||
{"flag garbage defaults false", "yes-please", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
svc := New(&fakeSettingsReader{value: tc.stored})
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("Enabled = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledPropagatesError(t *testing.T) {
|
||||
wantErr := errors.New("db down")
|
||||
svc := New(&fakeSettingsReader{err: wantErr})
|
||||
_, err := svc.Enabled(context.Background())
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Enabled error = %v, want %v wrapped", err, wantErr)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Step 6.2: Run the test to verify it fails
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
|
||||
Expected: build failure with messages like `undefined: New` and
|
||||
`package audiobooks not found`.
|
||||
|
||||
### Step 6.3: Write the package doc
|
||||
|
||||
- [ ] **Create** `internal/audiobooks/doc.go` with:
|
||||
|
||||
```go
|
||||
// Package audiobooks owns silo's first-party audiobook + podcast feature,
|
||||
// absorbed from the historical silo-plugin-audiobooks. Sub-plan 1 lands
|
||||
// only the package scaffold and the kill-switch reader; ABS-compat REST,
|
||||
// Socket.io, scanner branches, podcast feed refresh, and the silo SPA
|
||||
// pages arrive in later sub-plans.
|
||||
//
|
||||
// See docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md
|
||||
// for the design and docs/superpowers/plans/2026-05-24-audiobooks-*.md
|
||||
// for the staged implementation plans.
|
||||
package audiobooks
|
||||
```
|
||||
|
||||
### Step 6.4: Write the service
|
||||
|
||||
- [ ] **Create** `internal/audiobooks/service.go` with:
|
||||
|
||||
```go
|
||||
package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SettingsReader is the minimal slice of the server-settings store that
|
||||
// the audiobooks service needs. The production implementation is
|
||||
// internal/serversettings.Store (or whatever silo names that helper at
|
||||
// wiring time); tests pass a fake.
|
||||
type SettingsReader interface {
|
||||
GetString(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
// Service is the audiobooks feature's top-level orchestrator. Sub-plan 1
|
||||
// exposes only Enabled(); subsequent sub-plans hang additional methods
|
||||
// off Service as new capabilities (scanner branches, ABS handlers, etc.)
|
||||
// come online.
|
||||
type Service struct {
|
||||
settings SettingsReader
|
||||
}
|
||||
|
||||
// New constructs a Service. The constructor takes the dependencies it
|
||||
// will actually use; current sub-plan needs only the settings reader.
|
||||
func New(settings SettingsReader) *Service {
|
||||
return &Service{settings: settings}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
return false, nil
|
||||
}
|
||||
value, err := s.settings.GetString(ctx, "audiobooks.enabled")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("audiobooks: read audiobooks.enabled: %w", err)
|
||||
}
|
||||
return value == "true", nil
|
||||
}
|
||||
```
|
||||
|
||||
### Step 6.5: Run the test to verify it passes
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
Expected: `ok github.com/Silo-Server/silo-server/internal/audiobooks ...`
|
||||
|
||||
### Step 6.6: Wire the constructor into `cmd/silo/main.go`
|
||||
|
||||
The audit in Task 1 Step 1.9 located the function where first-party
|
||||
services are constructed at startup. Open `cmd/silo/main.go`, find the
|
||||
block where other internal services (e.g. metadata, downloads, catalog)
|
||||
are constructed and pass a settings store to.
|
||||
|
||||
- [ ] **Run** to find the right insertion point:
|
||||
```bash
|
||||
grep -nE 'serversettings|server_settings|settingsStore|metadata\.NewService|downloads\.NewService' cmd/silo/main.go | head -10
|
||||
```
|
||||
Note the file:line of the existing service construction block.
|
||||
|
||||
- [ ] **Modify** `cmd/silo/main.go`: in the service-construction block (immediately after another similar `service := pkg.New(...)` line), add:
|
||||
|
||||
```go
|
||||
audiobooksService := audiobooks.New(settingsStore)
|
||||
_ = audiobooksService // referenced by sub-plan 2 onward; no behavior in sub-plan 1
|
||||
```
|
||||
|
||||
Where `settingsStore` is whatever the existing services pass in for
|
||||
the server-settings reader. The `_ = audiobooksService` line keeps Go
|
||||
from rejecting an unused variable; it gets removed in sub-plan 2 when
|
||||
the variable is actually used.
|
||||
|
||||
- [ ] **Add** the import to the import block at the top of `cmd/silo/main.go`:
|
||||
|
||||
```go
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks"
|
||||
```
|
||||
|
||||
### Step 6.7: Verify everything still compiles and tests pass
|
||||
|
||||
- [ ] **Run:** `go build ./...`
|
||||
Expected: exit 0, no output.
|
||||
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/... ./cmd/...`
|
||||
Expected: all passes.
|
||||
|
||||
- [ ] **Run:** `make lint`
|
||||
Expected: no new lint findings introduced by this task.
|
||||
|
||||
### Step 6.8: Smoke-test the running binary
|
||||
|
||||
- [ ] **Run:** `sudo docker compose -p silo-prod up -d --force-recreate silo`
|
||||
- [ ] **Wait until healthy** (same command as Task 2 Step 2.4).
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
curl -sS -o /dev/null -w "HTTP %{http_code}\n" http://localhost:8090/api/v1/health
|
||||
curl -sS -o /dev/null -w "HTTP %{http_code}\n" http://localhost:8090/
|
||||
```
|
||||
Expected: same status codes silo returned before this sub-plan
|
||||
started (typically `200` for `/`, `404` for `/api/v1/health` since
|
||||
that path was 404 in the baseline logs). No new errors in `docker
|
||||
logs silo-prod-silo-1`.
|
||||
|
||||
### Step 6.9: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add internal/audiobooks/doc.go internal/audiobooks/service.go internal/audiobooks/service_test.go cmd/silo/main.go
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): scaffold internal/audiobooks package
|
||||
|
||||
Empty-but-compiling Service that reads the audiobooks.enabled feature
|
||||
flag from server_settings. Wired into cmd/silo so the package is
|
||||
referenced from the binary; no routes mounted, no scheduled tasks
|
||||
registered, no DB writes. Subsequent sub-plans hang scanner branches,
|
||||
ABS handlers, Socket.io, podcast refresher, and SPA pages off this
|
||||
Service.
|
||||
|
||||
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
|
||||
EOF
|
||||
)"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Self-review (done by the planner before handoff)
|
||||
|
||||
**Spec coverage:**
|
||||
|
||||
| Spec section | Task(s) that cover it |
|
||||
|---|---|
|
||||
| Data model: reused tables | Task 1 (verification only) |
|
||||
| Data model: `abs_sessions` (migration 147) | Task 2 |
|
||||
| Data model: `podcast_feeds` (migration 157) | Task 3 |
|
||||
| Data model: `media_folders.type` audiobook value (migration 159 no-op) | Task 4 |
|
||||
| Data model: data migration = none | n/a — confirmed in spec |
|
||||
| Feature flag (`audiobooks.enabled`) | Task 5 |
|
||||
| Architecture: `internal/audiobooks/` package | Task 6 |
|
||||
| Risks: `media_libraries` column check | Task 1, Step 1.3 |
|
||||
| Risks: `media_files.chapters` shape | Task 1, Step 1.4 |
|
||||
| Risks: progress profile-scoping | Task 1, Step 1.5 |
|
||||
| Risks: FTS handling of `audiobook` | Task 1, Step 1.8 |
|
||||
| Risks: `people` role conventions | Task 1, Step 1.7 |
|
||||
|
||||
Out of sub-plan 1 scope (deferred to later sub-plans):
|
||||
ABS REST port, Socket.io port, scanner branches, podcast refresher,
|
||||
silo SPA pages, plugin retirement, cutover. Each is its own sub-plan
|
||||
(2–6).
|
||||
|
||||
**Placeholder scan:** None. Every step has the actual command, the
|
||||
actual SQL, or the actual Go to write. Conditional language is confined
|
||||
to Task 4 (which is explicit about its condition and what to do if it
|
||||
fires).
|
||||
|
||||
**Type consistency:** The fake `SettingsReader` interface in Task 6's
|
||||
test matches the production interface defined in `service.go`. Method
|
||||
name `GetString`, signature `(ctx context.Context, key string) (string, error)` — consistent across both files.
|
||||
|
||||
**Note on migration numbering:** Tasks 2–5 originally used numbers `139..142`
|
||||
based on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,35 @@
|
||||
# Audiobooks Absorption — Sub-plan 5: Podcasts (RSS feed refresher)
|
||||
|
||||
**Goal:** Port the plugin's `podcastfeed` package as a first-party scheduled task in silo, so RSS-subscribed podcasts get their episode lists kept up to date.
|
||||
|
||||
**Source plugin:** `../silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Port the refresher package
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `internal/audiobooks/podcastfeed/`
|
||||
- Update imports (drop plugin SDK; use silo's `pgxpool` + `log/slog`)
|
||||
- Rewrite SQL queries against silo's `podcast_feeds` table (created in sub-plan 1 migration 140), `media_items` (type='podcast'), and `episodes`
|
||||
- Drop the plugin's `presentation_libraries` reference; subscriptions are tracked by `podcast_feeds.media_item_id`
|
||||
|
||||
### Task 2: Wire as scheduled task
|
||||
- Discovery D8 documented silo's task registration pattern: `taskMgr.Register(tasks.NewXxxTask(...))` in `cmd/silo/main.go`
|
||||
- Add `tasks.NewSyncPodcastFeedsTask(refresher)` and register
|
||||
- Default interval: 10 minutes (matches plugin)
|
||||
|
||||
### Task 3: ABS podcast endpoints (optional)
|
||||
The plugin's `abs/podcast.go` + `abs/rss_feed_handler.go` + `abs/podcast_handler.go` expose:
|
||||
- `GET /abs/api/libraries/{id}/podcasts/{podcastId}/episodes/{episodeId}`
|
||||
- `POST /abs/api/libraries/{id}/podcasts` (add subscription)
|
||||
- `GET /abs/api/podcast/feeds` (admin RSS feed browse)
|
||||
|
||||
If reach is required for ABS clients, port these. Otherwise stub 501 and ship. For initial release, stub.
|
||||
|
||||
### Task 4: Build + smoke
|
||||
- Verify scheduled task registers on startup (`docker logs` shows the task)
|
||||
- Verify nothing else breaks
|
||||
|
||||
## Risks
|
||||
|
||||
- Plugin's RSS parser may depend on third-party packages not in silo's go.mod. Check go.mod after porting and add if needed.
|
||||
- `episodes` table in silo has a different shape than the plugin assumed. Re-read the plugin's INSERT statements and adapt.
|
||||
@@ -979,7 +979,7 @@ Expected: `healthy`.
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT version, filename FROM schema_versions WHERE version=174;"
|
||||
"SELECT version_id FROM public.goose_db_version WHERE version_id=174 AND is_applied;"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='autoscan_sources' AND column_name='label';"
|
||||
```
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
# TheIntroDB read-path correctness (Phase 1 / Layer A) — implementation plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Fix the three read-path correctness gaps in the TheIntroDB provider so it matches the
|
||||
real v3 API contract: (1) honor **TVDB** ids in lookups (today they are silently dropped),
|
||||
(2) capture and use the **real per-segment `confidence`** instead of a hardcoded `0.9`, and
|
||||
(3) when the API returns multiple candidate segments, **pick the most-submitted / highest-confidence**
|
||||
one instead of the first usable. This is Phase 1 of
|
||||
[the marker sources & contribution design](../specs/2026-06-06-marker-sources-and-contribution-design.md);
|
||||
it stands alone, ships value immediately (anime / TheTVDB-first libraries start getting markers),
|
||||
and unblocks the "query-all / best-wins" merge in later phases.
|
||||
|
||||
**Architecture:** All changes are contained within `internal/markers/introdb` plus one added
|
||||
field on `markers.Marker`. The resolver already populates the TVDB id into
|
||||
`Request.ExternalIDs` — only the provider and HTTP client drop it. The HTTP `Client` gains a
|
||||
`tvdbID` argument on `FetchEpisode`/`FetchMovie` (query preference `tmdb → tvdb → imdb`, cache
|
||||
keys extended); the provider reads `ExternalIDKeyTVDB`; `segmentTimestamps` gains `confidence`
|
||||
and `submission_count`; `pickMarker` uses real confidence (with a named default) and ranks
|
||||
candidates. The write path (`BuildUpdatePayload` / `UpsertMarkers`) is **not** touched here.
|
||||
|
||||
**Out of scope (deferred to Phase 2):** threading *per-segment* confidence through
|
||||
`MarkerUpdatePayload` and `UpsertMarkers`. That refactor only becomes observable once
|
||||
`Registry.FetchMerged` can combine segments from different providers; with a single provider
|
||||
today the existing "max confidence across segments" collapse is equivalent, and changing it now
|
||||
would break `TestBuildUpdatePayloadAggregatesConfidence` for no functional gain. It lands with
|
||||
the merge dispatch in Phase 2.
|
||||
|
||||
**Tech Stack:** Go (the `internal/markers/introdb` package is pure Go — no libvips/CGO),
|
||||
PostgreSQL is not involved (no migration), no frontend changes. Tests run in a throwaway Go
|
||||
container (the host has no Go toolchain); a named volume caches module downloads.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
`internal/markers` and `internal/markers/introdb` are pure Go, so no libvips is needed:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
To run a single test, append `-run TestName` and narrow the package path, e.g.
|
||||
`go test ./internal/markers/introdb/ -run TestFetchEpisodeSendsTVDBWhenNoTMDB -v`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
- `internal/markers/introdb/types.go` — **modify**. Add `Confidence` + `SubmissionCount` to `segmentTimestamps`; add the `defaultConfidence` const.
|
||||
- `internal/markers/types.go` — **modify**. Add `SubmissionCount` to `markers.Marker`.
|
||||
- `internal/markers/introdb/client.go` — **modify**. `tvdbID` arg on `FetchEpisode`/`FetchMovie`; query preference `tmdb → tvdb → imdb`; cache keys include tvdb.
|
||||
- `internal/markers/introdb/provider.go` — **modify**. Read `ExternalIDKeyTVDB` and pass it through; rewrite `pickMarker` for real confidence + best-candidate.
|
||||
- `internal/markers/introdb/client_test.go` — **create**. httptest-based client tests (TVDB query, TMDB preference, caching).
|
||||
- `internal/markers/introdb/provider_test.go` — **create**. End-to-end provider tests (TVDB-only resolves, real/default confidence, best-candidate).
|
||||
|
||||
No migration, no wiring change (`cmd/silo/main.go` constructs the provider but never calls the
|
||||
client directly), no frontend change.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Type plumbing — capture confidence + submission_count
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/types.go`
|
||||
- Modify: `internal/markers/types.go`
|
||||
|
||||
This task is pure struct plumbing (no behavior yet); later tasks' tests exercise it. Verify by
|
||||
compilation.
|
||||
|
||||
- [ ] **Step 1: Add the response fields + default constant in `introdb/types.go`**
|
||||
|
||||
Replace the `Algorithm` const block to add `defaultConfidence` after it:
|
||||
|
||||
```go
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// defaultConfidence is applied when TheIntroDB omits a per-segment confidence
|
||||
// in the /media response. Real per-segment confidence is preferred when present.
|
||||
const defaultConfidence = 0.9
|
||||
```
|
||||
|
||||
Replace the `segmentTimestamps` struct with the version that decodes the two extra fields the
|
||||
v3 `/media` response carries:
|
||||
|
||||
```go
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end). Confidence and SubmissionCount are optional per-segment
|
||||
// quality signals used to pick among multiple candidates.
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
Confidence *float64 `json:"confidence,omitempty"`
|
||||
SubmissionCount *int `json:"submission_count,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add `SubmissionCount` to `markers.Marker`**
|
||||
|
||||
In `internal/markers/types.go`, replace the `Marker` struct:
|
||||
|
||||
```go
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the packages still compile**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./internal/markers/...
|
||||
```
|
||||
|
||||
Expected: no output, exit 0. (Existing `write_test.go` / `types_test.go` use field-keyed
|
||||
literals, so the added field does not break them.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/types.go internal/markers/types.go
|
||||
git commit -m "feat(markers): decode introdb per-segment confidence + submission_count"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: TVDB lookups (gap #1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/client.go`
|
||||
- Modify: `internal/markers/introdb/provider.go`
|
||||
- Create: `internal/markers/introdb/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing client tests**
|
||||
|
||||
Create `internal/markers/introdb/client_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchEpisodeSendsTVDBWhenNoTMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "55555", "tt1234567", 2, 3, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "55555" {
|
||||
t.Errorf("tvdb_id = %q, want 55555", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "" {
|
||||
t.Errorf("tmdb_id = %q, want empty", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("imdb_id should be omitted when tvdb present, got %q", gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodePrefersTMDBOverTVDBAndIMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "111", "222", "tt333", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "111" {
|
||||
t.Errorf("tmdb_id = %q, want 111", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "" || gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("only tmdb_id expected, got tvdb=%q imdb=%q", gotQuery.Get("tvdb_id"), gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMovieSendsTVDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"movie"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchMovie(context.Background(), "", "888", "", 0); err != nil {
|
||||
t.Fatalf("FetchMovie: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "888" {
|
||||
t.Errorf("tvdb_id = %q, want 888", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodeCachesByID(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "999", "", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("server hits = %d, want 1 (cached after first)", hits)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: FAIL — compile error (`FetchEpisode`/`FetchMovie` still take the old 6/4-arg
|
||||
signatures without `tvdbID`).
|
||||
|
||||
- [ ] **Step 3: Add `tvdbID` to `FetchEpisode` in `client.go`**
|
||||
|
||||
Replace the whole `FetchEpisode` method:
|
||||
|
||||
```go
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, tvdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `tvdbID` to `FetchMovie` in `client.go`**
|
||||
|
||||
Replace the whole `FetchMovie` method:
|
||||
|
||||
```go
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, tvdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, tvdbID, imdbID, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Extend the cache-key helpers in `client.go`**
|
||||
|
||||
Replace both `cacheKeyEpisode` and `cacheKeyMovie`:
|
||||
|
||||
```go
|
||||
func cacheKeyEpisode(tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:%s:s%de%d:d%d", tvdbID, season, episode, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, tvdbID, imdbID string, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:movie:%s:d%d", tvdbID, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Read TVDB in the provider and pass it through (`provider.go`)**
|
||||
|
||||
Replace the id-extraction block in `FetchMarkers` (the `tmdbID`/`imdbID` lines and the empty
|
||||
guard):
|
||||
|
||||
```go
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
tvdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTVDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Then update the two client calls in the `switch req.Kind` block to pass `tvdbID`:
|
||||
|
||||
```go
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, tvdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
```
|
||||
```go
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, tvdbID, imdbID, durationMS)
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the client tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestFetch*` tests).
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/client.go internal/markers/introdb/provider.go internal/markers/introdb/client_test.go
|
||||
git commit -m "fix(markers): honor TVDB ids in TheIntroDB lookups"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Real confidence + best-candidate selection (gaps #2, #3)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/provider.go` (`pickMarker`)
|
||||
- Create: `internal/markers/introdb/provider_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing provider tests**
|
||||
|
||||
Create `internal/markers/introdb/provider_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
func newProvider(t *testing.T, body string) *Provider {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
return NewProvider(c)
|
||||
}
|
||||
|
||||
func episodeReq(ids map[string]string) markers.Request {
|
||||
return markers.Request{
|
||||
Kind: markers.ItemKindEpisode,
|
||||
ExternalIDs: ids,
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Duration: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderResolvesTVDBOnly(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, err := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTVDB: "777"}))
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if len(res.Markers) != 1 || res.Markers[0].Kind != markers.MarkerKindIntro {
|
||||
t.Fatalf("expected one intro marker, got %+v", res.Markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderUsesRealConfidence(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000,"confidence":0.42}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if res.Markers[0].Confidence != 0.42 {
|
||||
t.Errorf("confidence = %v, want 0.42 (real value, not hardcoded)", res.Markers[0].Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderDefaultsConfidenceWhenAbsent(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if res.Markers[0].Confidence != defaultConfidence {
|
||||
t.Errorf("confidence = %v, want default %v", res.Markers[0].Confidence, defaultConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPicksMostSubmittedCandidate(t *testing.T) {
|
||||
body := `{"type":"episode","intro":[
|
||||
{"end_ms":50000,"confidence":0.6,"submission_count":2},
|
||||
{"end_ms":61000,"confidence":0.5,"submission_count":9}
|
||||
]}`
|
||||
p := newProvider(t, body)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if got := res.Markers[0].End; got != 61*time.Second {
|
||||
t.Errorf("picked end = %v, want 61s (the submission_count=9 candidate)", got)
|
||||
}
|
||||
if res.Markers[0].SubmissionCount != 9 {
|
||||
t.Errorf("submission_count = %d, want 9", res.Markers[0].SubmissionCount)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: FAIL — `TestProviderUsesRealConfidence` sees `0.9` (hardcoded), `TestProviderPicksMostSubmittedCandidate`
|
||||
gets the first usable (`end=50s`, `SubmissionCount=0`). (`TestProviderResolvesTVDBOnly` already
|
||||
passes from Task 2 — it guards against regression.)
|
||||
|
||||
- [ ] **Step 3: Rewrite `pickMarker` in `provider.go`**
|
||||
|
||||
Replace the entire `pickMarker` function:
|
||||
|
||||
```go
|
||||
// pickMarker selects the best usable segment from a TheIntroDB response array.
|
||||
// `requireEnd` is true for segments where the end timestamp is the load-bearing
|
||||
// field (intro, recap) — they're allowed to start at 0 if `start_ms` is omitted.
|
||||
// For trailing segments (credits, preview) the start is required but the end
|
||||
// defaults to the file duration. When several candidates are usable (e.g. no
|
||||
// duration match narrowed the set), the most-submitted one wins, with higher
|
||||
// confidence breaking ties; with a single candidate this is the previous
|
||||
// first-usable behavior. Real per-segment confidence is used when present,
|
||||
// falling back to defaultConfidence only when the API omits it.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
best := markers.Marker{}
|
||||
bestSubs := -1
|
||||
found := false
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
confidence := defaultConfidence
|
||||
if s.Confidence != nil {
|
||||
confidence = *s.Confidence
|
||||
}
|
||||
subs := 0
|
||||
if s.SubmissionCount != nil {
|
||||
subs = *s.SubmissionCount
|
||||
}
|
||||
if !found || subs > bestSubs || (subs == bestSubs && confidence > best.Confidence) {
|
||||
best = markers.Marker{Kind: kind, Start: start, End: end, Confidence: confidence, SubmissionCount: subs}
|
||||
bestSubs = subs
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestProvider*` tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/provider.go internal/markers/introdb/provider_test.go
|
||||
git commit -m "feat(markers): use real introdb confidence and pick best candidate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Full-package regression + integration build
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Run the entire markers test suite**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
Expected: PASS — the new `introdb` tests plus the existing `write_test.go` /
|
||||
`types_test.go` (unchanged behavior: `BuildUpdatePayload` still aggregates max confidence,
|
||||
`FetchFirstHit` still uses registration order).
|
||||
|
||||
- [ ] **Step 2: Build the binary to confirm nothing downstream broke**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Expected: exit 0. (The only callers of the changed client methods are in `provider.go`, updated
|
||||
in Task 2; `cmd/silo/main.go` constructs the provider but never calls the client directly.)
|
||||
|
||||
- [ ] **Step 3: Confirm the working tree is clean**
|
||||
|
||||
```bash
|
||||
git status
|
||||
```
|
||||
|
||||
Expected: clean (all changes committed in Tasks 1–3; this task adds no files).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Gap coverage:** TVDB lookups (Task 2, `TestFetch*` + `TestProviderResolvesTVDBOnly`), real
|
||||
confidence (Task 3, `TestProviderUsesRealConfidence` + `…DefaultsConfidenceWhenAbsent`),
|
||||
best-candidate (Task 3, `TestProviderPicksMostSubmittedCandidate`). Maps to design gaps #1–#3.
|
||||
- **Containment:** the client signature change touches exactly one production caller
|
||||
(`provider.go`); `markers.Marker` is constructed only in `provider.go`. Verified by grep before
|
||||
planning, re-verified by the Task 4 `go build ./...`.
|
||||
- **No behavior regressions:** the write path is untouched, so `BuildUpdatePayload`'s
|
||||
max-confidence aggregation and its test remain valid; per-segment confidence is deferred to
|
||||
Phase 2 where the merge actually consumes it.
|
||||
- **No placeholders:** every code step shows the complete replacement; every run step shows the
|
||||
command and the expected result. No migration, no frontend, no wiring changes.
|
||||
- **Default confidence:** `defaultConfidence = 0.9` preserves today's effective value for
|
||||
responses that omit confidence, so existing stored markers keep equivalent provenance.
|
||||
@@ -0,0 +1,482 @@
|
||||
# Marker sources & TheIntroDB contribution — implementation plan (all phases)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Implement phases in order; each phase is independently shippable.
|
||||
|
||||
**Goal:** Implement the design in
|
||||
[2026-06-06-marker-sources-and-contribution-design.md](../specs/2026-06-06-marker-sources-and-contribution-design.md):
|
||||
fix the TheIntroDB read-path correctness gaps, formalize a multi-source provider model
|
||||
(query-all / best-wins), and add the ability to contribute Silo's own markers back to
|
||||
TheIntroDB — configured **per provider, off by default**. Server-side API + data model only;
|
||||
web UI is a separate follow-up.
|
||||
|
||||
**Architecture:** Five sequential phases. Phase 1 fixes the existing provider (TVDB, real
|
||||
confidence, best-candidate). Phase 2 adds a per-provider config table, per-segment marker
|
||||
provenance, and `Registry.FetchMerged`. Phase 3 adds the submission client, the contribution
|
||||
audit table, and the `ContributionService` engine. Phase 4 exposes the full admin API (manual
|
||||
markers, provider config, contribute, history). Phase 5 adds the daily auto-contribution task.
|
||||
|
||||
**Tech Stack:** Go (chi handlers, pgx repositories), PostgreSQL (paired numbered migrations),
|
||||
no frontend in this plan. Most touched packages (`internal/markers`, `internal/markers/introdb`,
|
||||
`internal/taskmanager/tasks`) are pure Go; `internal/api/handlers` needs libvips (CGO image
|
||||
deps). Tests run in a throwaway Go container (host has no Go toolchain).
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Phase overview & dependencies
|
||||
|
||||
| Phase | Delivers | Depends on | New migrations |
|
||||
|------|----------|-----------|----------------|
|
||||
| 1 | TheIntroDB read-path correctness (TVDB, real confidence, best-candidate) | — | none |
|
||||
| 2 | `marker_provider_config` + per-segment provenance + `FetchMerged` | 1 (real confidence) | `marker_provider_config` |
|
||||
| 3 | Submission client + `marker_contributions` + `ContributionService` | 2 (`Submitter`, provider config) | `marker_contributions` |
|
||||
| 4 | Admin API: manual markers, provider config, contribute, history | 3 (service) | none |
|
||||
| 5 | Daily auto-contribution task | 3 (service), 4 (optional) | none |
|
||||
|
||||
Phase 1's line-level steps are also captured standalone in
|
||||
[2026-06-06-marker-introdb-readpath-correctness.md](2026-06-06-marker-introdb-readpath-correctness.md);
|
||||
they are summarized here so this document is self-contained.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
Pure-Go packages (`internal/markers`, `internal/markers/introdb`, `internal/taskmanager/...`):
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
`internal/api/handlers` (Phase 4) needs libvips:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
sh -c 'apt-get update >/dev/null && apt-get install -y --no-install-recommends libvips-dev >/dev/null && go test ./internal/api/handlers/ -run TestAdminMarkers -v'
|
||||
```
|
||||
|
||||
Whole-project compile: `go build ./...` (same container).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — TheIntroDB read-path correctness
|
||||
|
||||
**Goal:** Honor TVDB ids, capture/use real per-segment confidence + submission_count, and pick
|
||||
the best candidate among multiple. Contained to `internal/markers/introdb` plus one field on
|
||||
`markers.Marker`. No migration, no write-path change.
|
||||
|
||||
**Files:** `internal/markers/introdb/{types.go,client.go,provider.go}` (modify),
|
||||
`internal/markers/types.go` (modify, add `Marker.SubmissionCount`),
|
||||
`internal/markers/introdb/{client_test.go,provider_test.go}` (create).
|
||||
|
||||
### Task 1.1 — Type plumbing
|
||||
- [ ] Add `Confidence *float64` and `SubmissionCount *int` to `segmentTimestamps`
|
||||
(`introdb/types.go`); add `const defaultConfidence = 0.9`.
|
||||
- [ ] Add `SubmissionCount int` to `markers.Marker` (`markers/types.go`).
|
||||
- [ ] `go build ./internal/markers/...` → exit 0. Commit.
|
||||
|
||||
### Task 1.2 — TVDB lookups (gap #1)
|
||||
- [ ] Create `introdb/client_test.go` (httptest): asserts `tvdb_id` sent when only tvdb given,
|
||||
tmdb preferred over tvdb/imdb, movie tvdb, and cache collapses repeat calls. Run → FAIL (compile).
|
||||
- [ ] `client.go`: `FetchEpisode`/`FetchMovie` gain a `tvdbID` arg; query preference
|
||||
`tmdb → tvdb → imdb` via a `switch`; the empty-id guard includes tvdb; `cacheKeyEpisode`/
|
||||
`cacheKeyMovie` include a `tvdb:` branch.
|
||||
- [ ] `provider.go` `FetchMarkers`: read `req.ExternalIDs[markers.ExternalIDKeyTVDB]`, include it
|
||||
in the empty guard, pass it to both client calls.
|
||||
- [ ] Run client tests → PASS. Commit.
|
||||
|
||||
### Task 1.3 — Real confidence + best-candidate (gaps #2, #3)
|
||||
- [ ] Create `introdb/provider_test.go`: TVDB-only resolves; real confidence used; default when
|
||||
absent; most-submitted candidate wins (ties broken by confidence). Run → FAIL.
|
||||
- [ ] Rewrite `pickMarker` to use real confidence (fallback `defaultConfidence`), set
|
||||
`SubmissionCount`, and select the candidate with the highest `(submission_count, confidence)`.
|
||||
- [ ] Run → PASS. Commit.
|
||||
|
||||
### Task 1.4 — Regression
|
||||
- [ ] `go test ./internal/markers/... && go build ./...` → all green (existing `write_test.go`/
|
||||
`types_test.go` unaffected; write path untouched). Commit if needed.
|
||||
|
||||
(Full code for every step is in the standalone Phase 1 plan linked above.)
|
||||
|
||||
---
|
||||
|
||||
# Phase 2 — Per-provider config, per-segment provenance, FetchMerged
|
||||
|
||||
**Goal:** Add the `marker_provider_config` table that drives which providers are queried (and,
|
||||
later, submitted to); make the marker write path carry **per-segment** provider/confidence/
|
||||
algorithm so a merged multi-provider result writes correct provenance; add
|
||||
`Registry.FetchMerged` (query all enabled providers, keep the best candidate per segment) and
|
||||
switch the lazy-playback path to it. With only TheIntroDB enabled, behavior is unchanged.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/<next>_marker_provider_config.{up,down}.sql`
|
||||
- Create: `internal/markers/provider_config.go`, `internal/markers/provider_config_test.go`
|
||||
- Modify: `internal/markers/types.go` (`Marker.ProviderID`/`Algorithm`; `Registry.FetchMerged`)
|
||||
- Modify: `internal/markers/write.go` (per-segment provenance in `MarkerUpdatePayload`/`BuildUpdatePayload`)
|
||||
- Modify: `internal/scanner/file_repo.go` (`MarkerUpdate` + `applySegmentPatch` per-segment provenance)
|
||||
- Modify: the `markers.MarkerUpdatePayload` → `scanner.MarkerUpdate` conversion (in the lazy-markers handler/adapter)
|
||||
- Modify: `internal/api/handlers/playback_lazy_markers.go` (call `FetchMerged`)
|
||||
- Modify: `cmd/silo/main.go` (construct the config store; pass to the registry)
|
||||
|
||||
### Task 2.1 — Migration: `marker_provider_config`
|
||||
- [ ] Write `migrations/<next>_marker_provider_config.up.sql` (use the next free number; latest is 180):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY,
|
||||
fetch_enabled boolean NOT NULL DEFAULT true,
|
||||
fetch_priority integer NOT NULL DEFAULT 100,
|
||||
contribute_enabled boolean NOT NULL DEFAULT false,
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false,
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: `DROP TABLE public.marker_provider_config;`
|
||||
- [ ] Commit.
|
||||
|
||||
### Task 2.2 — Provider config store
|
||||
- [ ] Create `internal/markers/provider_config.go`:
|
||||
|
||||
```go
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
// ProviderConfigStore is a cached read/write façade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; Update writes through and refreshes.
|
||||
type ProviderConfigStore struct { /* pool, mu, cache map[string]ProviderConfig */ }
|
||||
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error // load all rows into cache
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig // snapshot copy
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool)
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, p ProviderConfig) error // upsert + refresh
|
||||
// EnabledForFetch returns fetch_enabled providers sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig
|
||||
```
|
||||
|
||||
- [ ] Hot-reload: subscribe the store's `Reload` to the existing `cache.EventSettingsChanged`
|
||||
Redis channel (mirror the `introdb.api_key` reload wiring in `cmd/silo/main.go`), or add a
|
||||
dedicated `marker_provider_config_changed` event published by `Update`. Either is acceptable;
|
||||
document which.
|
||||
- [ ] `provider_config_test.go`: with a real/pgxmock pool or an injected fake, assert
|
||||
`EnabledForFetch` filters disabled rows and sorts by priority. Commit.
|
||||
|
||||
### Task 2.3 — Per-segment marker provenance (write path)
|
||||
|
||||
This makes a merged result (intro from provider A, credits from provider B) persist the correct
|
||||
per-segment `*_markers_provider` / `*_markers_confidence` / `*_markers_algorithm`. The DB columns
|
||||
already exist; today `applySegmentPatch` applies one shared triple.
|
||||
|
||||
- [ ] Add to `markers.Marker`: `ProviderID string`, `Algorithm string` (so each segment carries
|
||||
its origin). `provider.go` (Phase 1) sets these from `ProviderID`/`Algorithm` per marker.
|
||||
- [ ] Rework `markers.MarkerUpdatePayload` (`write.go`) from one shared `Confidence`/`Provider`/
|
||||
`Algorithm` to a per-segment struct, e.g.:
|
||||
|
||||
```go
|
||||
type SegmentPayload struct { Start, End *float64; Provider *string; Confidence *float64; Algorithm string }
|
||||
type MarkerUpdatePayload struct {
|
||||
Intro, Credits, Recap, Preview SegmentPayload
|
||||
Source string // shared source class (e.g. "online"); per-kind provider differs
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Update `BuildUpdatePayload` to fill each `SegmentPayload` from the matching `Marker`
|
||||
(provider/confidence/algorithm per segment). Update `write_test.go`
|
||||
(`TestBuildUpdatePayloadAggregatesConfidence` → assert per-segment confidence: intro 0.7,
|
||||
credits 0.9, recap 0.5).
|
||||
- [ ] Read `applySegmentPatch` and the `MarkerUpdatePayload → scanner.MarkerUpdate` conversion
|
||||
first, then extend `scanner.MarkerUpdate` + `applySegmentPatch` so each segment's
|
||||
provider/confidence/algorithm come from that segment (not a shared field). `nextSharedMarkerAttribution`
|
||||
keeps populating the legacy shared `markers_source`/`markers_confidence` columns from the
|
||||
highest-confidence applied segment.
|
||||
- [ ] `go test ./internal/markers/... ./internal/scanner/...` → green. Commit.
|
||||
|
||||
### Task 2.4 — `Registry.FetchMerged`
|
||||
- [ ] Give `Registry` access to the enabled set (constructor takes a `ProviderConfigStore`, or
|
||||
`FetchMerged` takes an `[]ProviderConfig` snapshot). Add:
|
||||
|
||||
```go
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the candidate with the highest (SubmissionCount, Confidence),
|
||||
// breaking ties by the provider's fetch_priority. Per-segment ProviderID/
|
||||
// Algorithm are preserved so the write path records correct provenance.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error)
|
||||
```
|
||||
|
||||
Algorithm: fan out via goroutines (bounded) to enabled providers; collect `(ProviderConfig,
|
||||
Result, error)`; for each `MarkerKind`, choose the best `Marker` across results; assemble a
|
||||
`Result` whose `Markers` carry their winning provider's `ProviderID`/`Algorithm`. Log and skip
|
||||
provider errors (like `FetchFirstHit`). Return `ok=false` if no segment found.
|
||||
|
||||
- [ ] `types_test.go`: add a merge test — two fake providers, one wins intro, the other wins
|
||||
credits by submission_count; assert the merged result has both with correct per-segment
|
||||
ProviderID. Keep the existing `FetchFirstHit` tests. Commit.
|
||||
|
||||
### Task 2.5 — Switch lazy path + wire store
|
||||
- [ ] In `internal/api/handlers/playback_lazy_markers.go`, replace the `FetchFirstHit` call in
|
||||
`fetchOnlineMarkersForPlayback` with `FetchMerged`.
|
||||
- [ ] In `cmd/silo/main.go`, construct `NewProviderConfigStore`, `Reload` it at startup, wire its
|
||||
hot-reload, and pass it to the registry. (TheIntroDB stays registered as today.)
|
||||
- [ ] `go build ./... && go test ./internal/markers/...` → green. With only `introdb` enabled,
|
||||
`FetchMerged` returns the same result `FetchFirstHit` did — verify a provider test asserting
|
||||
single-provider parity. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 3 — Submission client, contribution tracking, service engine
|
||||
|
||||
**Goal:** Build the machinery to submit a marker to a provider, idempotently and audited —
|
||||
without yet wiring a trigger. Adds the `Submitter` capability, the introdb submission/stats
|
||||
client methods, the `marker_contributions` table, and the `ContributionService`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/types.go` (`Submitter`, `SubmissionRequest`/`SubmissionResult`/`UserStats`)
|
||||
- Modify: `internal/markers/introdb/{types.go,client.go,provider.go}` (submit + stats; implement `Submitter`)
|
||||
- Create: `migrations/<next>_marker_contributions.{up,down}.sql`
|
||||
- Create: `internal/markers/contribution_repo.go`, `internal/markers/contribute.go` (+ tests)
|
||||
- Modify: `cmd/silo/main.go` (construct the service)
|
||||
|
||||
### Task 3.1 — `Submitter` capability + DTOs
|
||||
- [ ] Add to `internal/markers/types.go`:
|
||||
|
||||
```go
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string // tmdb/imdb (tvdb not accepted by /submit; tmdb required)
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start, End *time.Duration // nil start ok for intro/recap; nil end ok for credits/preview
|
||||
Duration time.Duration
|
||||
}
|
||||
type SubmissionResult struct { ID string; Status string; Weight float64 } // status: pending|accepted|rejected
|
||||
type UserStats struct { Total, Accepted, Pending, Rejected int; AcceptanceRate float64; CurrentStreak, BestStreak int }
|
||||
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Commit (compiles; no implementer yet).
|
||||
|
||||
### Task 3.2 — introdb submission client
|
||||
- [ ] `introdb/types.go`: add the request/response shapes for `POST /v3/submit`
|
||||
(`tmdb_id`,`imdb_id`,`type`,`segment`,`season`,`episode`,`video_duration_ms`,`start_ms`,`end_ms`;
|
||||
response `{submissions:[{id,status,weight,...}]}`) and `GET /v3/user/stats`. Remove the
|
||||
"Submissions are intentionally not supported" package note.
|
||||
- [ ] `introdb/client.go`: add `SubmitSegment(ctx, ...) (*submitResponse, error)` (POST,
|
||||
`Authorization: Bearer` **required** — return an error if `apiKey == ""`; honor
|
||||
`X-UsageLimit-Reset` on 429) and `FetchUserStats(ctx) (*userStats, error)` (GET).
|
||||
- [ ] `introdb/provider.go`: implement `markers.Submitter` — map `SubmissionRequest` to the
|
||||
client call (apply the v3 null rules: intro/recap drop a zero start to `null`; credits/preview
|
||||
send `null` end when absent), translate `SubmissionResult`/`UserStats`.
|
||||
- [ ] `client_test.go`/`provider_test.go`: httptest asserts the submit body shape, bearer header,
|
||||
and that an empty key errors before any HTTP call; stats parse. Commit.
|
||||
|
||||
### Task 3.3 — `marker_contributions` table + repo
|
||||
- [ ] Migration `<next>_marker_contributions.up.sql` (per design §C4):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL,
|
||||
segment_kind text NOT NULL,
|
||||
source text NOT NULL,
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL,
|
||||
submission_id uuid,
|
||||
status text NOT NULL,
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: drop the table.
|
||||
- [ ] `internal/markers/contribution_repo.go`: `ContributionStore` with `AlreadySubmitted(ctx,
|
||||
fileID, provider, kind, contentHash) (bool, error)`, `Record(ctx, ContributionRow) error`
|
||||
(upsert on the unique key), `ListByFile(ctx, fileID) ([]ContributionRow, error)`. Add a
|
||||
`ContentHash(kind string, startMs, endMs, durationMs *int64) string` helper (stable SHA-256).
|
||||
- [ ] Repo test (value-hash idempotency, list). Commit.
|
||||
|
||||
### Task 3.4 — `ContributionService`
|
||||
- [ ] `internal/markers/contribute.go`:
|
||||
|
||||
```go
|
||||
type ContributionService struct { /* registry, resolver, cfg *ProviderConfigStore, store *ContributionStore, log */ }
|
||||
|
||||
// ContributeFile submits the file's eligible markers to enabled submitter providers.
|
||||
// opts may scope to one provider and/or specific segment kinds. Auto callers pass
|
||||
// requireAutoLocal=true so only providers with contribute_auto_local participate.
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error)
|
||||
```
|
||||
|
||||
Per provider (from `cfg.List()` filtered to `Submitter` + `contribute_enabled`, and
|
||||
`contribute_auto_local` when `opts.Auto`):
|
||||
1. **Eligibility (C2):** for each requested segment present on the file, require
|
||||
`source ∈ {scanner, manual}` (skip `online` — circular); resolve external IDs via the existing
|
||||
resolver (skip if none / tmdb required by `/submit`); for auto require `source=scanner`,
|
||||
`kind=intro`, `confidence ≥ contribute_min_confidence`.
|
||||
2. **Idempotency:** compute `ContentHash`; skip if `AlreadySubmitted`.
|
||||
3. **Submit:** `provider.(Submitter).SubmitMarker(...)`.
|
||||
4. **Persist:** `store.Record(...)` with returned status/submission_id (or `status="error"`).
|
||||
|
||||
- [ ] `contribute_test.go` with a fake `Submitter` + in-memory store: asserts online-sourced
|
||||
markers are skipped, sub-threshold auto markers skipped, duplicates skipped, a fresh manual
|
||||
marker submitted and recorded. Commit.
|
||||
|
||||
### Task 3.5 — Wire
|
||||
- [ ] `cmd/silo/main.go`: construct `ContributionService` (registry, resolver, config store,
|
||||
contribution store). Not yet called by any handler/task. `go build ./...` + tests green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 4 — Admin API (manual markers, provider config, contribute, history)
|
||||
|
||||
**Goal:** Expose the full backend contract under `RequireAdmin`. Mirror the structure of
|
||||
`internal/api/handlers/admin_intro.go` (interface-typed deps, `chi.URLParam`, `writeError`/
|
||||
`writeJSON`, `MarkerUpdateNotifier` after writes).
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/api/handlers/admin_markers.go` (+ test), `internal/api/handlers/admin_marker_providers.go` (+ test)
|
||||
- Modify: `internal/api/router.go` (register routes in the `RequireAdmin` group)
|
||||
- Modify: `cmd/silo/main.go` (construct + inject the handlers)
|
||||
|
||||
### Task 4.1 — Manual marker endpoints (design §C3)
|
||||
- [ ] `admin_markers.go`:
|
||||
- `GET /admin/files/{fileId}/markers` → current per-segment values + provenance
|
||||
(`source`,`provider`,`confidence`,`algorithm`,`detected_at`) for intro/recap/credits/preview.
|
||||
- `PUT /admin/files/{fileId}/markers` → body per segment `{start,end}` (seconds) or `null` to
|
||||
clear; write `source = "manual"` via `scanner.FileRepository.UpsertMarkers` (priority 4);
|
||||
then `MarkerUpdateNotifier.MarkersUpdated(file)`. Validation: `end>start`; intro/recap may
|
||||
omit start; credits/preview may omit end; within `[0, duration]`.
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` → clear one segment.
|
||||
- `GET|PUT /admin/items/{id}/markers` → resolve to the item's primary file (convenience).
|
||||
- [ ] Handler test (libvips container): PUT writes manual markers and returns provenance;
|
||||
invalid range → 400; clear nulls the segment. Commit.
|
||||
|
||||
### Task 4.2 — Provider config + validate endpoints (design §C7)
|
||||
- [ ] `admin_marker_providers.go`:
|
||||
- `GET /admin/markers/providers` → for each registered provider: id, `isSubmitter`, its
|
||||
`marker_provider_config` row, and (best-effort cached) `UserStats` for submitters with a key.
|
||||
- `PUT /admin/markers/providers/{provider}` → update the config row (validated; unknown → 404).
|
||||
- `POST /admin/markers/providers/{provider}/validate` → `Submitter.FetchUserStats`; non-submitter → 400.
|
||||
- [ ] Test: list returns introdb with its config; update flips `contribute_enabled`; validate on
|
||||
a non-submitter → 400. Commit.
|
||||
|
||||
### Task 4.3 — Contribution endpoints (design §C7)
|
||||
- [ ] In `admin_markers.go` (or a small `admin_contributions.go`):
|
||||
- `POST /admin/files/{fileId}/contribute` (optional body `{provider?, segments?[]}`) →
|
||||
`ContributionService.ContributeFile(...)`; returns per (provider, segment) outcome.
|
||||
- `GET /admin/files/{fileId}/contributions` → `ContributionStore.ListByFile`.
|
||||
- [ ] Test with a fake service/store. Commit.
|
||||
|
||||
### Task 4.4 — Routing + wiring
|
||||
- [ ] Register all new routes inside the existing `RequireAdmin` group in `internal/api/router.go`
|
||||
(mirror the `redetect-intro` registration). Inject the handlers in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/api/handlers/ -run 'TestAdminMarkers|TestAdminMarkerProviders'`
|
||||
(libvips container) → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 5 — Daily auto-contribution task
|
||||
|
||||
**Goal:** A scheduled task that, for each provider with `contribute_enabled &&
|
||||
contribute_auto_local`, submits high-confidence local intro detections that haven't been
|
||||
contributed. Mirror `internal/taskmanager/tasks/detect_intro_markers.go`.
|
||||
|
||||
**Files:** Create `internal/taskmanager/tasks/contribute_markers.go` (+ test); modify
|
||||
`cmd/silo/main.go` (register the task).
|
||||
|
||||
### Task 5.1 — `ContributeMarkersTask`
|
||||
- [ ] Implement the `taskmanager.Task` interface (`Key="contribute_markers"`, `Name`,
|
||||
`Description`, `Category=TaskCategoryLibrary`, `IsHidden() bool`, `DefaultTriggers()` daily e.g.
|
||||
`"04:00"` — after the 03:30 detection task, `Execute`).
|
||||
- [ ] `Execute`: short-circuit if no provider has `contribute_enabled && contribute_auto_local`
|
||||
(read `ProviderConfigStore`). Otherwise page through episode files with
|
||||
`intro_markers_source='scanner'` and `intro_markers_confidence ≥` the provider's
|
||||
`contribute_min_confidence` (a new `ContributionStore`/repo query, batched), and call
|
||||
`ContributionService.ContributeFile(file, ContributeOptions{Auto:true})`. Report progress; obey
|
||||
the introdb usage limit (the client already backs off on `X-UsageLimit-Reset`); idempotency
|
||||
makes interrupted runs resumable.
|
||||
- [ ] Test with fakes: enabled+auto submits eligible files; disabled → no-op; sub-threshold
|
||||
skipped. Commit.
|
||||
|
||||
### Task 5.2 — Register
|
||||
- [ ] Register the task where `DetectIntroMarkersTask` is registered in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/taskmanager/...` → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
## Final integration verification
|
||||
|
||||
**Files:** none (build + runtime).
|
||||
|
||||
- [ ] **Build the image** (compiles backend + frontend):
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse --short HEAD) --build-arg BUILD_DIRTY=false -t silo-server:markers-test .
|
||||
```
|
||||
- [ ] **Recreate the container** (applies the new migrations on startup) and wait for health:
|
||||
```bash
|
||||
docker compose up -d silo
|
||||
for i in $(seq 1 24); do s=$(docker inspect -f '{{.State.Health.Status}}' silo-silo-1); echo "$s"; [ "$s" = healthy ] && break; sleep 5; done
|
||||
```
|
||||
- [ ] **Verify migrations applied + tables exist:**
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT to_regclass('public.marker_provider_config'), to_regclass('public.marker_contributions');"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c "SELECT * FROM marker_provider_config;"
|
||||
```
|
||||
Expected: both tables present; an `introdb` row with `fetch_enabled=t, contribute_enabled=f`.
|
||||
- [ ] **Smoke the read path (TVDB):** trigger playback (or call the marker fetch) for a
|
||||
TVDB-only-matched episode and confirm markers now populate where they previously did not.
|
||||
- [ ] **Smoke contribution (manual, dry):** enable contribution for introdb against a **test/staging**
|
||||
TheIntroDB key, `PUT` a manual marker on a file, `POST /admin/files/{id}/contribute`, then
|
||||
`GET /admin/files/{id}/contributions` shows a `pending` row; a second contribute is idempotent
|
||||
(no new row). Use a non-production key so test data isn't submitted to the real database.
|
||||
- [ ] `git status` → clean (all changes committed per phase).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Phase independence:** each phase compiles, tests, and ships alone; with only `introdb`
|
||||
enabled and contribution off (defaults), Phases 2–5 are observationally inert until an operator
|
||||
enables a provider — so they can land ahead of any behavior change.
|
||||
- **Per-provider, off by default:** `contribute_enabled`/`contribute_auto_local` default `false`
|
||||
in the migration; the service refuses to submit otherwise; auto additionally gated on
|
||||
`contribute_auto_local`. The `provider` column on `marker_contributions` and the per-provider
|
||||
loop make multi-target submission first-class.
|
||||
- **Circular-contribution guard:** `source='online'` is never contributed (C2 eligibility) — has
|
||||
a dedicated unit test in Task 3.4.
|
||||
- **Idempotency:** value `content_hash` + the unique constraint mean repeat/interrupted runs
|
||||
don't double-submit, while a corrected value (new hash) does submit — matching TheIntroDB's
|
||||
weighted-average model.
|
||||
- **Write-path accuracy:** Task 2.3 requires reading `applySegmentPatch` and the
|
||||
`MarkerUpdatePayload → scanner.MarkerUpdate` conversion before editing; the per-segment columns
|
||||
already exist in `media_files`, so this is a plumbing change, not a schema change.
|
||||
- **Migration numbers:** `marker_provider_config` and `marker_contributions` take the next two
|
||||
free numbers at implementation time (181+; latest on disk is 180). Re-check for collisions
|
||||
before writing.
|
||||
- **Out of scope (tracked in the design doc):** all web UI; per-user TheIntroDB accounts; a
|
||||
`marker_provider.v1` plugin capability. Clean seams left for each.
|
||||
@@ -0,0 +1,153 @@
|
||||
# Audiobooks Absorption — Discovery Findings
|
||||
|
||||
Produced by sub-plan 1, Task 1. Locks data-model and integration
|
||||
decisions for the audiobook foundation migrations and downstream sub-plans.
|
||||
|
||||
## D1 — Next migration number
|
||||
|
||||
Original snapshot: `138_search_number_word_normalization.up.sql` was the
|
||||
highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
|
||||
## D2 — `media_libraries` kind/type column
|
||||
|
||||
The spec refers to `media_libraries` but the actual table is `media_folders`.
|
||||
`media_folders` has an existing column `type` (text, NOT NULL) that discriminates
|
||||
library content. Current values in production: `movies`, `series`, `mixed`.
|
||||
|
||||
Existing column 'type' (text) discriminates library content on `media_folders`.
|
||||
Task 4 (migration 159) should ADD the value `audiobooks` to the type vocabulary
|
||||
rather than add a new column. The audiobook scanner branch will set
|
||||
`media_folders.type = 'audiobooks'` for audiobook libraries. No schema change
|
||||
needed for the column itself; migration 159 becomes a no-op DDL migration that
|
||||
documents the new allowed value and adds any supporting indexes if needed.
|
||||
|
||||
No CHECK constraint or enum enforces the `type` column values, so adding
|
||||
`audiobooks` as a value requires no DDL constraint change.
|
||||
|
||||
## D3 — `media_files.chapters` JSONB shape
|
||||
|
||||
Sample chapter JSON (live data):
|
||||
[{"index": 0, "title": "Intro start", "source": "embedded", "end_seconds": 27.944, "start_seconds": 0}, {"index": 1, "title": "Intro end", "source": "embedded", "end_seconds": 1343, "start_seconds": 27.944}]
|
||||
|
||||
Sub-plan 2 (scanner) MUST emit objects with the same keys when writing
|
||||
audiobook chapters so the existing player and serialization code accept
|
||||
them without changes.
|
||||
|
||||
Required keys: `index` (integer), `title` (text), `source` (text),
|
||||
`start_seconds` (float), `end_seconds` (float).
|
||||
|
||||
## D4 — `user_watch_progress` scoping (profile vs user)
|
||||
|
||||
user_watch_progress is profile-scoped: column 'profile_id' (text, NOT NULL, FK via
|
||||
composite PK on user_id + profile_id + media_item_id). Audiobook progress slots in directly.
|
||||
|
||||
Additional context: the table also stores `last_file_id`, `last_resolution`,
|
||||
`last_hdr`, `last_codec_video`, and `last_edition_key`. For audiobooks, only
|
||||
`position_seconds`, `duration_seconds`, `completed`, and `last_file_id` are
|
||||
semantically relevant; the video-specific columns (`last_resolution`,
|
||||
`last_hdr`, `last_codec_video`) will be NULL for audiobook progress rows,
|
||||
which is acceptable.
|
||||
|
||||
## D5 — `user_playback_sessions` audiobook fit
|
||||
|
||||
Column list:
|
||||
Table "public.user_playback_sessions"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
------------------+--------------------------+-----------+----------+---------
|
||||
session_id | text | | not null |
|
||||
user_id | integer | | not null |
|
||||
profile_id | text | | not null |
|
||||
media_file_id | integer | | not null |
|
||||
play_method | text | | not null |
|
||||
position_seconds | double precision | | not null | 0
|
||||
is_paused | boolean | | not null | false
|
||||
started_at | timestamp with time zone | | not null | now()
|
||||
updated_at | timestamp with time zone | | not null | now()
|
||||
|
||||
Columns required by audiobook sessions: media_item_id (or equivalent),
|
||||
profile/user FK, started_at, current_position_seconds (or equivalent),
|
||||
status. Mark any required column as MISSING and surface in sub-plan 3.
|
||||
|
||||
Assessment:
|
||||
- media_item_id: MISSING — table stores `media_file_id` (FK to media_files) rather
|
||||
than `media_item_id`. For audiobooks, a file maps to one audiobook item, so the
|
||||
item can be looked up via the file join. No schema change strictly required, but
|
||||
sub-plan 3 should note this indirect join cost.
|
||||
- profile_id: PRESENT (text, not null)
|
||||
- user_id: PRESENT (integer, not null)
|
||||
- started_at: PRESENT
|
||||
- position_seconds: PRESENT (as `position_seconds`)
|
||||
- status / is_paused: PRESENT (as `is_paused`); no explicit `status` enum, but
|
||||
paused/playing state is representable.
|
||||
- `play_method`: required for existing sessions; audiobook sessions must supply a
|
||||
value (e.g. `'direct'`).
|
||||
|
||||
No blocking gaps. Audiobook sessions can be written to `user_playback_sessions`
|
||||
without migration using `media_file_id` as the join key.
|
||||
|
||||
## D6 — `people` / `item_people` role conventions
|
||||
|
||||
item_people.role storage: `kind` smallint (NOT NULL) — NOT a text `role` column.
|
||||
The column is named `kind` with type `smallint`. No CHECK constraint or enum.
|
||||
|
||||
Existing kind values in use (mapped from models/media.go):
|
||||
1 = Actor, 2 = Director, 3 = Writer, 4 = Producer, 5 = GuestStar, 6 = Composer
|
||||
(6 = Composer defined in code but 0 rows in production data)
|
||||
|
||||
Sub-plan 2 will UPSERT `author` and `narrator` into item_people for
|
||||
audiobook items. Since the role column is an unconstrained smallint (not
|
||||
a text role column and not an enum or CHECK constraint), Sub-plan 2 must:
|
||||
1. Add new PersonKind constants to `internal/models/media.go`:
|
||||
`PersonKindAuthor PersonKind = 7` and `PersonKindNarrator PersonKind = 8`
|
||||
2. Add corresponding cases to `PersonKind.String()` returning `"Author"` and
|
||||
`"Narrator"` respectively.
|
||||
No migration is needed to extend a constraint — the smallint column accepts
|
||||
any integer value.
|
||||
|
||||
## D7 — Catalog FTS handling of `type='audiobook'`
|
||||
|
||||
Indexes / generated columns that filter by media_items.type:
|
||||
- `001_schema.up.sql`: `idx_media_items_search` — GIN on `to_tsvector('english', title || ' ' || overview)` — NO type filter, indexes ALL rows
|
||||
- `001_schema.up.sql`: `idx_media_items_search_exact_title` — btree on `lower(title)` — NO type filter
|
||||
- `001_schema.up.sql`: `idx_media_items_search_overview` — GIN on overview tsvector — NO type filter
|
||||
- `001_schema.up.sql`: `idx_media_items_search_title_fields` — GIN on weighted title/original_title/sort_title tsvector — NO type filter (rebuilt by migrations 127 and 138)
|
||||
- `001_schema.up.sql`: `idx_media_items_type_created` — btree on `(type, created_at DESC)` — indexes ALL types, used for filtering by type
|
||||
- `057_calendar_indexes.up.sql`: `idx_media_items_movie_release_date` — btree WHERE `type = 'movie'` — movie-only, not FTS
|
||||
- `103_media_items_last_air_date_denorm.up.sql`: `idx_media_items_last_air_date_at` — btree WHERE `type = 'series'` — series-only, not FTS
|
||||
- `105_media_items_title_normalized.up.sql`: `idx_media_items_title_normalized_trgm` — gin trigram on `title_normalized` — NO type filter (rebuilt by 127 and 138)
|
||||
- `138_search_number_word_normalization.up.sql`: `idx_media_items_search_title_fields` (current) — GIN on weighted tsvector — NO type filter
|
||||
- `138_search_number_word_normalization.up.sql`: `idx_media_items_title_normalized_trgm` (current) — gin trigram — NO type filter
|
||||
|
||||
Verdict: audiobooks WILL be FTS-searchable out of the box.
|
||||
|
||||
All FTS indexes on `media_items` operate on the full table with no type
|
||||
restriction. A row with `type = 'audiobook'` will be indexed automatically
|
||||
by `idx_media_items_search_title_fields` and `idx_media_items_title_normalized_trgm`
|
||||
as soon as it is inserted. No extra migration is needed for Sub-plan 3 to
|
||||
extend type filters.
|
||||
|
||||
## D8 — First-party scheduled-task registration
|
||||
|
||||
First-party scheduled tasks register at: `cmd/silo/main.go:1239–1285`
|
||||
|
||||
Registration call shape (from existing first-party tasks):
|
||||
```go
|
||||
taskMgr.Register(tasks.NewSyncCollectionsTask(collectionSyncScheduler))
|
||||
```
|
||||
|
||||
Full interface required (from `internal/taskmanager/tasks/sync_collections.go`):
|
||||
- `Key() string` — unique string key e.g. `"sync_podcast_feeds"`
|
||||
- `Name() string` — human-readable name
|
||||
- `Description() string` — human-readable description
|
||||
- `Category() taskmanager.TaskCategory` — e.g. `taskmanager.TaskCategoryLibrary`
|
||||
- `IsHidden() bool`
|
||||
- `DefaultTriggers() []taskmanager.TriggerConfig` — e.g. interval trigger
|
||||
- `Execute(ctx context.Context, progress taskmanager.ProgressReporter) error`
|
||||
|
||||
Sub-plan 5 (podcasts) will register `podcastfeed.Refresher` at `cmd/silo/main.go`
|
||||
in the task registration block (around line 1259–1265) using the same pattern:
|
||||
```go
|
||||
taskMgr.Register(tasks.NewSyncPodcastFeedsTask(podcastFeedRefresher))
|
||||
```
|
||||
@@ -0,0 +1,382 @@
|
||||
# Audiobooks Absorption — Design Spec
|
||||
|
||||
**Date:** 2026-05-24
|
||||
**Branch context:** `feat/audiobooks` in `silo-server`
|
||||
**Status:** Approved — ready for implementation plan
|
||||
|
||||
## Goal
|
||||
|
||||
Absorb the `silo-plugin-audiobooks` plugin into the primary `silo-server`
|
||||
Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
|
||||
- Silo's existing web UI gains audiobook + podcast browse/playback.
|
||||
- Audiobookshelf-compatible (ABS) mobile/desktop clients connect to silo
|
||||
directly and play audiobooks from libraries silo already scans.
|
||||
|
||||
## Hard constraints
|
||||
|
||||
- **Minimize changes.** Reuse silo-server's existing tables and infrastructure
|
||||
wherever they fit. Add new tables only where there is no existing equivalent.
|
||||
- **No separate SPA.** Audiobook UI lives inside silo's existing React app at
|
||||
`web/src/`. The plugin's standalone SPA is dropped.
|
||||
- **Audiobook files = scanned by silo.** Silo's existing scanner discovers
|
||||
audiobook files in configured library paths the same way it scans movies
|
||||
and TV. The `silo-plugin-local-audiobooks` plugin is no longer needed.
|
||||
- **No external backends.** BookWarehouse / other audiobook backend plugins
|
||||
are out of scope. Local filesystem is the only source.
|
||||
- **No standalone HTTP listener.** Everything runs on silo's main `:8080`
|
||||
listener, including ABS Socket.io. The plugin's standalone-listener
|
||||
workaround existed only because the host plugin-proxy could not bridge
|
||||
WebSocket upgrades; that problem disappears once code runs in-process.
|
||||
|
||||
## Scope
|
||||
|
||||
### In
|
||||
|
||||
- Audiobook entity, library, playback (silo SPA + ABS clients).
|
||||
- ABS-compatible REST API + Socket.io realtime channel.
|
||||
- Podcasts: subscribed RSS feeds with episode refresh, plus filesystem podcasts.
|
||||
- Per-profile listening progress, active play sessions, basic chapter navigation.
|
||||
|
||||
### Out
|
||||
|
||||
- Audiobook requests flow (`silo-plugin-audiobook-requests` untouched).
|
||||
- Smart collections, share links, content restrictions, metadata-provider integration.
|
||||
- Embedding-powered "similar books" recommender.
|
||||
- External enrichment through the audiobook metadata plugin/provider — deferred;
|
||||
scanner extracts local tags and identifiers for v1.
|
||||
- Group listening / cowatch parity for audiobooks (silo already has cowatch for
|
||||
video; revisit later if needed).
|
||||
|
||||
## Architecture
|
||||
|
||||
### Package layout
|
||||
|
||||
New top-level Go package: `internal/audiobooks/`.
|
||||
|
||||
```text
|
||||
internal/audiobooks/
|
||||
abs/ ← ported from plugin's internal/abs/ (~6.8k LOC)
|
||||
ABS-compatible REST handlers, mounted under /abs/* and the
|
||||
legacy /api/* paths ABS clients hardcode.
|
||||
abssocket/ ← ported from plugin's internal/abssocket/ (~250 LOC)
|
||||
Socket.io protocol on top of gorilla/websocket; optional
|
||||
Redis pub/sub adapter for multi-replica fan-out.
|
||||
podcastfeed/ ← ported from plugin's internal/podcastfeed/ (~350 LOC)
|
||||
RSS feed refresher; registered as a scheduled task.
|
||||
service.go ← thin orchestrator wiring the above to silo's existing
|
||||
catalog/playback/session/auth services.
|
||||
```
|
||||
|
||||
### Wiring touchpoints in existing silo code
|
||||
|
||||
| Existing file | Change |
|
||||
|---|---|
|
||||
| `internal/api/router.go` | Mount silo-native audiobook routes under `/api/v1/audiobooks/*`; mount ABS routes under `/abs/*` plus the small set of legacy `/api/*` paths ABS clients hit; mount the `/abs/socket.io/` endpoint. |
|
||||
| `internal/scanner/` | Per-library dispatch on `media_libraries.kind`. New parsers for `audiobooks` and `podcasts`. |
|
||||
| `cmd/silo/main.go` (task wiring) | Register `podcastfeed.Refresher` as a scheduled task. |
|
||||
| `internal/playback/` | Accept `media_items.type='audiobook'` and `'podcast_episode'` (mostly already generic). |
|
||||
| `web/src/pages/`, `web/src/player/` | New audiobook/podcast pages and player component. |
|
||||
|
||||
### What the plugin contributed that is dropped
|
||||
|
||||
The plugin's other packages — `server`, `store`, `enrich`, `recommend`,
|
||||
`smartcoll`, `event`, `consumer`, `libsync`, `migrate`, `runtime`,
|
||||
`bookref`, `cdn`, `mediatoken`, `streaming` — are either dropped (out-of-scope
|
||||
features) or replaced by reuse of silo's existing equivalents (catalog store,
|
||||
media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### Total code budget
|
||||
|
||||
- ~7.4k LOC ported from the plugin.
|
||||
- ~1.5–2k LOC new TypeScript/TSX in silo's SPA.
|
||||
- ~400 LOC of new Go glue (scanner branches, router wiring, service.go).
|
||||
- Three new SQL migrations in the current stack (`147_abs_sessions`,
|
||||
`157_podcast_feeds`, `160_audiobooks_feature_flag`) plus one documented
|
||||
no-op migration for the already-present media-folder type column.
|
||||
|
||||
## Data model
|
||||
|
||||
### Reused silo tables (no schema changes)
|
||||
|
||||
| Audiobook concept | Silo table | Notes |
|
||||
|---|---|---|
|
||||
| Audiobook entity | `media_items` | New value: `type='audiobook'`. Existing title/year/overview/poster/sort_title cover the basics. |
|
||||
| Podcast (show) | `media_items` | New value: `type='podcast'`. |
|
||||
| Podcast episode | `episodes` | Parallel to TV episodes; FK to parent `media_items` row. |
|
||||
| Audio file + chapters | `media_files` | The `chapters jsonb` column from migration 066 stores chapters exactly as silo already does for video. |
|
||||
| Library scope | `media_libraries` + `media_folders` + `media_item_libraries` | Audiobook libraries are libraries with `kind='audiobooks'` (see below). |
|
||||
| Author / narrator | `people` + `item_people` | Two new role string constants: `'author'`, `'narrator'`. |
|
||||
| Listening position | `user_watch_progress` | Per-profile position in file. Naming reads as video-only but works fine for audio. |
|
||||
| Active play session | `user_playback_sessions` | Generic enough to host audiobook sessions. |
|
||||
| Generic auth | `auth_sessions` | Used by silo's own clients. ABS clients use the new `abs_sessions` table. |
|
||||
| Series / author shelves | `library_collections` + `library_collection_items` | Reuse for audiobook series, "by author" collections. |
|
||||
|
||||
### New tables (two migrations)
|
||||
|
||||
1. **`abs_sessions`** (migration 147) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token hash, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token_hash`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
|
||||
2. **`podcast_feeds`** (migration 157) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
|
||||
### Schema-touch column add
|
||||
|
||||
- `media_libraries.kind` — `'movies' | 'tv' | 'audiobooks' | 'podcasts'`. If a
|
||||
similar column already exists on `media_libraries`, reuse it; otherwise this
|
||||
is a third small migration. The scanner reads it to pick the per-library
|
||||
parser.
|
||||
|
||||
### Plugin migrations NOT ported
|
||||
|
||||
Smart collections, share links, embeddings, content restrictions,
|
||||
metadata-provider integration, request provider, file cache, listening stats aggregates,
|
||||
reading goals, notification prefs, standalone-mode tables, recommender
|
||||
embeddings. All represent features dropped from scope.
|
||||
|
||||
### Data migration
|
||||
|
||||
None. The plugin's `audiobooks.*` Postgres schema is left intact until cutover
|
||||
is confirmed, then dropped. Silo's scanner repopulates from the filesystem on
|
||||
first run. ABS clients re-authenticate once (their tokens lived in the plugin
|
||||
DB, not silo's). This is the one user-visible cost.
|
||||
|
||||
## Routing & ABS-compatibility
|
||||
|
||||
All routes mount on silo's main `:8080` HTTP listener.
|
||||
|
||||
### 1. Silo-native audiobook API
|
||||
|
||||
- Prefix: `/api/v1/audiobooks/*`.
|
||||
- Examples: `GET /api/v1/audiobooks/library/{id}/items`,
|
||||
`GET /api/v1/audiobooks/{item_id}`,
|
||||
`POST /api/v1/audiobooks/{item_id}/progress`.
|
||||
- Mounted next to other v1 routes. Standard silo auth middleware. JSON shapes
|
||||
follow silo's existing patterns — not the ABS shape.
|
||||
|
||||
### 2. ABS-compatible API
|
||||
|
||||
- Prefix: `/abs/*` for the bulk of client-facing endpoints, plus the small set
|
||||
of `/api/*` paths ABS clients hardcode (e.g. `/api/libraries`, `/api/me`,
|
||||
`/api/items/{id}`). To avoid colliding with silo's existing `/api/v1/*`,
|
||||
ABS gets its full namespace at `/abs/`; ABS-legacy `/api/*` paths are
|
||||
registered explicitly and scoped to ABS auth.
|
||||
- Auth: a separate middleware `audiobooks.RequireABSSession` validates the ABS
|
||||
bearer token against `abs_sessions`.
|
||||
- Login: `POST /abs/login` accepts ABS-shaped credentials, internally calls
|
||||
silo's existing auth backend (same code path as `POST /api/v1/auth/login`),
|
||||
then issues an ABS token bound to a row in `abs_sessions`. No password
|
||||
handling lives in audiobooks code itself.
|
||||
|
||||
### 3. ABS Socket.io endpoint
|
||||
|
||||
- Path: `/abs/socket.io/`.
|
||||
- Uses silo's existing `gorilla/websocket` upgrader. The ported `abssocket/`
|
||||
package handles Socket.io handshake, framing, and the long-polling fallback.
|
||||
- Multi-replica fan-out: if `REDIS_URL` is set, use a Redis pub/sub adapter
|
||||
(already implemented in plugin). Unset → in-memory single-replica.
|
||||
|
||||
### What goes away
|
||||
|
||||
- No standalone listener — main `:8080` only.
|
||||
- No `SILO_HOST_URL` / `SILO_PLUGIN_TOKEN` env plumbing — those existed because
|
||||
the plugin called back into silo over HTTP. Direct Go calls replace this.
|
||||
- No host plugin-proxy in the request path.
|
||||
|
||||
### Streaming reuse
|
||||
|
||||
ABS stream URLs internally rewrite to silo's existing
|
||||
`/api/v1/stream/{session_id}` machinery — direct play, range requests,
|
||||
transcode fallback if needed, media-token signing. **Zero new transcode or
|
||||
stream code.**
|
||||
|
||||
## Scanner integration
|
||||
|
||||
### Per-library dispatch (~30 LOC in `internal/scanner`)
|
||||
|
||||
The scanner reads `media_libraries.kind` and picks a parser. Existing
|
||||
`movies`/`tv` paths are unchanged. New branches: `audiobooks`, `podcasts`.
|
||||
|
||||
### Audiobook parser (~200 LOC, `internal/scanner/audiobook.go`)
|
||||
|
||||
- Folder convention: one audiobook = one folder. Files = chapters in filename
|
||||
order, or one `.m4b` with embedded chapters.
|
||||
- Recognized extensions: `.m4b`, `.mp3`, `.m4a`, `.flac`, `.opus`.
|
||||
- Single `.m4b`: one `media_items` row + one `media_files` row. Embedded
|
||||
chapters extracted via the existing ffprobe call site → `media_files.chapters`.
|
||||
- Multi-file folder: one `media_items` row + N `media_files` rows. Chapter
|
||||
index = filename ordering; each file's `chapters` carries one synthesized
|
||||
chapter for that file.
|
||||
- Metadata: ID3 / MP4 tags (title, author, narrator, series, year, cover).
|
||||
Reuses silo's existing tag-extraction helpers.
|
||||
|
||||
### Podcast parser (~150 LOC, `internal/scanner/podcast.go`)
|
||||
|
||||
Two paths:
|
||||
|
||||
1. **Filesystem podcasts** (downloaded episodes on disk): folder = podcast,
|
||||
file = episode. Same `series → episodes` shape silo already handles for TV.
|
||||
2. **RSS-subscribed podcasts**: no filesystem walk. The `podcastfeed.Refresher`
|
||||
scheduled task fetches RSS, upserts `media_items` (podcast) + `episodes`,
|
||||
optionally downloads enclosures to a configured cache dir (reusing silo's
|
||||
existing download/cache infrastructure). `media_files` rows point at the
|
||||
cached file once downloaded.
|
||||
|
||||
### Author / narrator extraction
|
||||
|
||||
Tag-derived names → upsert into `people` → link via `item_people` with roles
|
||||
`'author'` / `'narrator'`. Same upsert pattern silo already uses for actors
|
||||
and directors. No new code, just two more role string constants.
|
||||
|
||||
### External enrichment
|
||||
|
||||
Deferred. The scanner extracts local tags and identity hints only. Canonical
|
||||
enrichment remains the responsibility of the audiobook metadata plugin/provider
|
||||
path and is not ported in this foundation pass.
|
||||
|
||||
### Scheduled task
|
||||
|
||||
`podcastfeed.Refresher` runs every 10 minutes (matches plugin behavior).
|
||||
Registered with silo's existing task manager. No other new background jobs.
|
||||
|
||||
### Impact on existing scanner behavior
|
||||
|
||||
None for movie/TV libraries. Audiobook/podcast libraries were not being
|
||||
scanned before; now they are when `kind` is set appropriately.
|
||||
|
||||
## Web UI surfaces
|
||||
|
||||
### Stack
|
||||
|
||||
Silo's existing React 19 + Vite SPA at `web/src/`. React Query, radix-ui,
|
||||
tailwind. The plugin's SPA uses the same stack so component idioms transfer
|
||||
cleanly, but we are writing fresh silo pages — the plugin's `web/` is dropped.
|
||||
|
||||
### New pages under `web/src/pages/audiobooks/`
|
||||
|
||||
| Route | Purpose |
|
||||
|---|---|
|
||||
| `/audiobooks` | Home: continue-listening shelf, recent additions, library-wide browse. |
|
||||
| `/audiobooks/library/:id` | Library view: grid of audiobooks, filter/sort, paginated. |
|
||||
| `/audiobooks/book/:id` | Detail: cover, author, narrator, series, chapter list, play/continue CTA. |
|
||||
| `/audiobooks/authors`, `/audiobooks/series` | Index pages built on `library_collections`. |
|
||||
| `/podcasts` | Subscribed podcasts grid. |
|
||||
| `/podcasts/show/:id` | Podcast detail + episode list. |
|
||||
| `/podcasts/episode/:id` | Episode detail + play. |
|
||||
|
||||
### Player
|
||||
|
||||
`web/src/player/AudiobookPlayer.tsx`:
|
||||
|
||||
- HTML5 `<audio>` (no HLS / transcoding needed for typical audio). Falls back
|
||||
to silo's existing transcode flow only if the file needs it.
|
||||
- Chapter list panel, sleep timer, playback rate (0.5×–3×), 30-second seek
|
||||
buttons, skip-silence toggle.
|
||||
- Position updates via `POST /api/v1/audiobooks/{id}/progress` at 5–10s
|
||||
intervals + on pause/seek. Mirrors silo's existing video progress cadence.
|
||||
|
||||
### Navigation
|
||||
|
||||
Add "Audiobooks" and "Podcasts" to the existing sidebar/library switcher,
|
||||
driven by `media_libraries.kind`. Same dispatch point silo uses today for
|
||||
Movies vs TV.
|
||||
|
||||
### Reused silo components (no copies)
|
||||
|
||||
Card grids, library shelves, search box, profile/auth chrome, image cache,
|
||||
virtualized list (`@tanstack/react-virtual` already in `package.json`).
|
||||
|
||||
### Not building (out of scope)
|
||||
|
||||
Smart collections UI, share-link manager, request submission UI, embeddings
|
||||
"similar books", admin enrich button, content-restriction admin.
|
||||
|
||||
### Frontend volume
|
||||
|
||||
Roughly 8 new page components + 1 player + a handful of audiobook-specific
|
||||
hooks/types under `web/src/hooks/audiobooks/` and `web/src/lib/audiobooks/`.
|
||||
~1.5–2k LOC of new TS/TSX total. No new top-level dependencies beyond what
|
||||
silo's `package.json` already has.
|
||||
|
||||
## Plugin retirement & rollout
|
||||
|
||||
### Sibling repos
|
||||
|
||||
| Repo | Action |
|
||||
|---|---|
|
||||
| `silo-plugin-audiobooks` | Archive on GitHub; remove from catalog. |
|
||||
| `silo-plugin-local-audiobooks` | Archive (silo scanner replaces it). |
|
||||
| `silo-plugin-bookwarehouse-audio` | Archive (out of scope). |
|
||||
| `silo-plugin-audiobook-requests` | Untouched — request flow out of scope. |
|
||||
|
||||
Stance: **archive, do not delete.** Leaves a recoverable home for any feature
|
||||
dropped from this port that might come back later.
|
||||
|
||||
### Catalog update
|
||||
|
||||
One-line PR in `silo-plugins`: remove the three audiobook-related plugin
|
||||
entries from `manifest.json`. The fourth (audiobook-requests) stays.
|
||||
|
||||
### Rollout sequence
|
||||
|
||||
1. Land the absorbed code in silo-server behind a feature-flag server setting
|
||||
(`audiobooks.enabled`, default off).
|
||||
2. Side-by-side: silo-server has audiobooks compiled in, the plugin is still
|
||||
installed and running on this host. Verify silo's flow end-to-end
|
||||
(scanner → SPA → ABS clients).
|
||||
3. Stop the plugin runtime. Flip `audiobooks.enabled` on. ABS clients
|
||||
reconnect to silo's `/abs/*` endpoints (config change in the ABS app if
|
||||
the base URL differs).
|
||||
4. Remove plugin entries from catalog; archive repos.
|
||||
5. After confidence in cutover, `DROP SCHEMA audiobooks CASCADE` in Postgres.
|
||||
|
||||
### Rollback
|
||||
|
||||
Revert the silo flag. Until step 4 ships, the plugin is still installed and
|
||||
runnable — single-flag rollback.
|
||||
|
||||
### Data migration
|
||||
|
||||
None. Scanner repopulates from filesystem on first run. ABS clients
|
||||
re-authenticate once. This is the one user-visible cost.
|
||||
|
||||
## Testing
|
||||
|
||||
- **Unit tests** for the ported `abs/`, `abssocket/`, `podcastfeed/` packages
|
||||
carry over from the plugin where they apply; rewrite tests that depended on
|
||||
the plugin's separate schema.
|
||||
- **Integration tests** against the real silo Postgres (per project preference
|
||||
to avoid mocking the DB):
|
||||
- Scanner: audiobook folder → `media_items` + `media_files` + chapters.
|
||||
- ABS auth: `POST /abs/login` issues a token rooted in silo's `users`.
|
||||
- ABS playback: GET a library → GET an item → stream a chapter → progress upsert.
|
||||
- Podcast refresher: feed URL → upserted episodes.
|
||||
- **Manual E2E**: an ABS mobile client points at silo, browses, plays, scrubs
|
||||
through chapters, sleep-timer expires; silo SPA shows the listening
|
||||
progress and resumes from the right chapter.
|
||||
|
||||
## Risks & open questions
|
||||
|
||||
- **Socket.io protocol port quality.** The plugin's `abssocket/` is ~250 LOC of
|
||||
Socket.io framing on top of gorilla/websocket. It works today for the
|
||||
plugin; once mounted on silo's main listener, verify the long-polling
|
||||
fallback path still works under silo's middleware stack (auth, rate-limit,
|
||||
request-id).
|
||||
- **`media_libraries.kind` column.** If the existing schema already has an
|
||||
equivalent column under a different name, use that instead. Verify before
|
||||
writing migration 134/135.
|
||||
- **`media_files.chapters` shape compatibility.** Silo's chapter JSONB shape
|
||||
was designed for video chapters. Confirm the audiobook ffprobe output maps
|
||||
cleanly into the same shape, or extend the JSON tolerantly.
|
||||
- **Search.** The plugin had its own search index for books/authors. Silo's
|
||||
existing FTS (catalog/pg-fts) needs to index audiobooks too; verify that
|
||||
setting `type='audiobook'` is enough or whether the FTS index needs a small
|
||||
config tweak.
|
||||
- **Profile-scoped vs user-scoped state.** `user_watch_progress` is keyed —
|
||||
confirm it's keyed on profile, not just user (CLAUDE.md notes silo
|
||||
separates login accounts from household profiles).
|
||||
|
||||
## Next step
|
||||
|
||||
Hand off to the `writing-plans` skill to produce a phased implementation plan
|
||||
covering migrations, package ports, router wiring, scanner branches, SPA
|
||||
pages, and cutover. The plan's first phase should resolve the open questions
|
||||
in the **Risks** section before code lands.
|
||||
@@ -0,0 +1,472 @@
|
||||
# Marker sources & TheIntroDB contribution — design
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Status:** Draft (rev 2), pending review
|
||||
**Scope:** Review the current TheIntroDB marker integration, formalize a multi-source
|
||||
marker-provider model (query-all / best-wins), and add the ability to contribute Silo's own
|
||||
markers back to TheIntroDB. This round designs the **server-side API and data model only** —
|
||||
web-admin and mobile surfaces are deferred to a UI-only follow-up. No client (Android/Apple)
|
||||
changes required.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
**Rev 2 changes:** contribution is configured **per provider** (off by default); all new web
|
||||
UI is **deferred** and the backend API is specified in full instead.
|
||||
|
||||
## Decisions locked for this design
|
||||
|
||||
These were chosen up front and constrain everything below:
|
||||
|
||||
1. **Deliverable now:** design/plan only. No source changes land from this document; it is
|
||||
the blueprint a follow-up implementation plan expands.
|
||||
2. **What we contribute:** *both* local auto-detected markers *and* manual corrections. Local
|
||||
detection only produces **intro markers for episodes**, so credits/recap/preview and all
|
||||
movies can only be contributed via manual correction — this asymmetry drives the design.
|
||||
3. **Account model:** a single server-wide TheIntroDB account, reusing the existing
|
||||
`introdb.api_key` server setting. Per-user accounts are a noted future seam, not built now.
|
||||
4. **Multi-source dispatch:** query all enabled online sources and keep the highest-quality
|
||||
result per segment ("best wins"). This makes *real per-segment confidence* (gap #2 below)
|
||||
a hard prerequisite, not an optional nicety.
|
||||
5. **Contribution is configured per provider, off by default.** Every contribution-capable
|
||||
provider has its own enable switch and submit settings, so when more providers exist an
|
||||
operator chooses *which* ones to submit to. Both the master contribute switch and the
|
||||
auto-submit-local-detections switch default to off; a single setting flips each on.
|
||||
6. **API-first, UI deferred.** This round fully specifies the backend (manual marker
|
||||
read/write/clear, contribution trigger + status, per-provider provider config, key
|
||||
validation and stats) so any future surface — web admin or mobile — builds on a complete
|
||||
contract. No new web UI is designed here beyond noting where it will eventually attach.
|
||||
|
||||
## Problem
|
||||
|
||||
Silo currently detects intro/recap/credits/preview markers from two places — a local
|
||||
chromaprint/chapter analyzer (`internal/intromarkers`) and the online TheIntroDB provider
|
||||
(`internal/markers/introdb`) — reconciled by a source-priority ladder at write time. The
|
||||
online side is cleanly abstracted behind a `markers.Provider` interface and a
|
||||
`markers.Registry`, which is good, but three things are missing:
|
||||
|
||||
1. The current TheIntroDB provider has **correctness gaps** measured against the real v3 API
|
||||
contract (TVDB lookups dropped, real confidence discarded — see "Review findings").
|
||||
2. There is **no path to contribute** Silo's markers back to TheIntroDB, even though the v3
|
||||
API exposes a submission endpoint and the package was deliberately built read-only.
|
||||
3. The provider abstraction is **fetch-only and first-hit**, so it does not yet support
|
||||
"several online sources, keep the best answer" or "this source can also accept
|
||||
contributions, and I choose which sources to submit to."
|
||||
|
||||
## Review findings — current implementation
|
||||
|
||||
### What is already right (keep it)
|
||||
|
||||
- **Clean source abstraction.** `markers.Provider` (`ID()`, `FetchMarkers()`) + `markers.Registry`
|
||||
with `FetchFirstHit` (`internal/markers/types.go`). TheIntroDB is *not* hardcoded; a second
|
||||
provider is one `Register()` call (`cmd/silo/main.go`).
|
||||
- **Multi-source-aware schema.** `media_files` carries per-segment provenance for all four
|
||||
kinds: `{intro,credits,recap,preview}_markers_{source,provider,confidence,algorithm,detected_at}`
|
||||
(migrations 108/109/137).
|
||||
- **Write-time reconciliation across source classes.** `CanWriteMarker`
|
||||
(`internal/markers/write.go`) + `MarkerSourcePriority` (`internal/models/marker_source.go`):
|
||||
`manual(4) > online/plugin(3) > s3(2) > scanner(1)`, confidence breaks ties. Local detection
|
||||
uses its own priority-gated path (`intromarkers.Repository.PatchIntroMarker`).
|
||||
- **Solid HTTP hygiene** in `internal/markers/introdb/client.go`: conservative rate limiter
|
||||
(2 req/s, burst 5; documented limit is 30 req/10s), retry with exponential backoff,
|
||||
`Retry-After` handling, 24h TTL cache with negative caching, hot-reloadable API key over
|
||||
Redis pub/sub.
|
||||
|
||||
### Gaps (measured against TheIntroDB's own v3 client + Jellyfin plugin)
|
||||
|
||||
| # | Gap | Evidence | Impact |
|
||||
|---|-----|----------|--------|
|
||||
| 1 | **TVDB lookups silently dropped.** Provider reads only TMDB+IMDB; client can't send `tvdb_id`. | `internal/markers/introdb/provider.go` (reads `ExternalIDKeyTMDB`/`IMDB` only); `client.go` `FetchEpisode/FetchMovie` build `tmdb_id` else `imdb_id`. | Titles matched only by TVDB (very common for anime / TheTVDB-first libraries) get **zero** markers, even though the resolver already populates `TvdbID` and the v3 API + both official clients support `tvdb_id`. Correctness bug. |
|
||||
| 2 | **Real confidence + submission_count discarded; hardcoded 0.9.** | `internal/markers/introdb/types.go` `segmentTimestamps` has only `start_ms`/`end_ms`; `pickMarker` sets `Confidence: 0.9`. | The `/media` response carries per-segment `confidence` and `submission_count`. Without them, "best wins" merging can't rank sources, and write-time confidence tie-breaks are meaningless. Hard blocker for the chosen dispatch model. |
|
||||
| 3 | **Naive multi-candidate selection.** `pickMarker` takes the *first* usable entry. | `internal/markers/introdb/provider.go`. | When several candidates remain (e.g. no duration match), first-usable may not be the most-submitted/highest-confidence. Minor today (duration narrows the set), but compounds once we rank by confidence. |
|
||||
| 4 | **No contribution path** (by design). | `internal/markers/introdb/types.go` package doc: "Submissions are intentionally not supported." | The whole point of this effort. |
|
||||
| 5 | **Provider is fetch-only.** No submit capability in the interface. | `markers.Provider` in `internal/markers/types.go`. | Contribution needs an optional `Submitter` capability so non-contributing sources don't have to implement it, and so submission is per provider. |
|
||||
| 6 | **No key validation / contribution stats.** | `IntroDBCredentialCard` in `web/src/pages/admin-settings/IntegrationsSettings.tsx` only sets the key. | v3 exposes `GET /v3/user/stats` to validate a key and show contribution totals/streaks. Useful "connect & verify" probe, currently absent from the API. |
|
||||
| 7 | **Single collapsed confidence in the write payload.** `BuildUpdatePayload` promotes one max confidence across all segments. | `internal/markers/write.go`. | The DB has *per-segment* confidence columns, but a merged result (intro from source A, credits from source B) can't populate them independently until per-segment confidence is threaded through the payload. Coupled to gap #2. |
|
||||
|
||||
## Authoritative API contract (TheIntroDB v3)
|
||||
|
||||
Confirmed from TheIntroDB's own `theintrodb` npm client (`src/funcs.ts`, `src/types.ts`) and
|
||||
the official Jellyfin plugin (`TheIntroDbClient.cs`). Base URL: `https://api.theintrodb.org/v3`.
|
||||
|
||||
### `GET /media` — read (public; optional key)
|
||||
|
||||
Query: one of `tmdb_id` / `tvdb_id` / `imdb_id` (preference order tmdb → tvdb → imdb),
|
||||
plus `season` + `episode` together for TV, plus optional `duration_ms` (selects the closest
|
||||
release version). Optional `Authorization: Bearer <key>` additionally includes that user's
|
||||
*pending* submissions.
|
||||
|
||||
Response — each segment kind is an array of:
|
||||
```jsonc
|
||||
{ "start_ms": 30000|null, "end_ms": 90000|null, "confidence": 0.93|null, "submission_count": 7|null }
|
||||
```
|
||||
`start_ms:null` = "from the beginning" (intro/recap); `end_ms:null` = "to the end"
|
||||
(credits/preview). Arrays may hold multiple entries (multiple release versions) — pick by
|
||||
duration match, then by `submission_count`/`confidence`.
|
||||
|
||||
### `POST /submit` — contribute (requires key)
|
||||
|
||||
`Authorization: Bearer <user key>` required; submissions are credited to that account.
|
||||
Body (one segment per call):
|
||||
```jsonc
|
||||
{
|
||||
"tmdb_id": 1234, // required
|
||||
"imdb_id": "tt0903747", // optional
|
||||
"type": "tv" | "movie",
|
||||
"segment": "intro" | "recap" | "credits" | "preview",
|
||||
"season": 1, // TV only (number, or "1,2,3" multi-select)
|
||||
"episode": 1, // TV only
|
||||
"video_duration_ms": 2760000, // optional, strongly recommended (release-version match)
|
||||
"start_ms": 0|null, // intro/recap may be null (= beginning)
|
||||
"end_ms": 90000|null // credits/preview may be null (= end)
|
||||
}
|
||||
```
|
||||
Response: `{ "submissions": [ { "id": uuid, "status": "pending"|"accepted"|"rejected",
|
||||
"weight": number, ... } ] }`. Submissions enter as `pending`; the backend weight-averages
|
||||
timestamps across submissions and moderates to accepted/rejected.
|
||||
|
||||
Validation rules to mirror client-side: intro/recap require an **end**; credits/preview
|
||||
require a **start**; TV requires season+episode; movies omit them; max timestamp 6h
|
||||
(21,600,000 ms); `video_duration_ms` is 0 (unknown) or ≥ 300,000 ms.
|
||||
|
||||
### `GET /user/stats` — validate key + contribution stats
|
||||
|
||||
`Authorization: Bearer <key>`. Returns `{ total, accepted, pending, rejected,
|
||||
acceptance_rate, current_streak, best_streak, total_time_saved_ms, top_media }`. A 2xx with no
|
||||
`error` means the key is valid. This is both our key-validation probe and the stats display.
|
||||
|
||||
### Rate / usage limits
|
||||
|
||||
429 responses carry `X-RateLimit-{Limit,Remaining,Reset}` and `X-UsageLimit-{Limit,Remaining,Reset}`.
|
||||
Reads count against the rate limit; **submissions count against the usage limit**. The
|
||||
contribution pipeline must honor `X-UsageLimit-Reset` and back off — reuse the existing
|
||||
limiter and add usage-limit awareness.
|
||||
|
||||
## Target architecture
|
||||
|
||||
Three layers, each independently shippable. Layer A is pure correctness and unblocks B and C.
|
||||
|
||||
### Layer A — Correctness foundation (gaps #1, #2, #3, #7)
|
||||
|
||||
- **TVDB in the client + provider.** Add `tvdb_id` to the query builder in
|
||||
`internal/markers/introdb/client.go` (`FetchEpisode`/`FetchMovie` gain a `tvdbID` arg,
|
||||
preference tmdb → tvdb → imdb, cache keys extended). Provider reads
|
||||
`req.ExternalIDs[markers.ExternalIDKeyTVDB]`. Resolver already supplies it — no resolver
|
||||
change.
|
||||
- **Capture confidence + submission_count.** Extend `segmentTimestamps` with
|
||||
`Confidence *float64` and `SubmissionCount *int`; add `SubmissionCount int` to
|
||||
`markers.Marker`; `pickMarker` uses the real confidence (fallback to a named default
|
||||
constant, e.g. `0.9`, only when the field is null) instead of hardcoding.
|
||||
- **Best-candidate selection.** `pickMarker` ranks candidates by `(submission_count desc,
|
||||
confidence desc)` after duration filtering, instead of first-usable.
|
||||
- **Per-segment confidence in the payload.** Replace the single `Confidence *float64` on
|
||||
`markers.MarkerUpdatePayload` with per-segment confidence (e.g. `IntroConfidence`,
|
||||
`CreditsConfidence`, …) so a merged multi-source result writes the correct
|
||||
`{kind}_markers_confidence` per kind. Update `BuildUpdatePayload` and the
|
||||
`scanner.FileRepository.UpsertMarkers` column mapping accordingly.
|
||||
|
||||
### Layer B — Multi-source dispatch ("query all, best wins")
|
||||
|
||||
- **`Submitter` capability (optional interface).**
|
||||
```go
|
||||
// internal/markers/types.go
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
// FetchUserStats validates the configured key and returns contribution stats.
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
Only `introdb.Provider` implements it; the contribution service type-asserts
|
||||
`provider.(markers.Submitter)`. This is also what makes contribution per-provider: the
|
||||
service only ever submits to registered providers that satisfy `Submitter` *and* are enabled
|
||||
in `marker_provider_config` (C6).
|
||||
|
||||
- **Merge dispatch.** Add `Registry.FetchMerged(ctx, req) (Result, bool, error)` that fans out
|
||||
to all *fetch-enabled* providers concurrently (bounded), then merges **per segment kind**
|
||||
keeping the candidate with the highest `(submission_count, confidence)`. `FetchFirstHit`
|
||||
stays as the trivial single-provider path / fallback. With only TheIntroDB enabled,
|
||||
`FetchMerged` behaves identically to today — zero behavior change until a second source
|
||||
exists.
|
||||
|
||||
- **Cross-source-class reconciliation is unchanged.** `FetchMerged` reconciles *within* the
|
||||
online class; `CanWriteMarker` still arbitrates online vs scanner vs manual at write time.
|
||||
No double logic.
|
||||
|
||||
- **Provider enable/order config.** Driven by the `marker_provider_config` table (C6):
|
||||
`fetch_enabled` selects which providers are queried, `fetch_priority` is the merge
|
||||
tiebreaker. The metadata `library_provider_chains` DB pattern (`internal/metadata/chain.go`)
|
||||
is the upgrade path *if* per-library control is ever needed — noted, not built.
|
||||
|
||||
- **Future plugin sources (deferred seam).** A `marker_provider.v1` plugin capability would let
|
||||
third-party sources plug in. The clean seam is a `PluginMarkerProvider` adapter implementing
|
||||
`markers.Provider` (mirroring `internal/metadata/plugin_provider.go`), so the registry never
|
||||
changes. Cross-repo (silo-plugin-sdk) — explicitly out of scope here.
|
||||
|
||||
### Layer C — Contribution to TheIntroDB
|
||||
|
||||
#### C1. Submission client
|
||||
|
||||
Add to `internal/markers/introdb/client.go`:
|
||||
- `SubmitSegment(ctx, SubmissionRequest) (SubmissionResult, error)` → `POST /v3/submit`,
|
||||
Bearer key **required** (error early if `introdb.api_key` is empty), usage-limit aware.
|
||||
- `FetchUserStats(ctx) (UserStats, error)` → `GET /v3/user/stats`.
|
||||
`introdb.Provider` implements `markers.Submitter` by delegating to these.
|
||||
|
||||
#### C2. What is eligible to contribute
|
||||
|
||||
A marker is contributable iff:
|
||||
- the target provider is a registered `Submitter` with `contribute_enabled = true` (C6), and
|
||||
- its segment source ∈ `{scanner, manual}` (**never** `online` — that came *from* an online
|
||||
provider; contributing it back is circular), and
|
||||
- the file resolves to a usable external ID (tmdb/tvdb/imdb) via the existing resolver, and
|
||||
- for **auto** contribution: provider `contribute_auto_local = true`, source = `scanner`,
|
||||
kind = `intro`, confidence ≥ that provider's `contribute_min_confidence`. (Local detection is
|
||||
intro-only/episodes-only.)
|
||||
- for **manual** contribution: any kind, episodes or movies, source = `manual`.
|
||||
|
||||
#### C3. Manual marker API (write path; UI deferred)
|
||||
|
||||
Markers live on `media_files`, and an item can have several versions, so the manual API is
|
||||
**file-scoped** for precision, with an item-level convenience resolver. All under the existing
|
||||
`RequireAdmin` group (`internal/api/middleware/auth.go`). New `internal/api/handlers/admin_markers.go`:
|
||||
|
||||
- `GET /admin/files/{fileId}/markers` — current per-segment values **plus full provenance**
|
||||
(`source`, `provider`, `confidence`, `algorithm`, `detected_at`) for each of
|
||||
intro/recap/credits/preview. The read side an editor (or a script) needs.
|
||||
- `PUT /admin/files/{fileId}/markers` — upsert the **manual** layer. Body carries, per segment,
|
||||
either `{start, end}` (seconds) or `null` to clear that segment's manual value. Writes
|
||||
`source = manual` (priority 4, beats every other source) through the existing priority-gated
|
||||
`scanner.FileRepository.UpsertMarkers` path, then notifies live sessions via
|
||||
`playback.MarkerUpdateNotifier.MarkersUpdated` (mirrors `admin_intro.go`).
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` — explicit single-segment clear.
|
||||
- `GET|PUT /admin/items/{id}/markers` — item-level convenience that resolves to the primary
|
||||
file (and reports the per-version set), for callers that don't track file IDs.
|
||||
|
||||
**Clear semantics.** Clearing a manual segment nulls that segment's value and provenance;
|
||||
`markers_source` drops to the next-highest segment still present. A subsequent local
|
||||
re-detection or online lazy-fetch can repopulate it. (We can't "un-overwrite" a value a manual
|
||||
edit replaced, so clearing means "remove the manual value," not "restore the previous source.")
|
||||
|
||||
**Validation** mirrors the contribution rules so a manual edit can always be contributed:
|
||||
`end > start`; intro/recap may omit start (= 0); credits/preview may omit end (= file
|
||||
duration); values within [0, file duration].
|
||||
|
||||
A web editor (player-chrome control with a "use current time" helper) attaches to this API
|
||||
later; its UX is intentionally **out of scope** for this round.
|
||||
|
||||
#### C4. Contribution tracking (idempotency + audit)
|
||||
|
||||
The audit log that makes submission idempotent and lets us report status. Per-provider by
|
||||
construction (the `provider` column), so the same marker can be tracked/submitted to several
|
||||
providers independently.
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_contributions.up.sql (next free number at implementation time)
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL, -- 'introdb'
|
||||
segment_kind text NOT NULL, -- intro|recap|credits|preview
|
||||
source text NOT NULL, -- what we contributed: 'scanner'|'manual'
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL, -- hash(segment_kind,start_ms,end_ms,duration_ms)
|
||||
submission_id uuid, -- returned by /submit
|
||||
status text NOT NULL, -- pending|accepted|rejected|error
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
`content_hash` over the *value* gives idempotency: the same times are never resubmitted to the
|
||||
same provider, but a manual correction (different times → different hash) submits the new
|
||||
value — which is exactly how TheIntroDB's weighted averaging expects corrections to arrive.
|
||||
|
||||
#### C5. Contribution service + triggers
|
||||
|
||||
A single `internal/markers/contribute.go` `ContributionService` funnels both triggers:
|
||||
`resolve enabled submitter providers (C6) → per provider: eligibility (C2) → idempotency check
|
||||
(hash vs marker_contributions) → Submitter.SubmitMarker → persist result`. Triggers:
|
||||
- **Manual / on-demand:** `POST /admin/files/{fileId}/contribute` (C7) submits a file's eligible
|
||||
markers to enabled providers (optionally scoped to one provider/segments).
|
||||
- **Auto local detections:** a daily `ContributeMarkersTask` (`internal/taskmanager/tasks/`)
|
||||
that, for each provider with `contribute_enabled && contribute_auto_local`, scans for episode
|
||||
files with `intro_markers_source = 'scanner'` and `intro_markers_confidence ≥` that provider's
|
||||
`contribute_min_confidence` lacking a current-hash `marker_contributions` row, and submits
|
||||
them rate/usage-limited. Runs after the existing 03:30 detection task.
|
||||
|
||||
#### C6. Per-provider configuration
|
||||
|
||||
Contribution and fetch behavior is **per provider**, so a single global toggle won't do.
|
||||
Credentials stay in `server_settings` (sensitive handling: `introdb.api_key`); behavioral
|
||||
config moves to a new table keyed by provider id:
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_provider_config.up.sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY, -- 'introdb'
|
||||
fetch_enabled boolean NOT NULL DEFAULT true, -- queried during reads (Layer B)
|
||||
fetch_priority integer NOT NULL DEFAULT 100, -- merge tiebreaker
|
||||
contribute_enabled boolean NOT NULL DEFAULT false, -- master submit switch (off)
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false, -- auto-submit local detections (off)
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- A provider's submissions require `contribute_enabled = true`. With it on, **manual**
|
||||
corrections submit on demand; **auto** submission of local detections additionally requires
|
||||
`contribute_auto_local = true`. Both contribute flags default off — "easily enabled with a
|
||||
setting," exactly one row per provider.
|
||||
- This same table carries the Layer B fetch enable/order, so "which providers do we read from"
|
||||
and "which do we submit to" live in one place. New providers insert a row (contribute off).
|
||||
- Surfaced and edited through the provider API (C7), so the future admin surface lists
|
||||
providers generically rather than hard-coding `introdb`.
|
||||
|
||||
The `markers.mode` / `markers.lazy_playback` globals and the `introdb.api_key` credential are
|
||||
unchanged.
|
||||
|
||||
#### C7. Admin provider + contribution API (fully fledged)
|
||||
|
||||
The complete backend surface, all under `RequireAdmin`, so a UI is purely additive later:
|
||||
|
||||
- `GET /admin/markers/providers` — list every registered provider: id, whether it is a
|
||||
`Submitter`, its `marker_provider_config` row, and (best-effort, cached) live `/user/stats`
|
||||
for submitter providers that have a key.
|
||||
- `PUT /admin/markers/providers/{provider}` — update that provider's config row
|
||||
(`fetch_enabled`, `fetch_priority`, `contribute_enabled`, `contribute_auto_local`,
|
||||
`contribute_min_confidence`). Validated; unknown provider → 404.
|
||||
- `POST /admin/markers/providers/{provider}/validate` — validate the configured key and return
|
||||
fresh `UserStats` (calls `Submitter.FetchUserStats`). Non-submitter → 400.
|
||||
- `POST /admin/files/{fileId}/contribute` — submit this file's eligible markers (C2) to enabled
|
||||
submitter providers; optional body `{provider?, segments?[]}` to scope to one provider /
|
||||
specific kinds. Returns per (provider, segment) `SubmissionResult`. Honors idempotency (C4):
|
||||
unchanged values are skipped, not resubmitted.
|
||||
- `GET /admin/files/{fileId}/contributions` — contribution history for the file from
|
||||
`marker_contributions` (per provider/segment: status, submitted_at, error).
|
||||
|
||||
Plus the manual marker endpoints from C3. Together these cover read, edit, submit, and audit
|
||||
without any web code.
|
||||
|
||||
## Data model summary
|
||||
|
||||
- **New table `marker_contributions`** (C4) — idempotent submission audit, keyed per
|
||||
(file, provider, segment, value-hash).
|
||||
- **New table `marker_provider_config`** (C6) — per-provider fetch + contribute behavior,
|
||||
seeded with `introdb` (fetch on, contribute off).
|
||||
- **No new flat server settings.** `introdb.api_key`, `markers.mode`, `markers.lazy_playback`
|
||||
are unchanged; behavioral provider config lives in `marker_provider_config`, not string keys.
|
||||
- **No new marker columns.** Per-segment confidence columns already exist; Layer A only changes
|
||||
how they're *populated* (per-segment instead of collapsed). `submission_count` stays
|
||||
transient (merge-ranking only) unless a later need appears.
|
||||
|
||||
## Client / cross-repo impact
|
||||
|
||||
- **No Android/Apple changes required.** This adds no new marker *kinds* and no new
|
||||
client-facing read contract: markers still flow to clients via the existing `WatchDetail`
|
||||
fields, the jellycompat `GET /MediaSegments` endpoint, and the `markers_updated` realtime
|
||||
event. Manual editing and contribution are server-side (admin API); their eventual UI is
|
||||
deferred web-admin work.
|
||||
- **Plugin SDK:** untouched. `marker_provider.v1` is a deferred future capability (Layer B
|
||||
seam), not part of this work.
|
||||
- A future *per-user* contribution model (each user links their own TheIntroDB account) *would*
|
||||
be client work (a "Connect TheIntroDB" screen) — explicitly deferred.
|
||||
|
||||
## Build sequence
|
||||
|
||||
Each phase is shippable on its own; later phases depend only on earlier ones. Every phase is
|
||||
**backend-only** — no web UI.
|
||||
|
||||
1. **Phase 1 — Correctness foundation (Layer A).** TVDB lookups; capture
|
||||
confidence/submission_count; best-candidate selection; per-segment confidence in the
|
||||
payload. *Directly answers "are we doing this properly," low risk, no new surface.*
|
||||
2. **Phase 2 — Multi-source dispatch + provider config (Layer B + C6).** `Submitter` interface;
|
||||
`FetchMerged` query-all/best-wins; `marker_provider_config` table + repo driving fetch
|
||||
enable/order. *No behavior change with only TheIntroDB enabled.*
|
||||
3. **Phase 3 — Submission client + tracking + service (C1, C4, C5 core).** `SubmitSegment` /
|
||||
`FetchUserStats`; `marker_contributions` migration + repo; `ContributionService` (eligibility,
|
||||
idempotency, per-provider gating).
|
||||
4. **Phase 4 — Manual marker API (C3) + provider/contribution API (C7).** File- and item-level
|
||||
manual read/write/clear; provider list/config/validate; per-file contribute + history. The
|
||||
full admin contract, no UI.
|
||||
5. **Phase 5 — Auto-contribution (C5 task).** `ContributeMarkersTask`: per provider, when
|
||||
`contribute_enabled && contribute_auto_local`, submit episode intro markers above the
|
||||
provider's `contribute_min_confidence`, usage-limit aware, idempotent.
|
||||
|
||||
Deferred (separate UI-only follow-up): web admin surfaces over C3/C6/C7 — a marker editor in
|
||||
the player chrome and a provider/contribution panel in admin settings.
|
||||
|
||||
## Risks & trade-offs
|
||||
|
||||
- **Data-quality / pollution.** Auto-submitting chromaprint detections (confidence 0.65–0.90)
|
||||
risks polluting a provider's data. Mitigation: `contribute_enabled` and `contribute_auto_local`
|
||||
default **off** per provider, and the default `contribute_min_confidence` of 0.95 admits only
|
||||
chapter / chapter+silence detections even once enabled. Manual, high-trust corrections are the
|
||||
intended first contribution. Revisit chromaprint auto-submission after watching acceptance
|
||||
rates via `/user/stats`.
|
||||
- **Single-account weighting.** All contributions credit one server account; TheIntroDB's
|
||||
weighting may treat a single account's mass submissions differently than many users'. Note
|
||||
the per-user seam; acceptable for v1.
|
||||
- **Circular contribution.** Never contribute `source = online`. Enforced in C2 eligibility and
|
||||
worth a unit test.
|
||||
- **Usage limits.** Bulk auto-submission can hit the submission usage limit; the task must obey
|
||||
`X-UsageLimit-Reset` and resume next run. Idempotency means an interrupted run safely resumes.
|
||||
- **"Query all" cost.** `FetchMerged` issues one lookup per fetch-enabled provider; with only
|
||||
TheIntroDB it's identical to today, and per-provider caching bounds the cost as sources are
|
||||
added.
|
||||
- **Movies & non-intro segments** have no local detector, so their only contribution route is
|
||||
manual correction — acceptable and explicit.
|
||||
|
||||
## Open questions for review
|
||||
|
||||
1. **Contribute to all enabled providers, or require explicit per-call selection?** The design
|
||||
defaults `POST /admin/files/{id}/contribute` to "every enabled submitter," with an optional
|
||||
`provider` scope. Acceptable, or should submission always name its target provider?
|
||||
2. **Manual-clear behavior.** Clearing a manual segment removes the manual value and lets a
|
||||
later detection/fetch repopulate it (we can't restore the exact pre-edit value). Is
|
||||
"remove, then re-detect" the right contract, or do you want a marker history/undo stack?
|
||||
3. **Per-provider config home.** This design uses a `marker_provider_config` table (clean for N
|
||||
providers, future per-library override) rather than flat `server_settings` keys. Confirm the
|
||||
table is the direction you want.
|
||||
|
||||
## File-level blueprint
|
||||
|
||||
**Backend (Go) — Layer A**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `tvdb_id` query support; cache-key extension.
|
||||
- `internal/markers/introdb/types.go` — *modify*: `confidence`/`submission_count` fields.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: read TVDB; real confidence; best-candidate pick.
|
||||
- `internal/markers/types.go` — *modify*: per-segment confidence in `Marker`/Result plumbing.
|
||||
- `internal/markers/write.go` — *modify*: per-segment confidence in `MarkerUpdatePayload` + `BuildUpdatePayload`.
|
||||
- `internal/scanner/file_repo.go` — *modify*: map per-segment confidence in `UpsertMarkers`.
|
||||
|
||||
**Backend (Go) — Layer B + per-provider config**
|
||||
- `internal/markers/types.go` — *modify*: `Submitter` interface; `SubmissionRequest`/`Result`, `UserStats`; `Registry.FetchMerged`.
|
||||
- `internal/markers/provider_config.go` — *create*: `marker_provider_config` repo (load/update; drives fetch enable/order + contribute gating).
|
||||
- `migrations/<next>_marker_provider_config.{up,down}.sql` — *create*.
|
||||
|
||||
**Backend (Go) — Layer C**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `SubmitSegment` (`POST /v3/submit`), `FetchUserStats` (`GET /v3/user/stats`), usage-limit handling.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: implement `markers.Submitter`; drop the "submissions not supported" note in `types.go`.
|
||||
- `internal/markers/contribute.go` — *create*: `ContributionService` (eligibility, idempotency, per-provider gating, submit, persist).
|
||||
- `internal/markers/contribution_repo.go` — *create*: `marker_contributions` CRUD + value-hash check.
|
||||
- `migrations/<next>_marker_contributions.{up,down}.sql` — *create*.
|
||||
- `internal/api/handlers/admin_markers.go` — *create*: manual marker read/write/clear (C3) + per-file contribute + history (C7 file endpoints).
|
||||
- `internal/api/handlers/admin_marker_providers.go` — *create*: provider list/config/validate (C7 provider endpoints).
|
||||
- `internal/api/router.go` — *modify*: register the new `RequireAdmin` routes.
|
||||
- `internal/taskmanager/tasks/contribute_markers.go` — *create*: daily auto-contribution task.
|
||||
- `cmd/silo/main.go` — *modify*: wire `ContributionService`, provider-config repo, the task, and the `Submitter`.
|
||||
|
||||
**Frontend (React/TS) — deferred (not this round)**
|
||||
- Marker editor over `PUT /admin/files/{id}/markers`; provider/contribution panel over the C7
|
||||
endpoints; `IntroDBCredentialCard` "validate key & show stats". Listed for traceability only.
|
||||
|
||||
**Reference (read-only, not in this repo)**
|
||||
- TheIntroDB v3 contract verified against `github.com/TheIntroDB/theintrodb-npm`
|
||||
(`src/funcs.ts`, `src/types.ts`, `docs/`) and `github.com/TheIntroDB/jellyfin-plugin`
|
||||
(`TheIntroDB/Api/*.cs`).
|
||||
@@ -28,8 +28,10 @@ require (
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/pressly/goose/v3 v3.27.1
|
||||
go.n16f.net/thumbhash v1.1.0
|
||||
golang.org/x/image v0.39.0
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
)
|
||||
|
||||
@@ -37,18 +39,21 @@ require (
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/oklog/run v1.1.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
@@ -67,15 +72,14 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sony/sonyflake v1.3.0
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/text v0.36.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220 h1:tiwfULdh9yOW7HO6fYf8v+lXIsrCrGfp3GM8ymhrBME=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0 h1:0yJpD0RP1AetGsWVCXoPih2ggLjN8hVp8blS+2RkIWU=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
@@ -38,6 +38,7 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
|
||||
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -45,6 +46,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
@@ -95,8 +98,8 @@ github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -112,12 +115,16 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
@@ -127,22 +134,28 @@ github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2
|
||||
github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
|
||||
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/sony/sonyflake v1.3.0 h1:tiB4Dlp0lnmKp/h6BLXA14P8Qi+LYS9+0QRpcrKHvg4=
|
||||
github.com/sony/sonyflake v1.3.0/go.mod h1:LORtCywH/cq10ZbyfhKrHYgAUGH7mOBa76enV9txy/Y=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -175,20 +188,22 @@ go.n16f.net/thumbhash v1.1.0 h1:aBEvuAd4yiwzeQ7Sm4BZoHJYbrQ1ewjrmrRlCE79snk=
|
||||
go.n16f.net/thumbhash v1.1.0/go.mod h1:mo9pP7WtfdV9ojIamGFR/Vc0PaPA2l0CUtmYQf/SweU=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
@@ -204,19 +219,18 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
|
||||
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
|
||||
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
|
||||
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -233,3 +247,11 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
|
||||
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
|
||||
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
|
||||
modernc.org/libc v1.72.1 h1:db1xwJ6u1kE3KHTFTTbe2GCrczHPKzlURP0aDC4NGD0=
|
||||
modernc.org/libc v1.72.1/go.mod h1:HRMiC/PhPGLIPM7GzAFCbI+oSgE3dhZ8FWftmRrHVlY=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
|
||||
@@ -113,20 +113,34 @@ func NewAdminHandler(
|
||||
|
||||
// createUserRequest represents the JSON body for POST /admin/users.
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions []string `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions createStringSliceField `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
}
|
||||
|
||||
type createStringSliceField struct {
|
||||
Set bool
|
||||
Value []string
|
||||
}
|
||||
|
||||
func (f *createStringSliceField) UnmarshalJSON(data []byte) error {
|
||||
f.Set = true
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
f.Value = []string{}
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(data, &f.Value)
|
||||
}
|
||||
|
||||
type updateLibraryIDsField struct {
|
||||
@@ -391,7 +405,11 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "max_profiles must be at least 1")
|
||||
return
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(req.Permissions)
|
||||
permissions := auth.DefaultUserPermissions()
|
||||
if req.Permissions.Set {
|
||||
permissions = req.Permissions.Value
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(permissions)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
// AdminMarkerProvidersHandler serves the per-provider marker config + key
|
||||
// validation API under the RequireAdmin group.
|
||||
type AdminMarkerProvidersHandler struct {
|
||||
Registry *markers.Registry
|
||||
Config *markers.ProviderConfigStore
|
||||
EventBus cache.EventBus
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewAdminMarkerProvidersHandler constructs the handler.
|
||||
func NewAdminMarkerProvidersHandler(registry *markers.Registry, config *markers.ProviderConfigStore, eventBus cache.EventBus, logger *slog.Logger) *AdminMarkerProvidersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &AdminMarkerProvidersHandler{Registry: registry, Config: config, EventBus: eventBus, logger: logger}
|
||||
}
|
||||
|
||||
type providerConfigResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
SourceType string `json:"source_type,omitempty"`
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
PluginInstallationID int `json:"plugin_installation_id,omitempty"`
|
||||
CapabilityID string `json:"capability_id,omitempty"`
|
||||
IsSubmitter bool `json:"is_submitter"`
|
||||
FetchEnabled bool `json:"fetch_enabled"`
|
||||
FetchPriority int `json:"fetch_priority"`
|
||||
ContributeEnabled bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
|
||||
type markerUserStatsResponse struct {
|
||||
Total int `json:"total"`
|
||||
Accepted int `json:"accepted"`
|
||||
Pending int `json:"pending"`
|
||||
Rejected int `json:"rejected"`
|
||||
AcceptanceRate float64 `json:"acceptance_rate"`
|
||||
CurrentStreak int `json:"current_streak"`
|
||||
BestStreak int `json:"best_streak"`
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) submitterIDs() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if _, ok := p.(markers.Submitter); ok {
|
||||
out[p.ID()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) providerDescriptions() map[string]markers.ProviderDescriptor {
|
||||
out := map[string]markers.ProviderDescriptor{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
desc := markers.ProviderDescriptor{ID: p.ID()}
|
||||
if described, ok := p.(markers.DescribedProvider); ok {
|
||||
desc = described.ProviderDescription()
|
||||
}
|
||||
if desc.ID == "" {
|
||||
desc.ID = p.ID()
|
||||
}
|
||||
out[p.ID()] = desc
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HandleListProviders lists registered providers with their config + capability.
|
||||
func (h *AdminMarkerProvidersHandler) HandleListProviders(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
submitters := h.submitterIDs()
|
||||
descriptions := h.providerDescriptions()
|
||||
out := []providerConfigResponse{}
|
||||
if h.Registry != nil {
|
||||
for _, provider := range h.Registry.Providers() {
|
||||
c, ok := h.Config.Get(provider.ID())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
} else {
|
||||
for _, c := range h.Config.List() {
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"providers": out})
|
||||
}
|
||||
|
||||
// HandleUpdateProvider updates a provider's config row.
|
||||
func (h *AdminMarkerProvidersHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
existing, ok := h.Config.Get(provider)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Unknown marker provider")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
FetchEnabled *bool `json:"fetch_enabled"`
|
||||
FetchPriority *int `json:"fetch_priority"`
|
||||
ContributeEnabled *bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal *bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence *float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if body.FetchEnabled != nil {
|
||||
existing.FetchEnabled = *body.FetchEnabled
|
||||
}
|
||||
if body.FetchPriority != nil {
|
||||
existing.FetchPriority = *body.FetchPriority
|
||||
}
|
||||
if body.ContributeEnabled != nil {
|
||||
existing.ContributeEnabled = *body.ContributeEnabled
|
||||
}
|
||||
if body.ContributeAutoLocal != nil {
|
||||
existing.ContributeAutoLocal = *body.ContributeAutoLocal
|
||||
}
|
||||
if body.ContributeMinConfidence != nil {
|
||||
v := *body.ContributeMinConfidence
|
||||
if v < 0 || v > 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "contribute_min_confidence must be between 0 and 1")
|
||||
return
|
||||
}
|
||||
existing.ContributeMinConfidence = v
|
||||
}
|
||||
if err := h.Config.Update(r.Context(), existing); err != nil {
|
||||
h.logger.Error("admin markers: update provider config failed", "provider", provider, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update provider")
|
||||
return
|
||||
}
|
||||
if h.EventBus != nil {
|
||||
_ = h.EventBus.Publish(r.Context(), cache.ChannelAdmin, cache.Event{
|
||||
Type: cache.EventMarkerProviderConfigChanged,
|
||||
Payload: provider,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, toProviderConfigResponse(existing, h.submitterIDs()[provider], h.providerDescriptions()[provider]))
|
||||
}
|
||||
|
||||
// HandleValidateProvider validates the provider's configured key and returns stats.
|
||||
func (h *AdminMarkerProvidersHandler) HandleValidateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Registry == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
var submitter markers.Submitter
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if p.ID() == provider {
|
||||
if s, ok := p.(markers.Submitter); ok {
|
||||
submitter = s
|
||||
}
|
||||
}
|
||||
}
|
||||
if submitter == nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Provider does not support contribution")
|
||||
return
|
||||
}
|
||||
stats, err := submitter.FetchUserStats(r.Context())
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": true, "stats": toMarkerUserStatsResponse(stats)})
|
||||
}
|
||||
|
||||
func toProviderConfigResponse(c markers.ProviderConfig, isSubmitter bool, desc markers.ProviderDescriptor) providerConfigResponse {
|
||||
return providerConfigResponse{
|
||||
Provider: c.Provider,
|
||||
DisplayName: desc.DisplayName,
|
||||
SourceType: desc.SourceType,
|
||||
PluginID: desc.PluginID,
|
||||
PluginInstallationID: desc.PluginInstallationID,
|
||||
CapabilityID: desc.CapabilityID,
|
||||
IsSubmitter: isSubmitter,
|
||||
FetchEnabled: c.FetchEnabled,
|
||||
FetchPriority: c.FetchPriority,
|
||||
ContributeEnabled: c.ContributeEnabled,
|
||||
ContributeAutoLocal: c.ContributeAutoLocal,
|
||||
ContributeMinConfidence: c.ContributeMinConfidence,
|
||||
}
|
||||
}
|
||||
|
||||
func toMarkerUserStatsResponse(s markers.UserStats) markerUserStatsResponse {
|
||||
return markerUserStatsResponse{
|
||||
Total: s.Total,
|
||||
Accepted: s.Accepted,
|
||||
Pending: s.Pending,
|
||||
Rejected: s.Rejected,
|
||||
AcceptanceRate: s.AcceptanceRate,
|
||||
CurrentStreak: s.CurrentStreak,
|
||||
BestStreak: s.BestStreak,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
type fakeMarkerStatsSubmitter struct{}
|
||||
|
||||
func (fakeMarkerStatsSubmitter) ID() string { return "introdb" }
|
||||
func (fakeMarkerStatsSubmitter) FetchMarkers(context.Context, markers.Request) (markers.Result, error) {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) SubmitMarker(context.Context, markers.SubmissionRequest) (markers.SubmissionResult, error) {
|
||||
return markers.SubmissionResult{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) FetchUserStats(context.Context) (markers.UserStats, error) {
|
||||
return markers.UserStats{Total: 10, Accepted: 7, Pending: 2, Rejected: 1, AcceptanceRate: 0.7, CurrentStreak: 3, BestStreak: 5}, nil
|
||||
}
|
||||
|
||||
func TestValidateMarkerProviderUsesSnakeCaseStats(t *testing.T) {
|
||||
reg := markers.NewRegistry(nil)
|
||||
if err := reg.Register(fakeMarkerStatsSubmitter{}); err != nil {
|
||||
t.Fatalf("register provider: %v", err)
|
||||
}
|
||||
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/introdb/validate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("provider", "introdb")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleValidateProvider(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"acceptance_rate":0.7`) || strings.Contains(body, "AcceptanceRate") {
|
||||
t.Fatalf("unexpected stats response shape: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerProviderResponseIncludesPluginMetadata(t *testing.T) {
|
||||
resp := toProviderConfigResponse(
|
||||
markers.ProviderConfig{Provider: "plugin:4:markers", FetchEnabled: true, FetchPriority: 25},
|
||||
true,
|
||||
markers.ProviderDescriptor{
|
||||
DisplayName: "Plugin Markers",
|
||||
SourceType: markers.ProviderSourcePlugin,
|
||||
PluginID: "silo.plugin.markers",
|
||||
PluginInstallationID: 4,
|
||||
CapabilityID: "markers",
|
||||
},
|
||||
)
|
||||
if resp.DisplayName != "Plugin Markers" || resp.SourceType != markers.ProviderSourcePlugin {
|
||||
t.Fatalf("plugin metadata fields missing: %+v", resp)
|
||||
}
|
||||
if resp.PluginID != "silo.plugin.markers" ||
|
||||
resp.PluginInstallationID != 4 ||
|
||||
resp.CapabilityID != "markers" ||
|
||||
!resp.IsSubmitter {
|
||||
t.Fatalf("plugin identity fields missing: %+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -391,13 +391,17 @@ func (h *EventsHandler) writeSnapshotFrame(
|
||||
) error {
|
||||
data, err := h.snapshotForChannel(r, claims, channel)
|
||||
if err != nil {
|
||||
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
|
||||
_ = writeWebSocketControl(
|
||||
conn,
|
||||
websocket.CloseMessage,
|
||||
websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "snapshot failed"),
|
||||
slog.Error(
|
||||
"events: failed to build initial snapshot",
|
||||
"channel",
|
||||
channel,
|
||||
"user_id",
|
||||
claims.UserID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
return err
|
||||
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
|
||||
return nil
|
||||
}
|
||||
return writeWebSocketJSON(conn, evt.EventsSnapshotMessage{
|
||||
Type: "snapshot",
|
||||
|
||||
@@ -150,7 +150,7 @@ func (h *ItemsHandler) maybeRequestStaleEpisodeMetadataRefresh(ctx context.Conte
|
||||
if h == nil || h.metadataRefreshRequester == nil || episode == nil {
|
||||
return
|
||||
}
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, time.Now()) {
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, time.Now()) {
|
||||
h.requestStaleMetadataRefresh(ctx, "episode", episode.ContentID)
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func (h *ItemsHandler) maybeRequestStaleSeasonMetadataRefresh(ctx context.Contex
|
||||
}
|
||||
now := time.Now()
|
||||
for _, episode := range episodes {
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
h.requestStaleMetadataRefresh(ctx, "season", seasonID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,849 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimw "github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
// MarkerFileResolver loads a media file by id for the manual-marker API.
|
||||
type MarkerFileResolver interface {
|
||||
GetByID(ctx context.Context, id int) (*models.MediaFile, error)
|
||||
GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error)
|
||||
GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
// ManualMarkerWriter persists/clears manual markers.
|
||||
type ManualMarkerWriter interface {
|
||||
UpsertMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate) (bool, error)
|
||||
ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error)
|
||||
UpsertAndClearMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate, clearSegments []string) (bool, error)
|
||||
}
|
||||
|
||||
// MarkerContributor submits a file's eligible markers to enabled providers.
|
||||
type MarkerContributor interface {
|
||||
ContributeFile(ctx context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error)
|
||||
}
|
||||
|
||||
// MarkerContributionLister reads contribution history for a file.
|
||||
type MarkerContributionLister interface {
|
||||
ListByFile(ctx context.Context, fileID int) ([]markers.ContributionRow, error)
|
||||
}
|
||||
|
||||
type MarkerAuditLister interface {
|
||||
ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
ListAllMarkerEditAudit(ctx context.Context, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
}
|
||||
|
||||
type MarkerPermissionUserLoader interface {
|
||||
GetByID(ctx context.Context, id int) (*models.User, error)
|
||||
}
|
||||
|
||||
// MarkersHandler serves the manual-marker + contribution API. The marker
|
||||
// read/write/clear routes are mounted for any authenticated viewer (users fix
|
||||
// and create markers from the player); the contribution + history routes stay
|
||||
// admin-only. A successful manual write fires a background contribution run
|
||||
// (see maybeContribute) so corrected markers reach enabled providers.
|
||||
type MarkersHandler struct {
|
||||
Files MarkerFileResolver
|
||||
Writer ManualMarkerWriter
|
||||
Contributor MarkerContributor
|
||||
Contributions MarkerContributionLister
|
||||
AuditHistory MarkerAuditLister
|
||||
Notifier PlaybackMarkerUpdateNotifier
|
||||
Users MarkerPermissionUserLoader
|
||||
// Authorizer enforces per-item access on file lookups so a viewer can only
|
||||
// edit markers for content they can actually watch. When nil (tests) the
|
||||
// handler falls back to an unchecked lookup.
|
||||
Authorizer *MediaFileAuthorizer
|
||||
// BaseContext is the lifetime context for detached background work (set from
|
||||
// the app context) so contribution goroutines cancel on shutdown.
|
||||
BaseContext context.Context
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewMarkersHandler constructs the handler.
|
||||
func NewMarkersHandler(files MarkerFileResolver, writer ManualMarkerWriter, contributor MarkerContributor, contributions MarkerContributionLister, notifier PlaybackMarkerUpdateNotifier, logger *slog.Logger) *MarkersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &MarkersHandler{Files: files, Writer: writer, Contributor: contributor, Contributions: contributions, Notifier: notifier, logger: logger}
|
||||
}
|
||||
|
||||
var manualMarkerConfidence = 1.0
|
||||
|
||||
const manualMarkerAlgorithm = "manual:v1"
|
||||
|
||||
// markerContributeTimeout bounds the detached contribution run kicked off after
|
||||
// a manual save so a slow or rate-limited provider can't leak goroutines.
|
||||
const markerContributeTimeout = 30 * time.Second
|
||||
|
||||
var markerSegmentNames = []string{"intro", "credits", "recap", "preview"}
|
||||
|
||||
type segmentInput struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
}
|
||||
|
||||
type segmentMarker struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Source *string `json:"source"`
|
||||
Provider *string `json:"provider"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
Algorithm *string `json:"algorithm"`
|
||||
DetectedAt *time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
type fileMarkersResponse struct {
|
||||
FileID int `json:"file_id"`
|
||||
Intro segmentMarker `json:"intro"`
|
||||
Credits segmentMarker `json:"credits"`
|
||||
Recap segmentMarker `json:"recap"`
|
||||
Preview segmentMarker `json:"preview"`
|
||||
}
|
||||
|
||||
type contributionOutcomeResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Status string `json:"status"`
|
||||
SubmissionID string `json:"submission_id,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RetryAfterSeconds int `json:"retry_after_seconds,omitempty"`
|
||||
}
|
||||
|
||||
type contributionRowResponse struct {
|
||||
ID string `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Source string `json:"source"`
|
||||
SubmittedStartMS *int64 `json:"submitted_start_ms,omitempty"`
|
||||
SubmittedEndMS *int64 `json:"submitted_end_ms,omitempty"`
|
||||
VideoDurationMS *int64 `json:"video_duration_ms,omitempty"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
SubmissionID *string `json:"submission_id,omitempty"`
|
||||
Status string `json:"status"`
|
||||
HTTPStatus *int `json:"http_status,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
SubmittedAt *time.Time `json:"submitted_at,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type markerEditAuditResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
ItemID *string `json:"item_id,omitempty"`
|
||||
ItemType *string `json:"item_type,omitempty"`
|
||||
MediaTitle *string `json:"media_title,omitempty"`
|
||||
FilePath *string `json:"file_path,omitempty"`
|
||||
Segment string `json:"segment"`
|
||||
Action string `json:"action"`
|
||||
Before *segmentMarker `json:"before"`
|
||||
After *segmentMarker `json:"after"`
|
||||
UserID *int `json:"user_id,omitempty"`
|
||||
Username *string `json:"username,omitempty"`
|
||||
ImpersonatorUserID *int `json:"impersonator_user_id,omitempty"`
|
||||
ImpersonatorUsername *string `json:"impersonator_username,omitempty"`
|
||||
APIKeyID *int64 `json:"api_key_id,omitempty"`
|
||||
RequestID *string `json:"request_id,omitempty"`
|
||||
ClientIP *string `json:"client_ip,omitempty"`
|
||||
UserAgent *string `json:"user_agent,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func fileMarkers(file *models.MediaFile) fileMarkersResponse {
|
||||
return fileMarkersResponse{
|
||||
FileID: file.ID,
|
||||
Intro: segmentMarker{file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersProvider,
|
||||
file.IntroMarkersConfidence, file.IntroMarkersAlgorithm, file.IntroMarkersDetectedAt},
|
||||
Credits: segmentMarker{file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersProvider,
|
||||
file.CreditsMarkersConfidence, file.CreditsMarkersAlgorithm, file.CreditsMarkersDetectedAt},
|
||||
Recap: segmentMarker{file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersProvider,
|
||||
file.RecapMarkersConfidence, file.RecapMarkersAlgorithm, file.RecapMarkersDetectedAt},
|
||||
Preview: segmentMarker{file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersProvider,
|
||||
file.PreviewMarkersConfidence, file.PreviewMarkersAlgorithm, file.PreviewMarkersDetectedAt},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
fileID, err := strconv.Atoi(chi.URLParam(r, "fileId"))
|
||||
if err != nil || fileID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid file id is required")
|
||||
return nil, false
|
||||
}
|
||||
return h.authorizeFile(w, r, fileID)
|
||||
}
|
||||
|
||||
// authorizeFile loads a file, enforcing per-item access when an authorizer is
|
||||
// configured. The 404-on-denial mirrors the playback/subtitle paths so marker
|
||||
// editing can't be used to probe content outside a profile's library scope.
|
||||
func (h *MarkersHandler) authorizeFile(w http.ResponseWriter, r *http.Request, fileID int) (*models.MediaFile, bool) {
|
||||
if h.Authorizer != nil {
|
||||
file, err := h.Authorizer.Authorize(r, fileID)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
default:
|
||||
h.logger.Error("markers: authorize failed", "file_id", fileID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
file, err := h.Files.GetByID(r.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadItemPrimaryFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
// Re-validate the resolved primary file through the authorizer so item-id
|
||||
// edits are access-checked the same way file-id edits are.
|
||||
if h.Authorizer != nil {
|
||||
for _, file := range files {
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
authorized, err := h.Authorizer.Authorize(r, file.ID)
|
||||
if err == nil {
|
||||
return authorized, true
|
||||
}
|
||||
if errors.Is(err, catalog.ErrItemNotFound) || errors.Is(err, catalog.ErrEpisodeNotFound) {
|
||||
continue
|
||||
}
|
||||
h.logger.Error("markers: authorize item file failed", "item_id", itemID, "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
return nil, false
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
return file, true
|
||||
}
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) requireMarkerEdit(w http.ResponseWriter, r *http.Request) bool {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return false
|
||||
}
|
||||
if claims.Role == "admin" {
|
||||
return true
|
||||
}
|
||||
if h == nil || h.Users == nil {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
user, err := h.Users.GetByID(r.Context(), claims.UserID)
|
||||
if err != nil || user == nil || !auth.HasEffectivePermission(user, auth.PermissionMarkerEdit) {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) auditContext(r *http.Request) context.Context {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
return r.Context()
|
||||
}
|
||||
audit := scanner.MarkerAuditContext{
|
||||
UserID: &claims.UserID,
|
||||
ImpersonatorUserID: claims.ImpersonatorUserID,
|
||||
RequestID: chimw.GetReqID(r.Context()),
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
UserAgent: r.UserAgent(),
|
||||
}
|
||||
if claims.APIKeyID > 0 {
|
||||
apiKeyID := claims.APIKeyID
|
||||
audit.APIKeyID = &apiKeyID
|
||||
}
|
||||
return scanner.WithMarkerAuditContext(r.Context(), audit)
|
||||
}
|
||||
|
||||
// HandleGetFileMarkers returns the current markers + provenance for a file.
|
||||
func (h *MarkersHandler) HandleGetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleGetItemMarkers returns markers for the item's primary file.
|
||||
func (h *MarkersHandler) HandleGetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleSetFileMarkers upserts the manual marker layer. Each segment key may be
|
||||
// an object {start, end} to set, or null to clear; absent keys are unchanged.
|
||||
func (h *MarkersHandler) HandleSetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
// HandleSetItemMarkers upserts manual markers on the item's primary file.
|
||||
func (h *MarkersHandler) HandleSetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) setMarkersForFile(w http.ResponseWriter, r *http.Request, file *models.MediaFile) {
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
duration := float64(file.Duration)
|
||||
update := scanner.MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceManual,
|
||||
MarkersConfidence: &manualMarkerConfidence,
|
||||
MarkersAlgorithm: manualMarkerAlgorithm,
|
||||
}
|
||||
var clears []string
|
||||
var setSegs []string
|
||||
|
||||
for _, seg := range markerSegmentNames {
|
||||
val, present := raw[seg]
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
if isJSONNull(val) {
|
||||
clears = append(clears, seg)
|
||||
continue
|
||||
}
|
||||
var in segmentInput
|
||||
if err := json.Unmarshal(val, &in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid "+seg+" marker")
|
||||
return
|
||||
}
|
||||
start, end, err := normalizeManualSegment(seg, in, duration)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
applyManualSegment(&update, seg, start, end)
|
||||
setSegs = append(setSegs, seg)
|
||||
}
|
||||
|
||||
if _, err := h.Writer.UpsertAndClearMarkers(h.auditContext(r), file.ID, update, clears); err != nil {
|
||||
h.logger.Error("markers: save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save markers")
|
||||
return
|
||||
}
|
||||
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Markers saved but failed to reload")
|
||||
return
|
||||
}
|
||||
h.maybeContribute(refreshed, setSegs)
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleClearFileSegment clears a single segment.
|
||||
func (h *MarkersHandler) HandleClearFileSegment(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
segment := chi.URLParam(r, "segment")
|
||||
if !isMarkerSegment(segment) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown marker segment")
|
||||
return
|
||||
}
|
||||
if _, err := h.Writer.ClearMarkers(h.auditContext(r), file.ID, []string{segment}); err != nil {
|
||||
h.logger.Error("markers: clear segment failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to clear marker")
|
||||
return
|
||||
}
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after clear failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Marker cleared but failed to reload")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleContributeFile submits the file's eligible markers to enabled providers.
|
||||
func (h *MarkersHandler) HandleContributeFile(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributor == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Contribution is not configured")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Provider string `json:"provider"`
|
||||
Segments []string `json:"segments"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
var kinds []markers.MarkerKind
|
||||
for _, name := range body.Segments {
|
||||
kind, ok := markerKindForName(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown segment "+name)
|
||||
return
|
||||
}
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
outcomes, err := h.Contributor.ContributeFile(r.Context(), file, markers.ContributeOptions{Provider: body.Provider, Segments: kinds})
|
||||
if err != nil {
|
||||
h.logger.Error("markers: contribute failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Contribution failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"outcomes": contributionOutcomeResponses(outcomes)})
|
||||
}
|
||||
|
||||
// HandleListFileContributions returns the contribution history for a file.
|
||||
func (h *MarkersHandler) HandleListFileContributions(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributions == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": []contributionRowResponse{}})
|
||||
return
|
||||
}
|
||||
rows, err := h.Contributions.ListByFile(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list contributions failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load contributions")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": contributionRowResponses(rows)})
|
||||
}
|
||||
|
||||
// HandleListFileMarkerHistory returns recent semantic marker edit audit rows for one file.
|
||||
func (h *MarkersHandler) HandleListFileMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, []int{file.ID})
|
||||
}
|
||||
|
||||
// HandleListItemMarkerHistory returns recent marker edit audit rows for every file version on an item.
|
||||
func (h *MarkersHandler) HandleListItemMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker history is not configured")
|
||||
return
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fileIDs := make([]int, 0, len(files))
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
fileIDs = append(fileIDs, file.ID)
|
||||
}
|
||||
}
|
||||
if len(fileIDs) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, fileIDs)
|
||||
}
|
||||
|
||||
// HandleListMarkerHistory returns recent semantic marker edit audit rows across all files.
|
||||
func (h *MarkersHandler) HandleListMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListAllMarkerEditAudit(r.Context(), limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list all history failed", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) listMarkerHistory(w http.ResponseWriter, r *http.Request, fileIDs []int) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListMarkerEditAudit(r.Context(), fileIDs, limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list history failed", "file_ids", fileIDs, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func markerHistoryLimit(w http.ResponseWriter, r *http.Request) (int, bool) {
|
||||
limit := 25
|
||||
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid limit is required")
|
||||
return 0, false
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
return limit, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) reloadAndNotify(ctx context.Context, fileID int) (*models.MediaFile, error) {
|
||||
refreshed, err := h.Files.GetByID(ctx, fileID)
|
||||
if err != nil || refreshed == nil {
|
||||
if err == nil {
|
||||
err = scanner.ErrFileNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if h.Notifier != nil {
|
||||
h.Notifier.MarkersUpdated(ctx, refreshed)
|
||||
}
|
||||
return refreshed, nil
|
||||
}
|
||||
|
||||
// maybeContribute fires a best-effort, detached contribution run for the
|
||||
// segments just set by a manual save. ContributeFile self-gates on each
|
||||
// provider's contribute_enabled flag, only submits scanner/manual-sourced
|
||||
// segments, and is idempotent — so this is a no-op when contribution is off or
|
||||
// the marker was already submitted. It runs in the background so the save
|
||||
// responds immediately and is never blocked by provider rate limits.
|
||||
func (h *MarkersHandler) maybeContribute(file *models.MediaFile, segments []string) {
|
||||
if h == nil || h.Contributor == nil || file == nil || len(segments) == 0 {
|
||||
return
|
||||
}
|
||||
kinds := make([]markers.MarkerKind, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
if kind, ok := markerKindForName(seg); ok {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
}
|
||||
if len(kinds) == 0 {
|
||||
return
|
||||
}
|
||||
base := h.BaseContext
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(base, markerContributeTimeout)
|
||||
defer cancel()
|
||||
if _, err := h.Contributor.ContributeFile(ctx, file, markers.ContributeOptions{Segments: kinds}); err != nil {
|
||||
h.logger.Warn("markers: background contribution failed", "file_id", file.ID, "error", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func isJSONNull(raw json.RawMessage) bool {
|
||||
return string(bytes.TrimSpace(raw)) == "null"
|
||||
}
|
||||
|
||||
func isMarkerSegment(seg string) bool {
|
||||
_, ok := markerKindForName(seg)
|
||||
return ok
|
||||
}
|
||||
|
||||
func markerKindForName(name string) (markers.MarkerKind, bool) {
|
||||
switch name {
|
||||
case "intro":
|
||||
return markers.MarkerKindIntro, true
|
||||
case "credits":
|
||||
return markers.MarkerKindCredits, true
|
||||
case "recap":
|
||||
return markers.MarkerKindRecap, true
|
||||
case "preview":
|
||||
return markers.MarkerKindPreview, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func markerNameForKind(kind markers.MarkerKind) string {
|
||||
switch kind {
|
||||
case markers.MarkerKindIntro:
|
||||
return "intro"
|
||||
case markers.MarkerKindCredits:
|
||||
return "credits"
|
||||
case markers.MarkerKindRecap:
|
||||
return "recap"
|
||||
case markers.MarkerKindPreview:
|
||||
return "preview"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func contributionOutcomeResponses(outcomes []markers.ContributionOutcome) []contributionOutcomeResponse {
|
||||
resp := make([]contributionOutcomeResponse, 0, len(outcomes))
|
||||
for _, o := range outcomes {
|
||||
item := contributionOutcomeResponse{
|
||||
Provider: o.Provider,
|
||||
Segment: markerNameForKind(o.Segment),
|
||||
Status: o.Status,
|
||||
SubmissionID: o.SubmissionID,
|
||||
Reason: o.Reason,
|
||||
}
|
||||
if o.RetryAfter > 0 {
|
||||
item.RetryAfterSeconds = int(o.RetryAfter.Seconds())
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func contributionRowResponses(rows []markers.ContributionRow) []contributionRowResponse {
|
||||
resp := make([]contributionRowResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
item := contributionRowResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
Provider: row.Provider,
|
||||
Segment: row.SegmentKind,
|
||||
Source: row.Source,
|
||||
SubmittedStartMS: row.SubmittedStartMs,
|
||||
SubmittedEndMS: row.SubmittedEndMs,
|
||||
VideoDurationMS: row.VideoDurationMs,
|
||||
ContentHash: row.ContentHash,
|
||||
SubmissionID: row.SubmissionID,
|
||||
Status: row.Status,
|
||||
HTTPStatus: row.HTTPStatus,
|
||||
Error: row.Error,
|
||||
}
|
||||
if !row.SubmittedAt.IsZero() {
|
||||
submittedAt := row.SubmittedAt
|
||||
item.SubmittedAt = &submittedAt
|
||||
}
|
||||
if !row.UpdatedAt.IsZero() {
|
||||
updatedAt := row.UpdatedAt
|
||||
item.UpdatedAt = &updatedAt
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerEditAuditResponses(rows []scanner.MarkerEditAuditRow) []markerEditAuditResponse {
|
||||
resp := make([]markerEditAuditResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
resp = append(resp, markerEditAuditResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
ItemID: row.ItemID,
|
||||
ItemType: row.ItemType,
|
||||
MediaTitle: row.MediaTitle,
|
||||
FilePath: row.FilePath,
|
||||
Segment: row.SegmentKind,
|
||||
Action: row.Action,
|
||||
Before: markerAuditSegmentResponse(row.Before),
|
||||
After: markerAuditSegmentResponse(row.After),
|
||||
UserID: row.UserID,
|
||||
Username: row.Username,
|
||||
ImpersonatorUserID: row.ImpersonatorUserID,
|
||||
ImpersonatorUsername: row.ImpersonatorUsername,
|
||||
APIKeyID: row.APIKeyID,
|
||||
RequestID: row.RequestID,
|
||||
ClientIP: row.ClientIP,
|
||||
UserAgent: row.UserAgent,
|
||||
CreatedAt: row.CreatedAt,
|
||||
})
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerAuditSegmentResponse(segment *scanner.MarkerAuditSegment) *segmentMarker {
|
||||
if segment == nil {
|
||||
return nil
|
||||
}
|
||||
return &segmentMarker{
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Source: segment.Source,
|
||||
Provider: segment.Provider,
|
||||
Confidence: segment.Confidence,
|
||||
Algorithm: segment.Algorithm,
|
||||
DetectedAt: segment.DetectedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeManualSegment applies the start/end defaults and validation, mirroring
|
||||
// the contribution rules: intro/recap may omit start (=0); credits/preview may
|
||||
// omit end (=duration); end must exceed start and stay within the file.
|
||||
func normalizeManualSegment(seg string, in segmentInput, duration float64) (start, end float64, err error) {
|
||||
switch seg {
|
||||
case "intro", "recap":
|
||||
if in.Start != nil {
|
||||
start = *in.Start
|
||||
}
|
||||
if in.End == nil {
|
||||
return 0, 0, errSegment(seg, "end is required")
|
||||
}
|
||||
end = *in.End
|
||||
default: // credits, preview
|
||||
if in.Start == nil {
|
||||
return 0, 0, errSegment(seg, "start is required")
|
||||
}
|
||||
start = *in.Start
|
||||
if in.End != nil {
|
||||
end = *in.End
|
||||
} else if duration > 0 {
|
||||
end = duration
|
||||
} else {
|
||||
return 0, 0, errSegment(seg, "end is required when duration is unknown")
|
||||
}
|
||||
}
|
||||
if start < 0 || end <= start {
|
||||
return 0, 0, errSegment(seg, "end must be greater than start")
|
||||
}
|
||||
if duration > 0 && end > duration+1 {
|
||||
return 0, 0, errSegment(seg, "end exceeds the file duration")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
func applyManualSegment(update *scanner.MarkerUpdate, seg string, start, end float64) {
|
||||
s, e := start, end
|
||||
switch seg {
|
||||
case "intro":
|
||||
update.IntroStart, update.IntroEnd = &s, &e
|
||||
case "credits":
|
||||
update.CreditsStart, update.CreditsEnd = &s, &e
|
||||
case "recap":
|
||||
update.RecapStart, update.RecapEnd = &s, &e
|
||||
case "preview":
|
||||
update.PreviewStart, update.PreviewEnd = &s, &e
|
||||
}
|
||||
}
|
||||
|
||||
func errSegment(seg, msg string) error {
|
||||
return &markerValidationError{seg: seg, msg: msg}
|
||||
}
|
||||
|
||||
type markerValidationError struct {
|
||||
seg string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *markerValidationError) Error() string { return e.seg + " marker: " + e.msg }
|
||||
@@ -0,0 +1,524 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
type fakeMarkerFiles struct {
|
||||
file *models.MediaFile
|
||||
byID map[int]*models.MediaFile
|
||||
contentFiles []*models.MediaFile
|
||||
episodeFiles []*models.MediaFile
|
||||
}
|
||||
|
||||
func (f fakeMarkerFiles) GetByID(_ context.Context, id int) (*models.MediaFile, error) {
|
||||
if f.byID != nil {
|
||||
return f.byID[id], nil
|
||||
}
|
||||
return f.file, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByContentID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.contentFiles, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByEpisodeID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.episodeFiles, nil
|
||||
}
|
||||
|
||||
type fakeMarkerWriter struct {
|
||||
upserts []scanner.MarkerUpdate
|
||||
clears [][]string
|
||||
audits []scanner.MarkerAuditContext
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) captureAudit(ctx context.Context) {
|
||||
if audit, ok := scanner.MarkerAuditContextFromContext(ctx); ok {
|
||||
f.audits = append(f.audits, audit)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) UpsertMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.upserts = append(f.upserts, u)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) ClearMarkers(ctx context.Context, _ int, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.clears = append(f.clears, segs)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) UpsertAndClearMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
if u.HasAnySegment() {
|
||||
f.upserts = append(f.upserts, u)
|
||||
}
|
||||
if len(segs) > 0 {
|
||||
f.clears = append(f.clears, segs)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func markerPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/files/5", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func markerItemPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/items/episode-1", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func withMarkerAdminClaims(req *http.Request) *http.Request {
|
||||
return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{
|
||||
UserID: 1,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
SessionID: "session-1",
|
||||
}))
|
||||
}
|
||||
|
||||
func newMarkersHandler(writer ManualMarkerWriter) *MarkersHandler {
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
return NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
}
|
||||
|
||||
func TestSetFileMarkersWritesManual(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
u := writer.upserts[0]
|
||||
if u.MarkersSource != models.MarkerSourceManual {
|
||||
t.Errorf("source = %q, want manual", u.MarkersSource)
|
||||
}
|
||||
if u.IntroStart == nil || *u.IntroStart != 0 || u.IntroEnd == nil || *u.IntroEnd != 60 {
|
||||
t.Errorf("intro = %v..%v, want 0..60", u.IntroStart, u.IntroEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetItemMarkersWritesPrimaryEpisodeFile(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
file := &models.MediaFile{ID: 8, Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{8: file},
|
||||
episodeFiles: []*models.MediaFile{file},
|
||||
}
|
||||
h := NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetItemMarkers(rec, markerItemPutRequest(`{"recap":{"start":0,"end":45}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].RecapEnd == nil || *writer.upserts[0].RecapEnd != 45 {
|
||||
t.Errorf("recap end = %v, want 45", writer.upserts[0].RecapEnd)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerItemAccess map[string]error
|
||||
|
||||
func (f fakeMarkerItemAccess) EnsureAccessible(_ context.Context, contentID string, _ catalog.AccessFilter) error {
|
||||
return f[contentID]
|
||||
}
|
||||
|
||||
type fakeMarkerEpisodeLookup map[string]*models.Episode
|
||||
|
||||
func (f fakeMarkerEpisodeLookup) GetByID(_ context.Context, contentID string) (*models.Episode, error) {
|
||||
return f[contentID], nil
|
||||
}
|
||||
|
||||
type fakeMarkerUsers map[int]*models.User
|
||||
|
||||
func (f fakeMarkerUsers) GetByID(_ context.Context, id int) (*models.User, error) {
|
||||
return f[id], nil
|
||||
}
|
||||
|
||||
func TestGetItemMarkersUsesFirstAuthorizedFile(t *testing.T) {
|
||||
denied := &models.MediaFile{ID: 8, EpisodeID: "episode-denied", Duration: 1800}
|
||||
allowed := &models.MediaFile{ID: 9, EpisodeID: "episode-allowed", Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{
|
||||
8: denied,
|
||||
9: allowed,
|
||||
},
|
||||
episodeFiles: []*models.MediaFile{denied, allowed},
|
||||
}
|
||||
h := NewMarkersHandler(files, nil, nil, nil, nil, nil)
|
||||
h.Authorizer = &MediaFileAuthorizer{
|
||||
FileResolver: files,
|
||||
ItemAccess: fakeMarkerItemAccess{
|
||||
"series-denied": catalog.ErrItemNotFound,
|
||||
},
|
||||
EpisodeLookup: fakeMarkerEpisodeLookup{
|
||||
"episode-denied": {ContentID: "episode-denied", SeriesID: "series-denied"},
|
||||
"episode-allowed": {ContentID: "episode-allowed", SeriesID: "series-allowed"},
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/items/episode-1", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetItemMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body fileMarkersResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if body.FileID != allowed.ID {
|
||||
t.Fatalf("file_id = %d, want first authorized file %d", body.FileID, allowed.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFileMarkersDoesNotRequireMarkerEditPermission(t *testing.T) {
|
||||
h := newMarkersHandler(nil)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/files/5", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsUserWithoutMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("permission failure wrote markers: %+v", writer.upserts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersAllowsUserWithMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: []string{"marker_edit"}},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected marker write, got %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersPassesAuditContextToWriter(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
apiKeyID := int64(99)
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req.Header.Set("User-Agent", "marker-test-agent")
|
||||
ctx := clientip.SetContext(req.Context(), "203.0.113.10")
|
||||
ctx = apimw.SetClaims(ctx, &auth.Claims{
|
||||
UserID: 7,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAPIKey,
|
||||
APIKeyID: apiKeyID,
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.audits) != 1 {
|
||||
t.Fatalf("audit contexts = %d, want 1", len(writer.audits))
|
||||
}
|
||||
audit := writer.audits[0]
|
||||
if audit.UserID == nil || *audit.UserID != 7 {
|
||||
t.Fatalf("audit user id = %v, want 7", audit.UserID)
|
||||
}
|
||||
if audit.APIKeyID == nil || *audit.APIKeyID != apiKeyID {
|
||||
t.Fatalf("audit api key id = %v, want %d", audit.APIKeyID, apiKeyID)
|
||||
}
|
||||
if audit.ClientIP != "203.0.113.10" || audit.UserAgent != "marker-test-agent" {
|
||||
t.Fatalf("audit request metadata = ip %q ua %q", audit.ClientIP, audit.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearsOnNull(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("expected no upsert for a null clear, got %d", len(writer.upserts))
|
||||
}
|
||||
if len(writer.clears) != 1 || len(writer.clears[0]) != 1 || writer.clears[0][0] != "credits" {
|
||||
t.Errorf("clears = %v, want [[credits]]", writer.clears)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsInvalidRange(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":60,"end":10}}`))
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("invalid range must not write, got %d upserts", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearFileSegmentRejectsUnknown(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/markers/files/5/bogus", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
rctx.URLParams.Add("segment", "bogus")
|
||||
req = withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleClearFileSegment(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerContributor struct{ outcomes []markers.ContributionOutcome }
|
||||
|
||||
func (f fakeMarkerContributor) ContributeFile(context.Context, *models.MediaFile, markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
return f.outcomes, nil
|
||||
}
|
||||
|
||||
type captureMarkerContributor struct {
|
||||
opts markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (c *captureMarkerContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
c.opts = opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// signalContributor reports every ContributeFile call on a channel so a test
|
||||
// can wait for the detached background contribution kicked off by a save.
|
||||
type signalContributor struct {
|
||||
called chan markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (s signalContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
s.called <- opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestSetFileMarkersTriggersBackgroundContribution(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case opts := <-called:
|
||||
if len(opts.Segments) != 1 || opts.Segments[0] != markers.MarkerKindIntro {
|
||||
t.Fatalf("contribute segments = %v, want [intro]", opts.Segments)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("expected a background contribution for the saved intro segment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearOnlyDoesNotContribute(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-called:
|
||||
t.Fatal("a clear-only save must not trigger contribution")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
type fakeContributionLister struct{ rows []markers.ContributionRow }
|
||||
|
||||
func (f fakeContributionLister) ListByFile(context.Context, int) ([]markers.ContributionRow, error) {
|
||||
return f.rows, nil
|
||||
}
|
||||
|
||||
func TestContributeFileUsesSnakeCaseResponse(t *testing.T) {
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
fakeMarkerContributor{outcomes: []markers.ContributionOutcome{{
|
||||
Provider: "introdb", Segment: markers.MarkerKindCredits, Status: markers.OutcomeStatusRateLimited,
|
||||
Reason: "usage limited", RetryAfter: 30 * time.Second,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Outcomes []map[string]any `json:"outcomes"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(body.Outcomes) != 1 || body.Outcomes[0]["segment"] != "credits" {
|
||||
t.Fatalf("outcomes = %+v, want credits segment name", body.Outcomes)
|
||||
}
|
||||
if _, ok := body.Outcomes[0]["Segment"]; ok {
|
||||
t.Fatalf("response leaked internal Segment field: %s", rec.Body.String())
|
||||
}
|
||||
if body.Outcomes[0]["retry_after_seconds"] != float64(30) {
|
||||
t.Fatalf("retry_after_seconds = %v, want 30", body.Outcomes[0]["retry_after_seconds"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeFileDecodesUnknownLengthBody(t *testing.T) {
|
||||
contributor := &captureMarkerContributor{}
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
contributor,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{"provider":"introdb","segments":["credits"]}`))
|
||||
req.ContentLength = -1
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if contributor.opts.Provider != "introdb" {
|
||||
t.Fatalf("provider = %q, want introdb", contributor.opts.Provider)
|
||||
}
|
||||
if len(contributor.opts.Segments) != 1 || contributor.opts.Segments[0] != markers.MarkerKindCredits {
|
||||
t.Fatalf("segments = %+v, want [credits]", contributor.opts.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFileContributionsUsesSnakeCaseResponse(t *testing.T) {
|
||||
now := time.Unix(100, 0).UTC()
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
nil,
|
||||
fakeContributionLister{rows: []markers.ContributionRow{{
|
||||
ID: "row1", MediaFileID: 5, Provider: "introdb", SegmentKind: "intro",
|
||||
Source: "manual", ContentHash: "hash", Status: "pending", SubmittedAt: now, UpdatedAt: now,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/files/5/contributions", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListFileContributions(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"media_file_id":5`) || strings.Contains(body, "MediaFileID") {
|
||||
t.Fatalf("unexpected contribution response shape: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -2257,6 +2257,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2270,6 +2271,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2284,6 +2286,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2295,7 +2298,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
// active encode range), either restart at the exact manifest-derived
|
||||
// timeline position or return 404 for copy-mode segments outside the
|
||||
// current manifest window.
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
|
||||
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
|
||||
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
|
||||
slog.Error("resolve transcode seek target", "error", seekErr, "segment", segmentName, "session", sessionID, "playback_session_id", sessionID)
|
||||
@@ -2310,6 +2313,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"seek_seconds", seekSeconds,
|
||||
"session", sessionID,
|
||||
|
||||
@@ -272,7 +272,7 @@ func (h *PlaybackHandler) fetchOnlineMarkersForPlayback(ctx context.Context, fil
|
||||
EpisodeNumber: ids.EpisodeNumber,
|
||||
Duration: time.Duration(file.Duration) * time.Second,
|
||||
}
|
||||
result, ok, err := h.MarkerRegistry.FetchFirstHit(ctx, req)
|
||||
result, ok, err := h.MarkerRegistry.FetchMerged(ctx, req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -373,17 +373,31 @@ func hasLocalDetectionMarkers(file *models.MediaFile) bool {
|
||||
// each other.
|
||||
func markerUpdateFromPayload(p markers.MarkerUpdatePayload) scanner.MarkerUpdate {
|
||||
return scanner.MarkerUpdate{
|
||||
IntroStart: p.IntroStart,
|
||||
IntroEnd: p.IntroEnd,
|
||||
CreditsStart: p.CreditsStart,
|
||||
CreditsEnd: p.CreditsEnd,
|
||||
RecapStart: p.RecapStart,
|
||||
RecapEnd: p.RecapEnd,
|
||||
PreviewStart: p.PreviewStart,
|
||||
PreviewEnd: p.PreviewEnd,
|
||||
MarkersSource: p.Source,
|
||||
MarkersProvider: p.Provider,
|
||||
MarkersConfidence: p.Confidence,
|
||||
MarkersAlgorithm: p.Algorithm,
|
||||
IntroStart: p.Intro.Start,
|
||||
IntroEnd: p.Intro.End,
|
||||
CreditsStart: p.Credits.Start,
|
||||
CreditsEnd: p.Credits.End,
|
||||
RecapStart: p.Recap.Start,
|
||||
RecapEnd: p.Recap.End,
|
||||
PreviewStart: p.Preview.Start,
|
||||
PreviewEnd: p.Preview.End,
|
||||
MarkersSource: p.SummarySource(),
|
||||
MarkersConfidence: p.SummaryConfidence(),
|
||||
IntroProvenance: segmentProvenance(p.Intro),
|
||||
CreditsProvenance: segmentProvenance(p.Credits),
|
||||
RecapProvenance: segmentProvenance(p.Recap),
|
||||
PreviewProvenance: segmentProvenance(p.Preview),
|
||||
}
|
||||
}
|
||||
|
||||
func segmentProvenance(s markers.SegmentPayload) *scanner.SegmentProvenance {
|
||||
if !s.Present() {
|
||||
return nil
|
||||
}
|
||||
return &scanner.SegmentProvenance{
|
||||
Source: s.Source,
|
||||
Provider: s.Provider,
|
||||
Confidence: s.Confidence,
|
||||
Algorithm: s.Algorithm,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,6 +123,9 @@ type Dependencies struct {
|
||||
IntroAnalyzer *intromarkers.Analyzer
|
||||
MarkerRegistry *markers.Registry
|
||||
MarkerResolver markers.ExternalIDResolver
|
||||
MarkerProviderConfig *markers.ProviderConfigStore
|
||||
MarkerContributionStore *markers.ContributionStore
|
||||
MarkerContributionService *markers.ContributionService
|
||||
WatchProviderService handlers.WatchProviderService
|
||||
PluginService *plugins.Service
|
||||
PluginHTTPProxy *plugins.HTTPProxy
|
||||
@@ -769,6 +772,42 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
}
|
||||
|
||||
var markersHandler *handlers.MarkersHandler
|
||||
if deps.FileRepo != nil {
|
||||
var notifier handlers.PlaybackMarkerUpdateNotifier
|
||||
if playbackHandler != nil {
|
||||
notifier = playbackHandler.MarkerUpdateNotifier
|
||||
}
|
||||
var contributor handlers.MarkerContributor
|
||||
if deps.MarkerContributionService != nil {
|
||||
contributor = deps.MarkerContributionService
|
||||
}
|
||||
var contributions handlers.MarkerContributionLister
|
||||
if deps.MarkerContributionStore != nil {
|
||||
contributions = deps.MarkerContributionStore
|
||||
}
|
||||
markersHandler = handlers.NewMarkersHandler(
|
||||
deps.FileRepo, deps.FileRepo, contributor, contributions, notifier, slog.Default(),
|
||||
)
|
||||
markersHandler.BaseContext = deps.AppContext
|
||||
markersHandler.AuditHistory = deps.FileRepo
|
||||
markersHandler.Users = userRepo
|
||||
if itemRepo != nil {
|
||||
markersHandler.Authorizer = &handlers.MediaFileAuthorizer{
|
||||
FileResolver: deps.FileRepo,
|
||||
ItemAccess: itemRepo,
|
||||
EpisodeLookup: episodeRepo,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var adminMarkerProvidersHandler *handlers.AdminMarkerProvidersHandler
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
adminMarkerProvidersHandler = handlers.NewAdminMarkerProvidersHandler(
|
||||
deps.MarkerRegistry, deps.MarkerProviderConfig, deps.EventBus, slog.Default(),
|
||||
)
|
||||
}
|
||||
|
||||
// Admin subtitle config handler only needs the DB repo — no S3 required.
|
||||
var adminSubtitleHandler *handlers.AdminSubtitleHandler
|
||||
var subtitleManager *subtitles.Manager
|
||||
@@ -1299,6 +1338,21 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/events/ws", eventsHandler.HandleWebSocket)
|
||||
}
|
||||
|
||||
// Marker read/write/clear for any authenticated viewer: users
|
||||
// fix and create intro/recap/credits/preview markers from the
|
||||
// player. Writes are stamped source="manual" and contributed to
|
||||
// enabled providers in the background. Contribution + provider
|
||||
// config stay admin-only (see the /admin group below).
|
||||
if markersHandler != nil {
|
||||
r.Route("/markers", func(r chi.Router) {
|
||||
r.Get("/items/{id}", markersHandler.HandleGetItemMarkers)
|
||||
r.Put("/items/{id}", markersHandler.HandleSetItemMarkers)
|
||||
r.Get("/files/{fileId}", markersHandler.HandleGetFileMarkers)
|
||||
r.Put("/files/{fileId}", markersHandler.HandleSetFileMarkers)
|
||||
r.Delete("/files/{fileId}/{segment}", markersHandler.HandleClearFileSegment)
|
||||
})
|
||||
}
|
||||
|
||||
// Library management routes (admin-only).
|
||||
if libraryHandler != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
@@ -1843,6 +1897,21 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Post("/items/{id}/refresh-markers", adminIntroHandler.HandleRefreshEpisodeMarkers)
|
||||
r.Post("/items/{id}/redetect-intro", adminIntroHandler.HandleRedetectEpisodeIntro)
|
||||
}
|
||||
if markersHandler != nil {
|
||||
// Marker read/write/clear live on the authenticated
|
||||
// /markers routes; writes require marker_edit.
|
||||
// Contribution and audit history stay admin operations.
|
||||
r.Post("/files/{fileId}/contribute", markersHandler.HandleContributeFile)
|
||||
r.Get("/files/{fileId}/contributions", markersHandler.HandleListFileContributions)
|
||||
r.Get("/markers/history", markersHandler.HandleListMarkerHistory)
|
||||
r.Get("/markers/files/{fileId}/history", markersHandler.HandleListFileMarkerHistory)
|
||||
r.Get("/markers/items/{id}/history", markersHandler.HandleListItemMarkerHistory)
|
||||
}
|
||||
if adminMarkerProvidersHandler != nil {
|
||||
r.Get("/markers/providers", adminMarkerProvidersHandler.HandleListProviders)
|
||||
r.Put("/markers/providers/{provider}", adminMarkerProvidersHandler.HandleUpdateProvider)
|
||||
r.Post("/markers/providers/{provider}/validate", adminMarkerProvidersHandler.HandleValidateProvider)
|
||||
}
|
||||
if peopleHandler != nil {
|
||||
r.Post("/people/{id}/refresh", peopleHandler.HandleAdminRefreshPerson)
|
||||
r.Patch("/people/{id}", peopleHandler.HandleAdminUpdatePerson)
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// Package audiobooks owns silo's first-party audiobook + podcast feature,
|
||||
// absorbed from the historical silo-plugin-audiobooks. Sub-plan 1 lands
|
||||
// only the package scaffold and the kill-switch reader; ABS-compat REST,
|
||||
// Socket.io, scanner branches, podcast feed refresh, and the silo SPA
|
||||
// pages arrive in later sub-plans.
|
||||
//
|
||||
// See docs/superpowers/specs/2026-05-24-audiobooks-absorption-design.md
|
||||
// for the design and docs/superpowers/plans/2026-05-24-audiobooks-*.md
|
||||
// for the staged implementation plans.
|
||||
package audiobooks
|
||||
@@ -0,0 +1,43 @@
|
||||
package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// SettingsReader is the minimal slice of the server-settings store that
|
||||
// the audiobooks service needs. The production implementation is
|
||||
// internal/catalog.ServerSettingsRepo (or whatever silo names that helper at
|
||||
// wiring time); tests pass a fake.
|
||||
type SettingsReader interface {
|
||||
GetString(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
// Service is the audiobooks feature's top-level orchestrator. Sub-plan 1
|
||||
// exposes only Enabled(); subsequent sub-plans hang additional methods
|
||||
// off Service as new capabilities (scanner branches, ABS handlers, etc.)
|
||||
// come online.
|
||||
type Service struct {
|
||||
settings SettingsReader
|
||||
}
|
||||
|
||||
// New constructs a Service. The constructor takes the dependencies it
|
||||
// will actually use; current sub-plan needs only the settings reader.
|
||||
func New(settings SettingsReader) *Service {
|
||||
return &Service{settings: settings}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
return false, nil
|
||||
}
|
||||
value, err := s.settings.GetString(ctx, "audiobooks.enabled")
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("read audiobooks.enabled: %w", err)
|
||||
}
|
||||
return value == "true", nil
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type fakeSettingsReader struct {
|
||||
value string
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *fakeSettingsReader) GetString(_ context.Context, key string) (string, error) {
|
||||
if key != "audiobooks.enabled" {
|
||||
return "", errors.New("unexpected key: " + key)
|
||||
}
|
||||
return f.value, f.err
|
||||
}
|
||||
|
||||
func TestServiceEnabledReadsFlag(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
stored string
|
||||
want bool
|
||||
}{
|
||||
{"flag true", "true", true},
|
||||
{"flag false", "false", false},
|
||||
{"flag empty defaults false", "", false},
|
||||
{"flag garbage defaults false", "yes-please", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
tc := tc
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
svc := New(&fakeSettingsReader{value: tc.stored})
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got != tc.want {
|
||||
t.Fatalf("Enabled = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledPropagatesError(t *testing.T) {
|
||||
wantErr := errors.New("db down")
|
||||
svc := New(&fakeSettingsReader{err: wantErr})
|
||||
_, err := svc.Enabled(context.Background())
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("Enabled error = %v, want %v wrapped", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledNilReceiverReturnsFalse(t *testing.T) {
|
||||
var svc *Service
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("Enabled = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledNilSettingsReturnsFalse(t *testing.T) {
|
||||
svc := New(nil)
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("Enabled = true, want false")
|
||||
}
|
||||
}
|
||||
@@ -10,9 +10,13 @@ import (
|
||||
|
||||
type Permission string
|
||||
|
||||
const PermissionMetadataCuration Permission = "metadata_curation"
|
||||
const (
|
||||
PermissionMarkerEdit Permission = "marker_edit"
|
||||
PermissionMetadataCuration Permission = "metadata_curation"
|
||||
)
|
||||
|
||||
var assignablePermissions = map[Permission]struct{}{
|
||||
PermissionMarkerEdit: {},
|
||||
PermissionMetadataCuration: {},
|
||||
}
|
||||
|
||||
@@ -56,6 +60,10 @@ func NormalizePermissions(values []string) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func DefaultUserPermissions() []string {
|
||||
return []string{string(PermissionMarkerEdit)}
|
||||
}
|
||||
|
||||
func HasAssignedPermission(user *models.User, permission Permission) bool {
|
||||
if user == nil {
|
||||
return false
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
got, err := NormalizePermissions([]string{
|
||||
"marker_edit",
|
||||
" metadata_curation ",
|
||||
"metadata_curation",
|
||||
"",
|
||||
@@ -16,7 +17,7 @@ func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizePermissions returned error: %v", err)
|
||||
}
|
||||
want := []string{"metadata_curation"}
|
||||
want := []string{"marker_edit", "metadata_curation"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -28,11 +29,14 @@ func TestNormalizePermissions_RejectsUnknownPermission(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_AdminImpliesMetadataCuration(t *testing.T) {
|
||||
func TestHasEffectivePermission_AdminImpliesAssignablePermissions(t *testing.T) {
|
||||
user := &models.User{Role: "admin", Enabled: true}
|
||||
if !HasEffectivePermission(user, PermissionMetadataCuration) {
|
||||
t.Fatal("admin should have metadata curation")
|
||||
}
|
||||
if !HasEffectivePermission(user, PermissionMarkerEdit) {
|
||||
t.Fatal("admin should have marker edit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
@@ -45,3 +49,11 @@ func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
t.Fatal("assigned user should have metadata curation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultUserPermissionsIncludesMarkerEditOnly(t *testing.T) {
|
||||
got := DefaultUserPermissions()
|
||||
want := []string{"marker_edit"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("default permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,11 @@ func (r *UserRepository) Create(ctx context.Context, input models.CreateUserInpu
|
||||
localPasswordLoginEnabled = *input.LocalPasswordLoginEnabled
|
||||
}
|
||||
|
||||
permissions, err := NormalizePermissions(input.Permissions)
|
||||
permissions := append([]string(nil), input.Permissions...)
|
||||
if input.Permissions == nil && input.Role != "admin" {
|
||||
permissions = DefaultUserPermissions()
|
||||
}
|
||||
permissions, err = NormalizePermissions(permissions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Vendored
+12
-11
@@ -40,17 +40,18 @@ const (
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventMarkerProviderConfigChanged = "marker_provider_config_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -634,7 +634,7 @@ func (s *Service) exportFolders(ctx context.Context, libraryIDs []int) ([]Librar
|
||||
|
||||
func (s *Service) schemaVersion(ctx context.Context) (int, error) {
|
||||
var version int
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_versions").Scan(&version); err != nil {
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version_id), 0) FROM public.goose_db_version").Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("loading schema version: %w", err)
|
||||
}
|
||||
return version, nil
|
||||
|
||||
+210
-199
@@ -2,233 +2,244 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/pressly/goose/v3/lock"
|
||||
)
|
||||
|
||||
// MigrationFile represents a parsed migration file with its version number,
|
||||
// filename, and SQL content.
|
||||
type MigrationFile struct {
|
||||
Version int
|
||||
Filename string
|
||||
SQL string
|
||||
}
|
||||
const (
|
||||
schemaMigrationsLockID int64 = 8_034_219_741
|
||||
gooseVersionTable = "public.goose_db_version"
|
||||
)
|
||||
|
||||
const schemaMigrationsLockID int64 = 8_034_219_741
|
||||
|
||||
type migrationDB interface {
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error)
|
||||
}
|
||||
|
||||
// RunMigrations applies all pending .up.sql migrations from the given embed.FS.
|
||||
// It creates a schema_versions table to track which migrations have been applied,
|
||||
// parses migration filenames to extract version numbers, and applies each
|
||||
// pending migration within its own transaction.
|
||||
//
|
||||
// The dir parameter specifies the subdirectory within the fs.FS that contains
|
||||
// the migration files.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) (err error) {
|
||||
conn, err := pool.Acquire(ctx)
|
||||
// RunMigrations applies all pending Goose migrations from fsys/dir.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) error {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("acquiring migration connection: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
defer provider.Close()
|
||||
|
||||
if err := lockMigrations(ctx, conn); err != nil {
|
||||
return fmt.Errorf("acquiring migration lock: %w", err)
|
||||
if _, err := provider.Up(ctx); err != nil {
|
||||
return fmt.Errorf("running goose migrations: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if unlockErr := unlockMigrations(conn); unlockErr != nil {
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w; releasing migration lock: %v", err, unlockErr)
|
||||
return
|
||||
}
|
||||
err = fmt.Errorf("releasing migration lock: %w", unlockErr)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := ensureSchemaVersionsTable(ctx, conn); err != nil {
|
||||
return fmt.Errorf("ensuring schema_versions table: %w", err)
|
||||
}
|
||||
|
||||
migrations, err := loadMigrations(fsys, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading migrations: %w", err)
|
||||
}
|
||||
|
||||
applied, err := getAppliedVersions(ctx, conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting applied versions: %w", err)
|
||||
}
|
||||
|
||||
for _, m := range migrations {
|
||||
if applied[m.Version] {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := applyMigration(ctx, conn, m); err != nil {
|
||||
return fmt.Errorf("applying migration %s (version %d): %w", m.Filename, m.Version, err)
|
||||
}
|
||||
applied[m.Version] = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockMigrations(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_lock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
// MigrationStatus describes a migration source and whether Goose has applied it.
|
||||
type MigrationStatus struct {
|
||||
Version int64
|
||||
Source string
|
||||
State string
|
||||
AppliedAt time.Time
|
||||
}
|
||||
|
||||
func unlockMigrations(db migrationDB) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_unlock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
}
|
||||
|
||||
// ensureSchemaVersionsTable creates the schema_versions table if it does not
|
||||
// already exist.
|
||||
func ensureSchemaVersionsTable(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.schema_versions (
|
||||
version INT PRIMARY KEY,
|
||||
filename TEXT NOT NULL,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadMigrations reads all .up.sql files from the specified directory within
|
||||
// the filesystem and returns them sorted by version number.
|
||||
func loadMigrations(fsys fs.FS, dir string) ([]MigrationFile, error) {
|
||||
entries, err := fs.ReadDir(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading migrations directory %q: %w", dir, err)
|
||||
}
|
||||
|
||||
var migrations []MigrationFile
|
||||
seenVersions := make(map[int]string)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
name := entry.Name()
|
||||
if !strings.HasSuffix(name, ".up.sql") {
|
||||
continue
|
||||
}
|
||||
|
||||
version, err := parseVersion(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing version from %q: %w", name, err)
|
||||
}
|
||||
if existing, ok := seenVersions[version]; ok {
|
||||
return nil, fmt.Errorf("duplicate migration version %d: %s and %s", version, existing, name)
|
||||
}
|
||||
seenVersions[version] = name
|
||||
|
||||
content, err := fs.ReadFile(fsys, filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading migration file %q: %w", name, err)
|
||||
}
|
||||
|
||||
migrations = append(migrations, MigrationFile{
|
||||
Version: version,
|
||||
Filename: name,
|
||||
SQL: string(content),
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(migrations, func(i, j int) bool {
|
||||
return migrations[i].Version < migrations[j].Version
|
||||
})
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
// parseVersion extracts the numeric version prefix from a migration filename.
|
||||
// For example, "001_initial_schema.up.sql" returns 1.
|
||||
func parseVersion(filename string) (int, error) {
|
||||
// Split on the first underscore to get the version prefix.
|
||||
parts := strings.SplitN(filename, "_", 2)
|
||||
if len(parts) < 2 {
|
||||
return 0, fmt.Errorf("migration filename %q must be in format NNN_description.up.sql", filename)
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid version number in %q: %w", filename, err)
|
||||
}
|
||||
|
||||
if version <= 0 {
|
||||
return 0, fmt.Errorf("version number must be positive, got %d in %q", version, filename)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// getAppliedVersions returns a set of migration versions that have already
|
||||
// been applied.
|
||||
func getAppliedVersions(ctx context.Context, db migrationDB) (map[int]bool, error) {
|
||||
rows, err := db.Query(ctx, "SELECT version FROM public.schema_versions")
|
||||
// MigrationStatuses returns Goose status using the same legacy bootstrap and locking path as RunMigrations.
|
||||
func MigrationStatuses(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) ([]MigrationStatus, error) {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer provider.Close()
|
||||
|
||||
gooseStatuses, err := provider.Status(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading goose migration status: %w", err)
|
||||
}
|
||||
|
||||
statuses := make([]MigrationStatus, 0, len(gooseStatuses))
|
||||
for _, status := range gooseStatuses {
|
||||
var version int64
|
||||
var source string
|
||||
if status.Source != nil {
|
||||
version = status.Source.Version
|
||||
source = status.Source.Path
|
||||
}
|
||||
statuses = append(statuses, MigrationStatus{
|
||||
Version: version,
|
||||
Source: source,
|
||||
State: string(status.State),
|
||||
AppliedAt: status.AppliedAt,
|
||||
})
|
||||
}
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
func newMigrationProvider(pool *pgxpool.Pool, fsys fs.FS, dir string) (*goose.Provider, error) {
|
||||
migrationFS, err := migrationSubFS(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sqlDB := stdlib.OpenDBFromPool(pool)
|
||||
|
||||
locker, err := lock.NewPostgresSessionLocker(lock.WithLockID(schemaMigrationsLockID))
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration lock: %w", err)
|
||||
}
|
||||
|
||||
provider, err := goose.NewProvider(
|
||||
goose.DialectPostgres,
|
||||
sqlDB,
|
||||
migrationFS,
|
||||
goose.WithTableName(gooseVersionTable),
|
||||
goose.WithAllowOutofOrder(true),
|
||||
goose.WithSessionLocker(&legacyBootstrapLocker{delegate: locker}),
|
||||
)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration provider: %w", err)
|
||||
}
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
func migrationSubFS(fsys fs.FS, dir string) (fs.FS, error) {
|
||||
if dir == "" || dir == "." {
|
||||
return fsys, nil
|
||||
}
|
||||
|
||||
sub, err := fs.Sub(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening migration directory %q: %w", dir, err)
|
||||
}
|
||||
return sub, nil
|
||||
}
|
||||
|
||||
type legacyBootstrapLocker struct {
|
||||
delegate lock.SessionLocker
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionLock(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := l.delegate.SessionLock(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bootstrapLegacyGooseVersions(ctx, conn); err != nil {
|
||||
_ = l.delegate.SessionUnlock(context.WithoutCancel(ctx), conn)
|
||||
return fmt.Errorf("bootstrapping goose migration versions: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error {
|
||||
return l.delegate.SessionUnlock(ctx, conn)
|
||||
}
|
||||
|
||||
func bootstrapLegacyGooseVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := ensureGooseVersionTable(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureGooseZeroVersion(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasLegacyVersions, err := tableExists(ctx, conn, "public.schema_versions")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking legacy schema_versions table: %w", err)
|
||||
}
|
||||
if !hasLegacyVersions {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := copyLegacyVersions(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
return verifyLegacyVersions(ctx, conn)
|
||||
}
|
||||
|
||||
func ensureGooseVersionTable(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.goose_db_version (
|
||||
id integer PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
|
||||
version_id bigint NOT NULL,
|
||||
is_applied boolean NOT NULL,
|
||||
tstamp timestamp NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose version table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureGooseZeroVersion(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied)
|
||||
SELECT 0, true
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version
|
||||
WHERE version_id = 0 AND is_applied
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose zero version: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied, tstamp)
|
||||
SELECT sv.version, true, sv.applied_at::timestamp
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
rows, err := conn.QueryContext(ctx, `
|
||||
SELECT sv.version
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
LIMIT 10
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verifying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
applied := make(map[int]bool)
|
||||
var missing []int
|
||||
for rows.Next() {
|
||||
var version int
|
||||
if err := rows.Scan(&version); err != nil {
|
||||
return nil, err
|
||||
return fmt.Errorf("scanning missing legacy version: %w", err)
|
||||
}
|
||||
applied[version] = true
|
||||
missing = append(missing, version)
|
||||
}
|
||||
|
||||
return applied, rows.Err()
|
||||
}
|
||||
|
||||
// applyMigration runs a single migration within a transaction and records the
|
||||
// version in schema_versions.
|
||||
func applyMigration(ctx context.Context, db migrationDB, m MigrationFile) error {
|
||||
tx, err := db.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning transaction: %w", err)
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("reading missing legacy versions: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
// Rollback is a no-op if the tx has already been committed.
|
||||
_ = tx.Rollback(ctx)
|
||||
}()
|
||||
|
||||
if _, err := tx.Exec(ctx, m.SQL); err != nil {
|
||||
return fmt.Errorf("executing SQL: %w", err)
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("legacy schema_versions rows missing from goose_db_version: %v", missing)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
"INSERT INTO public.schema_versions (version, filename) VALUES ($1, $2)",
|
||||
m.Version, m.Filename,
|
||||
); err != nil {
|
||||
return fmt.Errorf("recording version: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("committing transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func tableExists(ctx context.Context, conn *sql.Conn, qualifiedName string) (bool, error) {
|
||||
var exists bool
|
||||
if err := conn.QueryRowContext(ctx, "SELECT to_regclass($1) IS NOT NULL", qualifiedName).Scan(&exists); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
@@ -12,11 +12,12 @@ import (
|
||||
)
|
||||
|
||||
type Analyzer struct {
|
||||
repo introRepository
|
||||
extractor fingerprintExtractor
|
||||
refiner boundaryRefiner
|
||||
config Config
|
||||
logger *slog.Logger
|
||||
repo introRepository
|
||||
extractor fingerprintExtractor
|
||||
refiner boundaryRefiner
|
||||
chromaprintRefiner chromaprintStartRefiner
|
||||
config Config
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
type introRepository interface {
|
||||
@@ -43,11 +44,12 @@ func NewAnalyzer(repo *Repository, config Config, logger *slog.Logger) *Analyzer
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Analyzer{
|
||||
repo: repo,
|
||||
extractor: NewChromaprintExtractor(config),
|
||||
refiner: NewSilenceBoundaryRefiner(config),
|
||||
config: config,
|
||||
logger: logger,
|
||||
repo: repo,
|
||||
extractor: NewChromaprintExtractor(config),
|
||||
refiner: NewSilenceBoundaryRefiner(config),
|
||||
chromaprintRefiner: NewDialogueBoundaryRefiner(config),
|
||||
config: config,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +146,9 @@ func (a *Analyzer) Run(ctx context.Context, progress ProgressFunc) (RunSummary,
|
||||
summary.FingerprintsComputed += groupSummary.FingerprintsComputed
|
||||
summary.FingerprintCacheHits += groupSummary.FingerprintCacheHits
|
||||
summary.ChromaprintMarkersWritten += groupSummary.ChromaprintMarkersWritten
|
||||
summary.DialogueRefinementsAttempted += groupSummary.DialogueRefinementsAttempted
|
||||
summary.DialogueRefinementsApplied += groupSummary.DialogueRefinementsApplied
|
||||
summary.DialogueRefinementErrors += groupSummary.DialogueRefinementErrors
|
||||
summary.GroupsNotFound += groupSummary.GroupsNotFound
|
||||
summary.GroupsSkipped += groupSummary.GroupsSkipped
|
||||
summary.Errors = append(summary.Errors, groupSummary.Errors...)
|
||||
@@ -249,6 +254,9 @@ func (a *Analyzer) AnalyzeEpisode(ctx context.Context, episodeID string) (RunSum
|
||||
summary.FingerprintsComputed += groupSummary.FingerprintsComputed
|
||||
summary.FingerprintCacheHits += groupSummary.FingerprintCacheHits
|
||||
summary.ChromaprintMarkersWritten += groupSummary.ChromaprintMarkersWritten
|
||||
summary.DialogueRefinementsAttempted += groupSummary.DialogueRefinementsAttempted
|
||||
summary.DialogueRefinementsApplied += groupSummary.DialogueRefinementsApplied
|
||||
summary.DialogueRefinementErrors += groupSummary.DialogueRefinementErrors
|
||||
summary.GroupsNotFound += groupSummary.GroupsNotFound
|
||||
summary.GroupsSkipped += groupSummary.GroupsSkipped
|
||||
summary.Errors = append(summary.Errors, groupSummary.Errors...)
|
||||
@@ -277,6 +285,8 @@ type chapterSourceMarker struct {
|
||||
segment Segment
|
||||
}
|
||||
|
||||
const episodeVersionCopyDurationToleranceSeconds = 3.0
|
||||
|
||||
func (a *Analyzer) processChapterCandidates(ctx context.Context, candidates []Candidate, opts chapterProcessingOptions) ([]Candidate, RunSummary) {
|
||||
summary := RunSummary{}
|
||||
remaining := make([]Candidate, 0, len(candidates))
|
||||
@@ -438,7 +448,7 @@ func compatibleEpisodeVersionDuration(source, target Candidate) bool {
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
return diff <= 1.0
|
||||
return diff <= episodeVersionCopyDurationToleranceSeconds
|
||||
}
|
||||
|
||||
func ownDetectionCandidates(candidates []Candidate) []Candidate {
|
||||
@@ -594,6 +604,7 @@ func (a *Analyzer) analyzeGroup(ctx context.Context, group candidateGroup, opts
|
||||
continue
|
||||
}
|
||||
candidate := byFileID[fileID]
|
||||
segment = a.refineChromaprintSegment(ctx, candidate, segment, &summary)
|
||||
applied, patchErr := a.repo.PatchIntroMarker(ctx, IntroMarkerPatch{
|
||||
FileID: fileID,
|
||||
Start: segment.Start,
|
||||
@@ -624,6 +635,24 @@ func (a *Analyzer) analyzeGroup(ctx context.Context, group candidateGroup, opts
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (a *Analyzer) refineChromaprintSegment(ctx context.Context, candidate Candidate, segment Segment, summary *RunSummary) Segment {
|
||||
if a.chromaprintRefiner == nil || !a.config.normalized().DialogueRefinementEnabled {
|
||||
return segment
|
||||
}
|
||||
summary.DialogueRefinementsAttempted++
|
||||
refined, ok, err := a.chromaprintRefiner.RefineChromaprintStart(ctx, candidate, segment)
|
||||
if err != nil {
|
||||
summary.DialogueRefinementErrors++
|
||||
a.logger.Warn("intro marker dialogue refinement failed", "file_id", candidate.FileID, "path", candidate.FilePath, "error", err)
|
||||
return segment
|
||||
}
|
||||
if ok {
|
||||
summary.DialogueRefinementsApplied++
|
||||
return refined
|
||||
}
|
||||
return segment
|
||||
}
|
||||
|
||||
func candidateFileIDs(candidates []Candidate) map[int]struct{} {
|
||||
fileIDs := make(map[int]struct{}, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
@@ -758,4 +787,7 @@ func mergeRunSummary(dst *RunSummary, src RunSummary) {
|
||||
dst.SilenceRefinementErrors += src.SilenceRefinementErrors
|
||||
dst.EpisodeVersionMarkersCopied += src.EpisodeVersionMarkersCopied
|
||||
dst.SilenceBackfillConsidered += src.SilenceBackfillConsidered
|
||||
dst.DialogueRefinementsAttempted += src.DialogueRefinementsAttempted
|
||||
dst.DialogueRefinementsApplied += src.DialogueRefinementsApplied
|
||||
dst.DialogueRefinementErrors += src.DialogueRefinementErrors
|
||||
}
|
||||
|
||||
@@ -102,6 +102,20 @@ func (f *fakeBoundaryRefiner) RefineChapterEnd(_ context.Context, candidate Cand
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
type fakeChromaprintStartRefiner struct {
|
||||
calls int
|
||||
segments map[int]Segment
|
||||
}
|
||||
|
||||
func (f *fakeChromaprintStartRefiner) RefineChromaprintStart(_ context.Context, candidate Candidate, segment Segment) (Segment, bool, error) {
|
||||
f.calls++
|
||||
refined, ok := f.segments[candidate.FileID]
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
func TestAnalyzeEpisodeNoCandidatesIsNoOp(t *testing.T) {
|
||||
repo := &fakeIntroRepository{episodeCandidates: map[string][]Candidate{}}
|
||||
extractor := &fakeFingerprintExtractor{}
|
||||
@@ -260,7 +274,7 @@ func TestAnalyzeEpisodeCopiesMarkerToCompatibleEpisodeVersion(t *testing.T) {
|
||||
{Index: 2, Title: "Part 1", StartSeconds: 120, EndSeconds: 900},
|
||||
},
|
||||
}
|
||||
target := Candidate{FileID: 11, EpisodeID: "ep1", DurationSeconds: 1200.5}
|
||||
target := Candidate{FileID: 11, EpisodeID: "ep1", DurationSeconds: 1202.5}
|
||||
repo := &fakeIntroRepository{episodeCandidates: map[string][]Candidate{"ep1": []Candidate{source, target}}}
|
||||
analyzer := &Analyzer{repo: repo, extractor: &fakeFingerprintExtractor{}, config: DefaultConfig("ffmpeg")}
|
||||
|
||||
@@ -420,6 +434,65 @@ func TestAnalyzeEpisodeChromaprintOnlyPatchesRequestedEpisode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeEpisodePersistsRefinedChromaprintSegment(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
target := Candidate{
|
||||
FileID: 1,
|
||||
EpisodeID: "ep1",
|
||||
SeasonID: "season1",
|
||||
MediaFolderID: 7,
|
||||
FileHash: "hash1",
|
||||
FileSize: 100,
|
||||
DurationSeconds: 1200,
|
||||
}
|
||||
sibling := Candidate{
|
||||
FileID: 2,
|
||||
EpisodeID: "ep2",
|
||||
SeasonID: "season1",
|
||||
MediaFolderID: 7,
|
||||
FileHash: "hash2",
|
||||
FileSize: 200,
|
||||
DurationSeconds: 1200,
|
||||
}
|
||||
groupCandidates := []Candidate{target, sibling}
|
||||
group := groupKey(target.MediaFolderID, target.SeasonID, target.AnalysisGroupKey())
|
||||
repo := &fakeIntroRepository{
|
||||
episodeCandidates: map[string][]Candidate{"ep1": []Candidate{target}},
|
||||
groupCandidates: map[string][]Candidate{group: groupCandidates},
|
||||
fingerprints: map[int]*Fingerprint{
|
||||
target.FileID: cachedFingerprint(target, cfg, sharedIntroPoints(1000)),
|
||||
sibling.FileID: cachedFingerprint(sibling, cfg, sharedIntroPoints(5000)),
|
||||
},
|
||||
}
|
||||
refiner := &fakeChromaprintStartRefiner{segments: map[int]Segment{
|
||||
target.FileID: {Start: 12.5, End: 36.5, Confidence: 0.85, Algorithm: ChromaprintDialogueAlgorithm},
|
||||
}}
|
||||
analyzer := &Analyzer{
|
||||
repo: repo,
|
||||
extractor: &fakeFingerprintExtractor{},
|
||||
chromaprintRefiner: refiner,
|
||||
config: cfg,
|
||||
}
|
||||
|
||||
summary, err := analyzer.AnalyzeEpisode(context.Background(), "ep1")
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeEpisode returned error: %v", err)
|
||||
}
|
||||
if refiner.calls != 1 {
|
||||
t.Fatalf("expected one refinement call for requested file, got %d", refiner.calls)
|
||||
}
|
||||
if summary.DialogueRefinementsAttempted != 1 || summary.DialogueRefinementsApplied != 1 {
|
||||
t.Fatalf("expected one applied dialogue refinement, got attempted=%d applied=%d",
|
||||
summary.DialogueRefinementsAttempted, summary.DialogueRefinementsApplied)
|
||||
}
|
||||
if len(repo.patches) != 1 {
|
||||
t.Fatalf("expected one patch, got %d", len(repo.patches))
|
||||
}
|
||||
if repo.patches[0].Start != 12.5 || repo.patches[0].Algorithm != ChromaprintDialogueAlgorithm {
|
||||
t.Fatalf("patch = %+v, want refined chromaprint marker", repo.patches[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBackfillsExistingChapterMarkerWithSilenceBudget(t *testing.T) {
|
||||
source := models.MarkerSourceScanner
|
||||
algorithm := ChapterAlgorithm
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
package intromarkers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/lang"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
)
|
||||
|
||||
type chromaprintStartRefiner interface {
|
||||
RefineChromaprintStart(ctx context.Context, candidate Candidate, segment Segment) (Segment, bool, error)
|
||||
}
|
||||
|
||||
type DialogueBoundaryRefiner struct {
|
||||
config Config
|
||||
readFile func(string) ([]byte, error)
|
||||
}
|
||||
|
||||
var htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
|
||||
|
||||
func NewDialogueBoundaryRefiner(config Config) *DialogueBoundaryRefiner {
|
||||
return &DialogueBoundaryRefiner{
|
||||
config: config.normalized(),
|
||||
readFile: os.ReadFile,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *DialogueBoundaryRefiner) RefineChromaprintStart(ctx context.Context, candidate Candidate, segment Segment) (Segment, bool, error) {
|
||||
cfg := r.config.normalized()
|
||||
if !cfg.DialogueRefinementEnabled || segment.End <= segment.Start || segment.Algorithm != ChromaprintAlgorithm {
|
||||
return segment, false, nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return segment, false, err
|
||||
}
|
||||
subtitle, ok := selectDialogueSubtitle(candidate)
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
readFile := r.readFile
|
||||
if readFile == nil {
|
||||
readFile = os.ReadFile
|
||||
}
|
||||
data, err := readFile(subtitle.Path)
|
||||
if err != nil {
|
||||
return segment, false, fmt.Errorf("read dialogue subtitle %q for file %d: %w", subtitle.Path, candidate.FileID, err)
|
||||
}
|
||||
cues, err := subtitles.ParseCues(data)
|
||||
if err != nil {
|
||||
return segment, false, fmt.Errorf("parse dialogue subtitle %q for file %d: %w", subtitle.Path, candidate.FileID, err)
|
||||
}
|
||||
refinedStart, ok := dialogueRefinedStart(cues, segment, cfg)
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
refined := segment
|
||||
refined.Start = refinedStart
|
||||
refined.Algorithm = ChromaprintDialogueAlgorithm
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
func selectDialogueSubtitle(candidate Candidate) (models.ExternalSubtitle, bool) {
|
||||
audioLanguage := lang.Canonical(candidate.AudioLanguage)
|
||||
bestScore := -1
|
||||
var best models.ExternalSubtitle
|
||||
for _, subtitle := range candidate.ExternalSubtitles {
|
||||
if strings.TrimSpace(subtitle.Path) == "" ||
|
||||
!isDialogueSubtitleFormat(subtitle.Format) ||
|
||||
subtitle.Forced ||
|
||||
subtitle.HearingImpaired {
|
||||
continue
|
||||
}
|
||||
score := 0
|
||||
subtitleLanguage := lang.Canonical(subtitle.Language)
|
||||
if audioLanguage != "" && subtitleLanguage == audioLanguage {
|
||||
score += 100
|
||||
} else if audioLanguage == "" && subtitleLanguage == "en" {
|
||||
score += 20
|
||||
}
|
||||
if subtitle.Default {
|
||||
score += 10
|
||||
}
|
||||
if subtitleLanguage != "" {
|
||||
score += 1
|
||||
}
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
best = subtitle
|
||||
}
|
||||
}
|
||||
if bestScore < 0 {
|
||||
return models.ExternalSubtitle{}, false
|
||||
}
|
||||
return best, true
|
||||
}
|
||||
|
||||
func isDialogueSubtitleFormat(format string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(format)) {
|
||||
case "srt", "subrip", "vtt", "webvtt":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func dialogueRefinedStart(cues []subtitles.SubtitleCue, segment Segment, cfg Config) (float64, bool) {
|
||||
if segment.End <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
windowEnd := math.Min(segment.Start+cfg.DialogueRefinementWindowSeconds, segment.End)
|
||||
if windowEnd <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
lastDialogueEnd := segment.Start
|
||||
for _, cue := range cues {
|
||||
if !cueLooksLikeDialogue(cue) {
|
||||
continue
|
||||
}
|
||||
start := cue.Start.Seconds()
|
||||
end := cue.End.Seconds()
|
||||
if end <= segment.Start {
|
||||
continue
|
||||
}
|
||||
if start > windowEnd {
|
||||
break
|
||||
}
|
||||
if end > lastDialogueEnd {
|
||||
lastDialogueEnd = end
|
||||
}
|
||||
}
|
||||
|
||||
if lastDialogueEnd <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
if lastDialogueEnd-segment.Start > cfg.DialogueRefinementMaxShiftSeconds {
|
||||
return 0, false
|
||||
}
|
||||
if segment.End-lastDialogueEnd < cfg.DialogueRefinementMinimumRemainingSeconds {
|
||||
return 0, false
|
||||
}
|
||||
return lastDialogueEnd, true
|
||||
}
|
||||
|
||||
func cueLooksLikeDialogue(cue subtitles.SubtitleCue) bool {
|
||||
for _, line := range cue.Lines {
|
||||
if lineLooksLikeDialogue(line) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lineLooksLikeDialogue(line string) bool {
|
||||
cleaned := strings.TrimSpace(stripSubtitleDialogueMarkup(line))
|
||||
if cleaned == "" || strings.Contains(cleaned, "♪") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(cleaned)
|
||||
if isBracketedCue(lower, "[", "]") ||
|
||||
isBracketedCue(lower, "(", ")") ||
|
||||
isBracketedCue(lower, "{", "}") {
|
||||
return false
|
||||
}
|
||||
for _, r := range cleaned {
|
||||
if unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stripSubtitleDialogueMarkup(line string) string {
|
||||
line = stripSubtitleASSTags(line)
|
||||
line = htmlTagPattern.ReplaceAllString(line, "")
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
func isBracketedCue(line, open, close string) bool {
|
||||
return strings.HasPrefix(line, open) && strings.HasSuffix(line, close)
|
||||
}
|
||||
|
||||
func stripSubtitleASSTags(line string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(line))
|
||||
inTag := false
|
||||
for _, r := range line {
|
||||
switch {
|
||||
case r == '{':
|
||||
inTag = true
|
||||
case r == '}':
|
||||
inTag = false
|
||||
case !inTag:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package intromarkers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestDialogueBoundaryRefinerMovesStartPastEarlyDialogue(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
candidate := Candidate{
|
||||
FileID: 636600,
|
||||
AudioLanguage: "eng",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte(swatS05E02IntroLeadInSRT), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected dialogue refinement to apply")
|
||||
}
|
||||
if math.Abs(refined.Start-330.852) > 0.001 {
|
||||
t.Fatalf("refined start = %.3f, want 330.852", refined.Start)
|
||||
}
|
||||
if refined.End != segment.End {
|
||||
t.Fatalf("refined end = %.3f, want %.3f", refined.End, segment.End)
|
||||
}
|
||||
if refined.Algorithm != ChromaprintDialogueAlgorithm {
|
||||
t.Fatalf("algorithm = %q, want %q", refined.Algorithm, ChromaprintDialogueAlgorithm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialogueBoundaryRefinerIgnoresMusicCues(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{Start: 100, End: 150, Confidence: 0.85, Algorithm: ChromaprintAlgorithm}
|
||||
candidate := Candidate{
|
||||
FileID: 1,
|
||||
AudioLanguage: "en",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte("1\n00:01:40,000 --> 00:01:48,000\n♪ Opening theme ♪\n\n"), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("music cue should not refine segment, got %+v", refined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialogueBoundaryRefinerKeepsMinimumRemainingDuration(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{Start: 100, End: 112, Confidence: 0.85, Algorithm: ChromaprintAlgorithm}
|
||||
candidate := Candidate{
|
||||
FileID: 1,
|
||||
AudioLanguage: "en",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte("1\n00:01:40,000 --> 00:01:45,000\nStill talking.\n\n"), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("refinement should not leave too-short intro, got %+v", refined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDialogueSubtitlePrefersAudioLanguage(t *testing.T) {
|
||||
candidate := Candidate{
|
||||
AudioLanguage: "eng",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.es.srt", Language: "es", Format: "srt"},
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
|
||||
subtitle, ok := selectDialogueSubtitle(candidate)
|
||||
if !ok {
|
||||
t.Fatal("expected subtitle selection")
|
||||
}
|
||||
if subtitle.Path != "/episode.en.srt" {
|
||||
t.Fatalf("selected %q, want English sidecar", subtitle.Path)
|
||||
}
|
||||
}
|
||||
|
||||
const swatS05E02IntroLeadInSRT = `142
|
||||
00:05:11,006 --> 00:05:13,051
|
||||
Okay.
|
||||
|
||||
143
|
||||
00:05:13,095 --> 00:05:15,358
|
||||
I'm in.
|
||||
|
||||
144
|
||||
00:05:15,402 --> 00:05:19,144
|
||||
Never be in a hurry to die.
|
||||
|
||||
145
|
||||
00:05:19,188 --> 00:05:20,972
|
||||
Weapons are nice,
|
||||
but we need intel.
|
||||
|
||||
146
|
||||
00:05:21,016 --> 00:05:22,539
|
||||
We don't know where Delfina is
|
||||
|
||||
147
|
||||
00:05:22,583 --> 00:05:24,106
|
||||
or how many people
|
||||
are holding her.
|
||||
|
||||
148
|
||||
00:05:24,149 --> 00:05:25,499
|
||||
It won't be easy.
|
||||
|
||||
149
|
||||
00:05:25,542 --> 00:05:27,196
|
||||
Senor Novak has
|
||||
friends everywhere.
|
||||
|
||||
150
|
||||
00:05:27,239 --> 00:05:28,806
|
||||
I got an idea
|
||||
of where we can start.
|
||||
|
||||
151
|
||||
00:05:28,850 --> 00:05:30,852
|
||||
Follow me.
|
||||
|
||||
152
|
||||
00:06:20,771 --> 00:06:22,643
|
||||
Oh, keep screaming, chica.
|
||||
`
|
||||
@@ -45,6 +45,8 @@ const baseCandidateSelect = `
|
||||
COALESCE(mf.edition_key, ''),
|
||||
mf.chapters,
|
||||
mf.audio_tracks,
|
||||
mf.subtitle_tracks,
|
||||
mf.external_subtitles,
|
||||
mf.intro_start,
|
||||
mf.intro_end,
|
||||
mf.intro_markers_source,
|
||||
@@ -200,7 +202,7 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
var candidates []Candidate
|
||||
for rows.Next() {
|
||||
var c Candidate
|
||||
var chaptersJSON, audioTracksJSON []byte
|
||||
var chaptersJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON []byte
|
||||
if err := rows.Scan(
|
||||
&c.FileID,
|
||||
&c.EpisodeID,
|
||||
@@ -214,6 +216,8 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
&c.EditionKey,
|
||||
&chaptersJSON,
|
||||
&audioTracksJSON,
|
||||
&subtitleTracksJSON,
|
||||
&externalSubtitlesJSON,
|
||||
&c.IntroStart,
|
||||
&c.IntroEnd,
|
||||
&c.IntroMarkersSource,
|
||||
@@ -235,6 +239,16 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
}
|
||||
c.AudioLanguage = effectiveAudioLanguage(tracks)
|
||||
}
|
||||
if len(subtitleTracksJSON) > 0 {
|
||||
if err := json.Unmarshal(subtitleTracksJSON, &c.SubtitleTracks); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling subtitle tracks for file %d: %w", c.FileID, err)
|
||||
}
|
||||
}
|
||||
if len(externalSubtitlesJSON) > 0 {
|
||||
if err := json.Unmarshal(externalSubtitlesJSON, &c.ExternalSubtitles); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling external subtitles for file %d: %w", c.FileID, err)
|
||||
}
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -484,7 +498,7 @@ func (r *Repository) LoadSeasonState(ctx context.Context, state SeasonState, cfg
|
||||
state.MediaFolderID,
|
||||
state.AnalysisGroupKey,
|
||||
AlgorithmVersion,
|
||||
cfg.ConfigHash(),
|
||||
cfg.AnalysisConfigHash(),
|
||||
).Scan(
|
||||
&existing.SeasonID,
|
||||
&existing.MediaFolderID,
|
||||
@@ -535,7 +549,7 @@ func (r *Repository) UpsertSeasonState(ctx context.Context, state SeasonState, c
|
||||
state.MediaFolderID,
|
||||
state.AnalysisGroupKey,
|
||||
AlgorithmVersion,
|
||||
cfg.ConfigHash(),
|
||||
cfg.AnalysisConfigHash(),
|
||||
state.InputSignature,
|
||||
state.EpisodeCount,
|
||||
state.FileCount,
|
||||
@@ -576,21 +590,57 @@ func shouldApplyIntroPatch(row markerRow, patch IntroMarkerPatch) bool {
|
||||
if models.MarkerSourcePriority(source) < models.MarkerSourcePriority(patch.Source) {
|
||||
return true
|
||||
}
|
||||
if row.IntroMarkersAlgorithm == nil || *row.IntroMarkersAlgorithm != patch.Algorithm {
|
||||
return true
|
||||
}
|
||||
if row.IntroMarkersConfidence == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
existingAlgorithm := ""
|
||||
if row.IntroMarkersAlgorithm != nil {
|
||||
existingAlgorithm = *row.IntroMarkersAlgorithm
|
||||
}
|
||||
if existingAlgorithm == "" {
|
||||
return true
|
||||
}
|
||||
patchPriority := scannerAlgorithmPriority(patch.Algorithm)
|
||||
existingPriority := scannerAlgorithmPriority(existingAlgorithm)
|
||||
if patchPriority > existingPriority {
|
||||
return true
|
||||
}
|
||||
if patchPriority < existingPriority {
|
||||
return false
|
||||
}
|
||||
if patch.Confidence > *row.IntroMarkersConfidence {
|
||||
return true
|
||||
}
|
||||
if patch.Confidence >= *row.IntroMarkersConfidence && introRangeDiffers(row, patch, 0.5) {
|
||||
if patch.Confidence < *row.IntroMarkersConfidence {
|
||||
return false
|
||||
}
|
||||
if existingAlgorithm != patch.Algorithm && patchPriority == 0 {
|
||||
return true
|
||||
}
|
||||
if existingAlgorithm == patch.Algorithm && introRangeDiffers(row, patch, 0.5) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func scannerAlgorithmPriority(algorithm string) int {
|
||||
switch algorithm {
|
||||
case ChapterSilenceAlgorithm:
|
||||
return 40
|
||||
case ChapterAlgorithm:
|
||||
return 30
|
||||
case EpisodeVersionCopyAlgorithm:
|
||||
return 20
|
||||
case ChromaprintDialogueAlgorithm:
|
||||
return 15
|
||||
case ChromaprintAlgorithm:
|
||||
return 10
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func introRangeDiffers(row markerRow, patch IntroMarkerPatch, tolerance float64) bool {
|
||||
if row.IntroStart == nil || row.IntroEnd == nil {
|
||||
return true
|
||||
@@ -622,9 +672,42 @@ func effectiveAudioLanguage(tracks []models.AudioTrack) string {
|
||||
func InputSignature(candidates []Candidate) string {
|
||||
parts := make([]string, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
parts = append(parts, fmt.Sprintf("%d:%s:%d:%.3f:%s", c.FileID, c.FileHash, c.FileSize, c.DurationSeconds, c.EpisodeID))
|
||||
parts = append(parts, fmt.Sprintf("%d:%s:%d:%.3f:%s:%s",
|
||||
c.FileID,
|
||||
c.FileHash,
|
||||
c.FileSize,
|
||||
c.DurationSeconds,
|
||||
c.EpisodeID,
|
||||
subtitleInventorySignature(c),
|
||||
))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func subtitleInventorySignature(candidate Candidate) string {
|
||||
parts := make([]string, 0, len(candidate.ExternalSubtitles)+len(candidate.SubtitleTracks))
|
||||
for _, sub := range candidate.ExternalSubtitles {
|
||||
parts = append(parts, fmt.Sprintf("ext:%s:%s:%s:%t:%t:%t",
|
||||
sub.Path,
|
||||
sub.Language,
|
||||
sub.Format,
|
||||
sub.Forced,
|
||||
sub.Default,
|
||||
sub.HearingImpaired,
|
||||
))
|
||||
}
|
||||
for _, track := range candidate.SubtitleTracks {
|
||||
parts = append(parts, fmt.Sprintf("emb:%d:%s:%s:%t:%t:%t",
|
||||
track.Index,
|
||||
track.Language,
|
||||
track.Codec,
|
||||
track.Forced,
|
||||
track.Default,
|
||||
track.HearingImpaired,
|
||||
))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
@@ -57,3 +57,107 @@ func TestShouldApplyIntroPatchRejectsLowerPrioritySource(t *testing.T) {
|
||||
t.Fatal("scanner patch should not overwrite manual marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsLowerConfidenceAlgorithmChange(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.95
|
||||
algorithm := ChapterAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("lower-confidence chromaprint patch should not overwrite chapter marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsEqualConfidenceLowerRankAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.85
|
||||
algorithm := EpisodeVersionCopyAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("equal-confidence chromaprint patch should not overwrite copied chapter marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsHigherConfidenceLowerRankAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.85
|
||||
algorithm := EpisodeVersionCopyAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.90,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("higher-confidence chromaprint patch should not replace copied marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchAllowsHigherRankLowerConfidenceAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.95
|
||||
algorithm := ChromaprintAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChapterAlgorithm,
|
||||
}
|
||||
|
||||
if !shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("higher-rank chapter patch should replace lower-rank chromaprint marker")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,31 +11,37 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AlgorithmVersion = 1
|
||||
ChapterAlgorithm = "chapter:v1"
|
||||
ChapterSilenceAlgorithm = "chapter:silence:v1"
|
||||
EpisodeVersionCopyAlgorithm = "episode-version-copy:v1"
|
||||
ChromaprintAlgorithm = "chromaprint:v1"
|
||||
ChromaprintFormat = "chromaprint:raw:uint32le"
|
||||
DefaultPointHopSeconds = 0.123
|
||||
AlgorithmVersion = 1
|
||||
AnalysisBehaviorVersion = 2
|
||||
ChapterAlgorithm = "chapter:v1"
|
||||
ChapterSilenceAlgorithm = "chapter:silence:v1"
|
||||
EpisodeVersionCopyAlgorithm = "episode-version-copy:v1"
|
||||
ChromaprintAlgorithm = "chromaprint:v1"
|
||||
ChromaprintDialogueAlgorithm = "chromaprint:dialogue:v1"
|
||||
ChromaprintFormat = "chromaprint:raw:uint32le"
|
||||
DefaultPointHopSeconds = 0.123
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
MaxParallelFFmpeg int
|
||||
AnalysisPercent int
|
||||
AnalysisLengthLimitMinutes int
|
||||
MinimumIntroDurationSeconds int
|
||||
MaximumIntroDurationSeconds int
|
||||
SilenceRefinementEnabled bool
|
||||
SilenceWindowBeforeSeconds float64
|
||||
SilenceWindowAfterSeconds float64
|
||||
SilenceMinimumDurationSeconds float64
|
||||
SilenceNoiseThresholdDB *int
|
||||
SilenceMinimumExtensionSeconds float64
|
||||
SilenceMaximumExtensionSeconds float64
|
||||
SilenceBackfillLimit int
|
||||
SilenceBackfillMaxDuration time.Duration
|
||||
FFmpegPath string
|
||||
MaxParallelFFmpeg int
|
||||
AnalysisPercent int
|
||||
AnalysisLengthLimitMinutes int
|
||||
MinimumIntroDurationSeconds int
|
||||
MaximumIntroDurationSeconds int
|
||||
SilenceRefinementEnabled bool
|
||||
SilenceWindowBeforeSeconds float64
|
||||
SilenceWindowAfterSeconds float64
|
||||
SilenceMinimumDurationSeconds float64
|
||||
SilenceNoiseThresholdDB *int
|
||||
SilenceMinimumExtensionSeconds float64
|
||||
SilenceMaximumExtensionSeconds float64
|
||||
SilenceBackfillLimit int
|
||||
SilenceBackfillMaxDuration time.Duration
|
||||
DialogueRefinementEnabled bool
|
||||
DialogueRefinementWindowSeconds float64
|
||||
DialogueRefinementMaxShiftSeconds float64
|
||||
DialogueRefinementMinimumRemainingSeconds float64
|
||||
}
|
||||
|
||||
func DefaultConfig(ffmpegPath string) Config {
|
||||
@@ -43,21 +49,25 @@ func DefaultConfig(ffmpegPath string) Config {
|
||||
ffmpegPath = "ffmpeg"
|
||||
}
|
||||
return Config{
|
||||
FFmpegPath: ffmpegPath,
|
||||
MaxParallelFFmpeg: 1,
|
||||
AnalysisPercent: 25,
|
||||
AnalysisLengthLimitMinutes: 10,
|
||||
MinimumIntroDurationSeconds: 15,
|
||||
MaximumIntroDurationSeconds: 120,
|
||||
SilenceRefinementEnabled: true,
|
||||
SilenceWindowBeforeSeconds: 3,
|
||||
SilenceWindowAfterSeconds: 30,
|
||||
SilenceMinimumDurationSeconds: 0.33,
|
||||
SilenceNoiseThresholdDB: intPtr(-50),
|
||||
SilenceMinimumExtensionSeconds: 0.5,
|
||||
SilenceMaximumExtensionSeconds: 30,
|
||||
SilenceBackfillLimit: 2000,
|
||||
SilenceBackfillMaxDuration: 45 * time.Minute,
|
||||
FFmpegPath: ffmpegPath,
|
||||
MaxParallelFFmpeg: 1,
|
||||
AnalysisPercent: 25,
|
||||
AnalysisLengthLimitMinutes: 10,
|
||||
MinimumIntroDurationSeconds: 15,
|
||||
MaximumIntroDurationSeconds: 120,
|
||||
SilenceRefinementEnabled: true,
|
||||
SilenceWindowBeforeSeconds: 3,
|
||||
SilenceWindowAfterSeconds: 30,
|
||||
SilenceMinimumDurationSeconds: 0.33,
|
||||
SilenceNoiseThresholdDB: intPtr(-50),
|
||||
SilenceMinimumExtensionSeconds: 0.5,
|
||||
SilenceMaximumExtensionSeconds: 30,
|
||||
SilenceBackfillLimit: 2000,
|
||||
SilenceBackfillMaxDuration: 45 * time.Minute,
|
||||
DialogueRefinementEnabled: true,
|
||||
DialogueRefinementWindowSeconds: 15,
|
||||
DialogueRefinementMaxShiftSeconds: 20,
|
||||
DialogueRefinementMinimumRemainingSeconds: 15,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,6 +114,15 @@ func (c Config) normalized() Config {
|
||||
if c.SilenceBackfillMaxDuration <= 0 {
|
||||
c.SilenceBackfillMaxDuration = 45 * time.Minute
|
||||
}
|
||||
if c.DialogueRefinementWindowSeconds <= 0 {
|
||||
c.DialogueRefinementWindowSeconds = 15
|
||||
}
|
||||
if c.DialogueRefinementMaxShiftSeconds <= 0 {
|
||||
c.DialogueRefinementMaxShiftSeconds = 20
|
||||
}
|
||||
if c.DialogueRefinementMinimumRemainingSeconds <= 0 {
|
||||
c.DialogueRefinementMinimumRemainingSeconds = float64(c.MinimumIntroDurationSeconds)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -122,6 +141,19 @@ func (c Config) ConfigHash() string {
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
func (c Config) AnalysisConfigHash() string {
|
||||
c = c.normalized()
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d:%t:%.3f:%.3f:%.3f",
|
||||
c.ConfigHash(),
|
||||
AnalysisBehaviorVersion,
|
||||
c.DialogueRefinementEnabled,
|
||||
c.DialogueRefinementWindowSeconds,
|
||||
c.DialogueRefinementMaxShiftSeconds,
|
||||
c.DialogueRefinementMinimumRemainingSeconds,
|
||||
)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
type Candidate struct {
|
||||
FileID int
|
||||
EpisodeID string
|
||||
@@ -135,6 +167,8 @@ type Candidate struct {
|
||||
EditionKey string
|
||||
AudioLanguage string
|
||||
Chapters []models.MediaChapter
|
||||
SubtitleTracks []models.SubtitleTrack
|
||||
ExternalSubtitles []models.ExternalSubtitle
|
||||
IntroStart *float64
|
||||
IntroEnd *float64
|
||||
IntroMarkersSource *string
|
||||
@@ -220,21 +254,24 @@ type SeasonState struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
LibrariesScanned int `json:"libraries_scanned"`
|
||||
FilesConsidered int `json:"files_considered"`
|
||||
SeasonGroupsConsidered int `json:"season_groups_considered"`
|
||||
FingerprintsComputed int `json:"fingerprints_computed"`
|
||||
FingerprintCacheHits int `json:"fingerprint_cache_hits"`
|
||||
ChapterMarkersWritten int `json:"chapter_markers_written"`
|
||||
ChromaprintMarkersWritten int `json:"chromaprint_markers_written"`
|
||||
GroupsNotFound int `json:"groups_not_found"`
|
||||
GroupsSkipped int `json:"groups_skipped"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
ChromaprintSupported bool `json:"chromaprint_supported"`
|
||||
ChromaprintSupportMessage string `json:"chromaprint_support_message,omitempty"`
|
||||
SilenceRefinementsAttempted int `json:"silence_refinements_attempted"`
|
||||
SilenceRefinementsApplied int `json:"silence_refinements_applied"`
|
||||
SilenceRefinementErrors int `json:"silence_refinement_errors"`
|
||||
EpisodeVersionMarkersCopied int `json:"episode_version_markers_copied"`
|
||||
SilenceBackfillConsidered int `json:"silence_backfill_considered"`
|
||||
LibrariesScanned int `json:"libraries_scanned"`
|
||||
FilesConsidered int `json:"files_considered"`
|
||||
SeasonGroupsConsidered int `json:"season_groups_considered"`
|
||||
FingerprintsComputed int `json:"fingerprints_computed"`
|
||||
FingerprintCacheHits int `json:"fingerprint_cache_hits"`
|
||||
ChapterMarkersWritten int `json:"chapter_markers_written"`
|
||||
ChromaprintMarkersWritten int `json:"chromaprint_markers_written"`
|
||||
GroupsNotFound int `json:"groups_not_found"`
|
||||
GroupsSkipped int `json:"groups_skipped"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
ChromaprintSupported bool `json:"chromaprint_supported"`
|
||||
ChromaprintSupportMessage string `json:"chromaprint_support_message,omitempty"`
|
||||
SilenceRefinementsAttempted int `json:"silence_refinements_attempted"`
|
||||
SilenceRefinementsApplied int `json:"silence_refinements_applied"`
|
||||
SilenceRefinementErrors int `json:"silence_refinement_errors"`
|
||||
EpisodeVersionMarkersCopied int `json:"episode_version_markers_copied"`
|
||||
SilenceBackfillConsidered int `json:"silence_backfill_considered"`
|
||||
DialogueRefinementsAttempted int `json:"dialogue_refinements_attempted"`
|
||||
DialogueRefinementsApplied int `json:"dialogue_refinements_applied"`
|
||||
DialogueRefinementErrors int `json:"dialogue_refinement_errors"`
|
||||
}
|
||||
|
||||
@@ -337,7 +337,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
|
||||
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
|
||||
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
|
||||
slog.Error("resolve transcode seek target",
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// OutcomeStatus values for a contribution attempt, in addition to the provider
|
||||
// SubmissionStatus* values.
|
||||
const (
|
||||
OutcomeStatusError = "error"
|
||||
OutcomeStatusRateLimited = "rate_limited"
|
||||
OutcomeStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
// ContributeOptions scopes a contribution run.
|
||||
type ContributeOptions struct {
|
||||
// Provider restricts the run to a single provider id ("" = all enabled).
|
||||
Provider string
|
||||
// Segments restricts the run to specific kinds (nil = every present segment).
|
||||
Segments []MarkerKind
|
||||
// Auto marks a background run: a segment is only contributed when the
|
||||
// provider has contribute_auto_local enabled and the segment is a local
|
||||
// (scanner) intro at or above the provider's confidence threshold.
|
||||
Auto bool
|
||||
}
|
||||
|
||||
// ContributionOutcome is the per (provider, segment) result of a run.
|
||||
type ContributionOutcome struct {
|
||||
Provider string
|
||||
Segment MarkerKind
|
||||
Status string // pending | accepted | rejected | error | rate_limited | skipped
|
||||
SubmissionID string
|
||||
Reason string // skip reason or error message
|
||||
RetryAfter time.Duration
|
||||
}
|
||||
|
||||
// providerConfigReader is the read surface ContributionService needs from the
|
||||
// provider config store (satisfied by *ProviderConfigStore).
|
||||
type providerConfigReader interface {
|
||||
Get(provider string) (ProviderConfig, bool)
|
||||
}
|
||||
|
||||
// contributionRecorder is the audit surface ContributionService needs
|
||||
// (satisfied by *ContributionStore).
|
||||
type contributionRecorder interface {
|
||||
AlreadySubmitted(ctx context.Context, fileID int, provider, segmentKind, contentHash string) (bool, error)
|
||||
Record(ctx context.Context, row ContributionRow) error
|
||||
}
|
||||
|
||||
// ContributionService submits a file's eligible markers to enabled submitter
|
||||
// providers, idempotently and audited. Both the on-demand admin path and the
|
||||
// daily auto-submission task funnel through ContributeFile.
|
||||
type ContributionService struct {
|
||||
registry *Registry
|
||||
resolver ExternalIDResolver
|
||||
config providerConfigReader
|
||||
store contributionRecorder
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewContributionService constructs the service.
|
||||
func NewContributionService(registry *Registry, resolver ExternalIDResolver, config providerConfigReader, store contributionRecorder, logger *slog.Logger) *ContributionService {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &ContributionService{registry: registry, resolver: resolver, config: config, store: store, logger: logger}
|
||||
}
|
||||
|
||||
type segmentData struct {
|
||||
kind MarkerKind
|
||||
name string
|
||||
start *float64
|
||||
end *float64
|
||||
source *string
|
||||
confidence *float64
|
||||
}
|
||||
|
||||
func fileSegments(file *models.MediaFile) []segmentData {
|
||||
return []segmentData{
|
||||
{MarkerKindIntro, "intro", file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersConfidence},
|
||||
{MarkerKindCredits, "credits", file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersConfidence},
|
||||
{MarkerKindRecap, "recap", file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersConfidence},
|
||||
{MarkerKindPreview, "preview", file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersConfidence},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ContributionService) submitters() []Submitter {
|
||||
if s == nil || s.registry == nil {
|
||||
return nil
|
||||
}
|
||||
var out []Submitter
|
||||
for _, p := range s.registry.Providers() {
|
||||
if sub, ok := p.(Submitter); ok {
|
||||
out = append(out, sub)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ContributeFile submits the file's eligible markers and returns one outcome
|
||||
// per (provider, segment) attempted (including explicit skips).
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error) {
|
||||
if s == nil || file == nil || s.registry == nil || s.resolver == nil || s.config == nil || s.store == nil {
|
||||
return nil, nil
|
||||
}
|
||||
ids, err := s.resolver.ResolveForFile(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ids.HasAnyID() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
want := func(k MarkerKind) bool {
|
||||
if len(opts.Segments) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, kind := range opts.Segments {
|
||||
if kind == k {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var outcomes []ContributionOutcome
|
||||
for _, sub := range s.submitters() {
|
||||
providerID := sub.ID()
|
||||
if opts.Provider != "" && opts.Provider != providerID {
|
||||
continue
|
||||
}
|
||||
cfg, ok := s.config.Get(providerID)
|
||||
if !ok || !cfg.ContributeEnabled {
|
||||
continue
|
||||
}
|
||||
for _, seg := range fileSegments(file) {
|
||||
if !want(seg.kind) {
|
||||
continue
|
||||
}
|
||||
if outcome, attempted := s.contributeSegment(ctx, sub, providerID, cfg, file, ids, seg, opts.Auto); attempted {
|
||||
outcomes = append(outcomes, outcome)
|
||||
if outcome.Status == OutcomeStatusRateLimited {
|
||||
return outcomes, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func (s *ContributionService) contributeSegment(
|
||||
ctx context.Context,
|
||||
sub Submitter,
|
||||
providerID string,
|
||||
cfg ProviderConfig,
|
||||
file *models.MediaFile,
|
||||
ids ExternalIDs,
|
||||
seg segmentData,
|
||||
auto bool,
|
||||
) (ContributionOutcome, bool) {
|
||||
if seg.start == nil || seg.end == nil {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
source := ""
|
||||
if seg.source != nil {
|
||||
source = *seg.source
|
||||
}
|
||||
// Only ever contribute our own data; never round-trip online markers.
|
||||
if source != models.MarkerSourceScanner && source != models.MarkerSourceManual {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
if auto {
|
||||
if !cfg.ContributeAutoLocal || source != models.MarkerSourceScanner || seg.kind != MarkerKindIntro {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
confidence := 0.0
|
||||
if seg.confidence != nil {
|
||||
confidence = *seg.confidence
|
||||
}
|
||||
if confidence < cfg.ContributeMinConfidence {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
}
|
||||
if missing := missingRequiredExternalIDs(sub, ids); len(missing) > 0 {
|
||||
return ContributionOutcome{
|
||||
Provider: providerID,
|
||||
Segment: seg.kind,
|
||||
Status: OutcomeStatusSkipped,
|
||||
Reason: strings.Join(missing, ",") + " id required",
|
||||
}, true
|
||||
}
|
||||
|
||||
startMs := int64(*seg.start * 1000)
|
||||
endMs := int64(*seg.end * 1000)
|
||||
durMs := int64(file.Duration) * 1000
|
||||
hash := ContentHash(seg.name, &startMs, &endMs, &durMs, contributionTargetParts(ids)...)
|
||||
|
||||
already, err := s.store.AlreadySubmitted(ctx, file.ID, providerID, seg.name, hash)
|
||||
if err != nil {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusError, Reason: err.Error()}, true
|
||||
}
|
||||
if already {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusSkipped, Reason: "already submitted"}, true
|
||||
}
|
||||
|
||||
startDur := time.Duration(*seg.start * float64(time.Second))
|
||||
endDur := time.Duration(*seg.end * float64(time.Second))
|
||||
req := SubmissionRequest{
|
||||
Kind: ids.Kind,
|
||||
ExternalIDs: ids.AsRequestMap(),
|
||||
SeasonNumber: ids.SeasonNumber,
|
||||
EpisodeNumber: ids.EpisodeNumber,
|
||||
Segment: seg.kind,
|
||||
Start: &startDur,
|
||||
End: &endDur,
|
||||
Duration: time.Duration(file.Duration) * time.Second,
|
||||
}
|
||||
|
||||
row := ContributionRow{
|
||||
MediaFileID: file.ID,
|
||||
Provider: providerID,
|
||||
SegmentKind: seg.name,
|
||||
Source: source,
|
||||
SubmittedStartMs: &startMs,
|
||||
SubmittedEndMs: &endMs,
|
||||
VideoDurationMs: &durMs,
|
||||
ContentHash: hash,
|
||||
}
|
||||
|
||||
result, err := sub.SubmitMarker(ctx, req)
|
||||
if err != nil {
|
||||
msg := err.Error()
|
||||
row.Status = OutcomeStatusError
|
||||
row.Error = &msg
|
||||
if recErr := s.store.Record(ctx, row); recErr != nil {
|
||||
s.logger.Warn("record contribution error failed", "file_id", file.ID, "provider", providerID, "segment", seg.name, "error", recErr)
|
||||
}
|
||||
if after, ok := RetryAfter(err); ok {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusRateLimited, Reason: msg, RetryAfter: after}, true
|
||||
}
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusError, Reason: msg}, true
|
||||
}
|
||||
|
||||
row.Status = result.Status
|
||||
if row.Status == "" {
|
||||
row.Status = SubmissionStatusPending
|
||||
}
|
||||
if result.ID != "" {
|
||||
id := result.ID
|
||||
row.SubmissionID = &id
|
||||
}
|
||||
if err := s.store.Record(ctx, row); err != nil {
|
||||
s.logger.Warn("record contribution failed", "file_id", file.ID, "provider", providerID, "segment", seg.name, "error", err)
|
||||
}
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: row.Status, SubmissionID: result.ID}, true
|
||||
}
|
||||
|
||||
func contributionTargetParts(ids ExternalIDs) []string {
|
||||
return []string{
|
||||
itemTypeName(ids.Kind),
|
||||
ids.TmdbID,
|
||||
ids.ImdbID,
|
||||
ids.TvdbID,
|
||||
intPart(ids.SeasonNumber),
|
||||
intPart(ids.EpisodeNumber),
|
||||
}
|
||||
}
|
||||
|
||||
func intPart(v int) string {
|
||||
if v == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.Itoa(v)
|
||||
}
|
||||
|
||||
func missingRequiredExternalIDs(sub Submitter, ids ExternalIDs) []string {
|
||||
reqProvider, ok := sub.(SubmissionRequirementProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
reqs := reqProvider.SubmissionRequirements()
|
||||
if len(reqs.RequiredExternalIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := ids.AsRequestMap()
|
||||
var missing []string
|
||||
for _, key := range reqs.RequiredExternalIDs {
|
||||
if strings.TrimSpace(values[key]) == "" {
|
||||
missing = append(missing, key)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakeSubmitter struct {
|
||||
id string
|
||||
submitted []SubmissionRequest
|
||||
result SubmissionResult
|
||||
err error
|
||||
required []string
|
||||
}
|
||||
|
||||
func (f *fakeSubmitter) ID() string { return f.id }
|
||||
func (f *fakeSubmitter) FetchMarkers(context.Context, Request) (Result, error) { return Result{}, nil }
|
||||
func (f *fakeSubmitter) SubmitMarker(_ context.Context, req SubmissionRequest) (SubmissionResult, error) {
|
||||
f.submitted = append(f.submitted, req)
|
||||
if f.err != nil {
|
||||
return SubmissionResult{}, f.err
|
||||
}
|
||||
if f.result.Status == "" {
|
||||
return SubmissionResult{ID: "id1", Status: SubmissionStatusPending}, nil
|
||||
}
|
||||
return f.result, nil
|
||||
}
|
||||
func (f *fakeSubmitter) FetchUserStats(context.Context) (UserStats, error) { return UserStats{}, nil }
|
||||
func (f *fakeSubmitter) SubmissionRequirements() SubmissionRequirements {
|
||||
return SubmissionRequirements{RequiredExternalIDs: f.required}
|
||||
}
|
||||
|
||||
type fakeResolver struct{ ids ExternalIDs }
|
||||
|
||||
func (f fakeResolver) ResolveForFile(context.Context, *models.MediaFile) (ExternalIDs, error) {
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
type fakeConfig map[string]ProviderConfig
|
||||
|
||||
func (f fakeConfig) Get(p string) (ProviderConfig, bool) { c, ok := f[p]; return c, ok }
|
||||
|
||||
type fakeRecorder struct {
|
||||
already bool
|
||||
recorded []ContributionRow
|
||||
}
|
||||
|
||||
func (f *fakeRecorder) AlreadySubmitted(context.Context, int, string, string, string) (bool, error) {
|
||||
return f.already, nil
|
||||
}
|
||||
func (f *fakeRecorder) Record(_ context.Context, row ContributionRow) error {
|
||||
f.recorded = append(f.recorded, row)
|
||||
return nil
|
||||
}
|
||||
|
||||
func floatPtr(v float64) *float64 { return &v }
|
||||
func strPtr(v string) *string { return &v }
|
||||
|
||||
func newContribFile() *models.MediaFile {
|
||||
return &models.MediaFile{ID: 7, EpisodeID: "ep1", SeasonNumber: 1, EpisodeNumber: 3, Duration: 1800}
|
||||
}
|
||||
|
||||
func newContribService(sub *fakeSubmitter, cfg fakeConfig, rec *fakeRecorder) *ContributionService {
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TmdbID: "1234", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
return NewContributionService(reg, resolver, cfg, rec, nil)
|
||||
}
|
||||
|
||||
func TestContributeSkipsOnlineSourced(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceOnline) // came FROM introdb
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, &fakeRecorder{})
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 0 {
|
||||
t.Errorf("online-sourced marker must not be submitted, got %d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 0 {
|
||||
t.Errorf("expected no outcomes, got %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSubmitsManualOnDemand(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.CreditsStart, file.CreditsEnd = floatPtr(1500), floatPtr(1800)
|
||||
file.CreditsMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 || sub.submitted[0].Segment != MarkerKindCredits {
|
||||
t.Fatalf("expected 1 credits submission, got %+v", sub.submitted)
|
||||
}
|
||||
if len(rec.recorded) != 1 || rec.recorded[0].Status != SubmissionStatusPending {
|
||||
t.Errorf("expected recorded pending, got %+v", rec.recorded)
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != SubmissionStatusPending {
|
||||
t.Errorf("outcomes = %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeAutoGatesOnThresholdAndKind(t *testing.T) {
|
||||
cfg := fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true, ContributeAutoLocal: true, ContributeMinConfidence: 0.9}}
|
||||
|
||||
subLow := &fakeSubmitter{id: "introdb"}
|
||||
fileLow := newContribFile()
|
||||
fileLow.IntroStart, fileLow.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
fileLow.IntroMarkersSource = strPtr(models.MarkerSourceScanner)
|
||||
fileLow.IntroMarkersConfidence = floatPtr(0.8)
|
||||
if _, err := newContribService(subLow, cfg, &fakeRecorder{}).ContributeFile(context.Background(), fileLow, ContributeOptions{Auto: true}); err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(subLow.submitted) != 0 {
|
||||
t.Errorf("below-threshold scanner intro must be skipped, got %d", len(subLow.submitted))
|
||||
}
|
||||
|
||||
subHigh := &fakeSubmitter{id: "introdb"}
|
||||
fileHigh := newContribFile()
|
||||
fileHigh.IntroStart, fileHigh.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
fileHigh.IntroMarkersSource = strPtr(models.MarkerSourceScanner)
|
||||
fileHigh.IntroMarkersConfidence = floatPtr(0.95)
|
||||
if _, err := newContribService(subHigh, cfg, &fakeRecorder{}).ContributeFile(context.Background(), fileHigh, ContributeOptions{Auto: true}); err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(subHigh.submitted) != 1 || subHigh.submitted[0].Segment != MarkerKindIntro {
|
||||
t.Errorf("above-threshold scanner intro should submit, got %+v", subHigh.submitted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSkipsDuplicate(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, &fakeRecorder{already: true})
|
||||
|
||||
outcomes, _ := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if len(sub.submitted) != 0 {
|
||||
t.Errorf("duplicate must not submit, got %d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusSkipped {
|
||||
t.Errorf("expected skipped outcome, got %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSkipsWhenProviderRequiredIDMissing(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb", required: []string{ExternalIDKeyTMDB}}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TvdbID: "777", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
svc := NewContributionService(reg, resolver, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec, nil)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 0 || len(rec.recorded) != 0 {
|
||||
t.Fatalf("tmdb-missing marker should be skipped before submit/record, submitted=%d recorded=%d", len(sub.submitted), len(rec.recorded))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusSkipped || outcomes[0].Reason != "tmdb id required" {
|
||||
t.Fatalf("outcomes = %+v, want skipped tmdb id required", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeAllowsTVDBOnlyWhenProviderDoesNotRequireTMDB(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "plugin:1:markers"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TvdbID: "777", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
svc := NewContributionService(reg, resolver, fakeConfig{"plugin:1:markers": {Provider: "plugin:1:markers", ContributeEnabled: true}}, rec, nil)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 || sub.submitted[0].ExternalIDs[ExternalIDKeyTVDB] != "777" {
|
||||
t.Fatalf("tvdb-only marker should submit to provider without tmdb requirement, submitted=%+v", sub.submitted)
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != SubmissionStatusPending {
|
||||
t.Fatalf("outcomes = %+v, want pending", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeStopsOnRetryAfterError(t *testing.T) {
|
||||
sub := &fakeSubmitter{
|
||||
id: "introdb",
|
||||
err: &RetryAfterError{Provider: "introdb", RetryAfter: 45 * time.Second, Message: "usage limited"},
|
||||
}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
file.CreditsStart, file.CreditsEnd = floatPtr(1700), floatPtr(1800)
|
||||
file.CreditsMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 {
|
||||
t.Fatalf("expected contribution loop to stop after first rate limit, submitted=%d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusRateLimited || outcomes[0].RetryAfter != 45*time.Second {
|
||||
t.Fatalf("outcomes = %+v, want one rate_limited retry-after outcome", outcomes)
|
||||
}
|
||||
if len(rec.recorded) != 1 || rec.recorded[0].Status != OutcomeStatusError {
|
||||
t.Fatalf("recorded = %+v, want one error audit row", rec.recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeDisabledProviderNoop(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
// contribute_enabled defaults false
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb"}}, &fakeRecorder{})
|
||||
|
||||
outcomes, _ := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if len(sub.submitted) != 0 || len(outcomes) != 0 {
|
||||
t.Errorf("disabled provider must be a no-op, submitted=%d outcomes=%d", len(sub.submitted), len(outcomes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentHashStableAndSensitive(t *testing.T) {
|
||||
s, e, d := int64(0), int64(60000), int64(1800000)
|
||||
target := contributionTargetParts(ExternalIDs{Kind: ItemKindEpisode, TmdbID: "1234", SeasonNumber: 1, EpisodeNumber: 3})
|
||||
h1 := ContentHash("intro", &s, &e, &d, target...)
|
||||
if h1 != ContentHash("intro", &s, &e, &d, target...) {
|
||||
t.Error("hash not stable for identical input")
|
||||
}
|
||||
e2 := int64(61000)
|
||||
if ContentHash("intro", &s, &e2, &d, target...) == h1 {
|
||||
t.Error("changed end should change hash")
|
||||
}
|
||||
rematched := contributionTargetParts(ExternalIDs{Kind: ItemKindEpisode, TmdbID: "9999", SeasonNumber: 1, EpisodeNumber: 3})
|
||||
if ContentHash("intro", &s, &e, &d, rematched...) == h1 {
|
||||
t.Error("changed resolved target should change hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ContributionRow is one submission audit record from marker_contributions.
|
||||
type ContributionRow struct {
|
||||
ID string
|
||||
MediaFileID int
|
||||
Provider string
|
||||
SegmentKind string
|
||||
Source string
|
||||
SubmittedStartMs *int64
|
||||
SubmittedEndMs *int64
|
||||
VideoDurationMs *int64
|
||||
ContentHash string
|
||||
SubmissionID *string
|
||||
Status string
|
||||
HTTPStatus *int
|
||||
Error *string
|
||||
SubmittedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// ContributionStore persists marker contribution attempts for idempotency and
|
||||
// audit.
|
||||
type ContributionStore struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// NewContributionStore constructs a store backed by the supplied pool.
|
||||
func NewContributionStore(pool *pgxpool.Pool) *ContributionStore {
|
||||
return &ContributionStore{pool: pool}
|
||||
}
|
||||
|
||||
// ContentHash is a stable hash over the contributed value and resolved target.
|
||||
// Identical submissions hash identically (never resubmitted); a marker
|
||||
// correction or rematch to a different provider identity hashes differently.
|
||||
func ContentHash(segmentKind string, startMs, endMs, durationMs *int64, targetParts ...string) string {
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "%s|%s|%s|%s", segmentKind, ptrIntStr(startMs), ptrIntStr(endMs), ptrIntStr(durationMs))
|
||||
for _, part := range targetParts {
|
||||
fmt.Fprintf(h, "|%s", part)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:32]
|
||||
}
|
||||
|
||||
func ptrIntStr(v *int64) string {
|
||||
if v == nil {
|
||||
return "null"
|
||||
}
|
||||
return strconv.FormatInt(*v, 10)
|
||||
}
|
||||
|
||||
// AlreadySubmitted reports whether a non-error contribution with this value-hash
|
||||
// already exists for the file+provider+segment. Errors are excluded so failed
|
||||
// attempts can be retried.
|
||||
func (s *ContributionStore) AlreadySubmitted(ctx context.Context, fileID int, provider, segmentKind, contentHash string) (bool, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return false, nil
|
||||
}
|
||||
var exists bool
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM marker_contributions
|
||||
WHERE media_file_id = $1 AND provider = $2 AND segment_kind = $3
|
||||
AND content_hash = $4 AND status <> 'error'
|
||||
)`, fileID, provider, segmentKind, contentHash).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check marker contribution: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// Record upserts a contribution row keyed by (file, provider, segment, hash).
|
||||
func (s *ContributionStore) Record(ctx context.Context, row ContributionRow) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("contribution store unavailable")
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_contributions (
|
||||
media_file_id, provider, segment_kind, source,
|
||||
submitted_start_ms, submitted_end_ms, video_duration_ms,
|
||||
content_hash, submission_id, status, http_status, error, updated_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, now())
|
||||
ON CONFLICT (media_file_id, provider, segment_kind, content_hash) DO UPDATE SET
|
||||
source = EXCLUDED.source,
|
||||
submitted_start_ms = EXCLUDED.submitted_start_ms,
|
||||
submitted_end_ms = EXCLUDED.submitted_end_ms,
|
||||
video_duration_ms = EXCLUDED.video_duration_ms,
|
||||
submission_id = EXCLUDED.submission_id,
|
||||
status = EXCLUDED.status,
|
||||
http_status = EXCLUDED.http_status,
|
||||
error = EXCLUDED.error,
|
||||
updated_at = now()`,
|
||||
row.MediaFileID, row.Provider, row.SegmentKind, row.Source,
|
||||
row.SubmittedStartMs, row.SubmittedEndMs, row.VideoDurationMs,
|
||||
row.ContentHash, row.SubmissionID, row.Status, row.HTTPStatus, row.Error,
|
||||
); err != nil {
|
||||
return fmt.Errorf("record marker contribution: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CandidateLocalIntroFiles returns ids of episode files carrying a local
|
||||
// (scanner) intro marker at or above minConfidence — the auto-contribution
|
||||
// candidates. Iterated by keyset (id > afterID) for stable paging.
|
||||
func (s *ContributionStore) CandidateLocalIntroFiles(ctx context.Context, minConfidence float64, afterID, limit int) ([]int, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id FROM media_files
|
||||
WHERE episode_id IS NOT NULL
|
||||
AND intro_markers_source = $1
|
||||
AND intro_start IS NOT NULL AND intro_end IS NOT NULL
|
||||
AND COALESCE(intro_markers_confidence, 0) >= $2
|
||||
AND id > $3
|
||||
ORDER BY id
|
||||
LIMIT $4`, models.MarkerSourceScanner, minConfidence, afterID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query contribution candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []int
|
||||
for rows.Next() {
|
||||
var id int
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scan candidate id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// ListByFile returns the contribution history for a file, newest first.
|
||||
func (s *ContributionStore) ListByFile(ctx context.Context, fileID int) ([]ContributionRow, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, media_file_id, provider, segment_kind, source,
|
||||
submitted_start_ms, submitted_end_ms, video_duration_ms,
|
||||
content_hash, submission_id, status, http_status, error,
|
||||
submitted_at, updated_at
|
||||
FROM marker_contributions WHERE media_file_id = $1
|
||||
ORDER BY updated_at DESC`, fileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list marker contributions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []ContributionRow
|
||||
for rows.Next() {
|
||||
var r ContributionRow
|
||||
if err := rows.Scan(
|
||||
&r.ID, &r.MediaFileID, &r.Provider, &r.SegmentKind, &r.Source,
|
||||
&r.SubmittedStartMs, &r.SubmittedEndMs, &r.VideoDurationMs,
|
||||
&r.ContentHash, &r.SubmissionID, &r.Status, &r.HTTPStatus, &r.Error,
|
||||
&r.SubmittedAt, &r.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan marker contribution: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRetries = 3
|
||||
maxResponseBody = 1 << 20 // 1 MB
|
||||
defaultTimeout = 15 * time.Second
|
||||
defaultCacheTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// Client is an HTTP client for the TheIntroDB /v3/media endpoint. Each
|
||||
// instance has its own rate limiter and response cache; concurrent fetches
|
||||
// for the same lookup key collapse to a single HTTP round trip via the cache.
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
mu sync.RWMutex
|
||||
apiKey string
|
||||
baseURL string
|
||||
limiter *rate.Limiter
|
||||
cache *cache.TTLCache[*mediaResponse]
|
||||
cacheTTL time.Duration
|
||||
}
|
||||
|
||||
// NewClient builds a Client with the canonical rate limit and cache TTL.
|
||||
// The apiKey may be empty — TheIntroDB serves read traffic without a key,
|
||||
// the key only gates access to the caller's own pending submissions.
|
||||
func NewClient(apiKey string) *Client {
|
||||
return &Client{
|
||||
httpClient: &http.Client{Timeout: defaultTimeout},
|
||||
apiKey: strings.TrimSpace(apiKey),
|
||||
baseURL: DefaultBaseURL,
|
||||
// TheIntroDB documents 30 requests / 10 seconds per IP. We stay
|
||||
// conservatively below that: 2 req/s sustained, burst 5.
|
||||
limiter: rate.NewLimiter(2, 5),
|
||||
cache: cache.NewTTLCache[*mediaResponse](),
|
||||
cacheTTL: defaultCacheTTL,
|
||||
}
|
||||
}
|
||||
|
||||
// SetBaseURL overrides the API base URL (used by tests).
|
||||
func (c *Client) SetBaseURL(u string) {
|
||||
c.mu.Lock()
|
||||
c.baseURL = u
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetAPIKey rotates the bearer token in-place. Safe to call concurrently
|
||||
// with in-flight requests; subsequent requests use the new key.
|
||||
func (c *Client) SetAPIKey(apiKey string) {
|
||||
c.mu.Lock()
|
||||
c.apiKey = strings.TrimSpace(apiKey)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Close releases the background sweeper goroutine inside the response cache.
|
||||
func (c *Client) Close() {
|
||||
if c.cache != nil {
|
||||
c.cache.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
if tmdbID != "" {
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
} else {
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
if tmdbID != "" {
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
} else {
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, imdbID, durationMS))
|
||||
}
|
||||
|
||||
func (c *Client) fetch(ctx context.Context, q url.Values, key string) (*mediaResponse, error) {
|
||||
if cached, ok := c.cache.Get(key); ok {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.mu.RLock()
|
||||
baseURL := c.baseURL
|
||||
apiKey := c.apiKey
|
||||
c.mu.RUnlock()
|
||||
|
||||
reqURL := baseURL + "/media?" + q.Encode()
|
||||
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("introdb: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("User-Agent", "Silo-Server/markers")
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("introdb: request failed: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
resp.Body.Close()
|
||||
// Cache negatives too so the next playback start doesn't trigger
|
||||
// another fetch for known-empty content.
|
||||
c.cache.Set(key, nil, c.cacheTTL)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
resp.Body.Close()
|
||||
if attempt < maxRetries {
|
||||
backoff := retryAfterOrDefault(resp, attempt)
|
||||
select {
|
||||
case <-time.After(backoff):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: rate limited after %d retries", maxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 {
|
||||
resp.Body.Close()
|
||||
if attempt < maxRetries {
|
||||
backoff := time.Duration(1<<attempt) * time.Second
|
||||
select {
|
||||
case <-time.After(backoff):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: server error %d after %d retries", resp.StatusCode, maxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseBody))
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("introdb: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var out mediaResponse
|
||||
decodeErr := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(&out)
|
||||
resp.Body.Close()
|
||||
if decodeErr != nil {
|
||||
return nil, fmt.Errorf("introdb: decode response: %w", decodeErr)
|
||||
}
|
||||
c.cache.Set(key, &out, c.cacheTTL)
|
||||
return &out, nil
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: max retries exceeded")
|
||||
}
|
||||
|
||||
func retryAfterOrDefault(resp *http.Response, attempt int) time.Duration {
|
||||
if val := resp.Header.Get("Retry-After"); val != "" {
|
||||
if secs, err := strconv.Atoi(val); err == nil && secs > 0 {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
}
|
||||
return time.Duration(1<<attempt) * time.Second
|
||||
}
|
||||
|
||||
func cacheKeyEpisode(tmdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
if tmdbID != "" {
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
}
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, imdbID string, durationMS int64) string {
|
||||
if tmdbID != "" {
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
}
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// Provider implements markers.Provider against the TheIntroDB API.
|
||||
// Movies are looked up by TMDB or IMDB ID; episodes additionally need a
|
||||
// season and episode number. The provider returns at most one Marker per
|
||||
// segment kind — when TheIntroDB has multiple candidate ranges (multiple
|
||||
// release versions), we pick the first usable one.
|
||||
type Provider struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewProvider constructs a Provider backed by the supplied client. Pass an
|
||||
// already-configured *Client (from NewClient) so callers control the API
|
||||
// key, base URL, and lifecycle.
|
||||
func NewProvider(client *Client) *Provider {
|
||||
return &Provider{client: client}
|
||||
}
|
||||
|
||||
// ID satisfies markers.Provider; it returns the canonical provider tag
|
||||
// stored in *_markers_provider columns and emitted in telemetry.
|
||||
func (p *Provider) ID() string { return ProviderID }
|
||||
|
||||
// FetchMarkers issues a single GET /v3/media call and converts the
|
||||
// response into a markers.Result. A nil Provider (or one with no usable
|
||||
// IDs) returns an empty result rather than an error so callers can
|
||||
// chain providers via the Registry.
|
||||
func (p *Provider) FetchMarkers(ctx context.Context, req markers.Request) (markers.Result, error) {
|
||||
if p == nil || p.client == nil {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
durationMS := int64(req.Duration / time.Millisecond)
|
||||
|
||||
var resp *mediaResponse
|
||||
var err error
|
||||
switch req.Kind {
|
||||
case markers.ItemKindEpisode:
|
||||
if req.SeasonNumber <= 0 || req.EpisodeNumber <= 0 {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
case markers.ItemKindMovie:
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, imdbID, durationMS)
|
||||
default:
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return markers.Result{}, err
|
||||
}
|
||||
if resp == nil {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
result := markers.Result{
|
||||
ProviderID: ProviderID,
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Algorithm: Algorithm,
|
||||
}
|
||||
if m, ok := pickMarker(resp.Intro, markers.MarkerKindIntro, req.Duration, true); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Credits, markers.MarkerKindCredits, req.Duration, false); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Recap, markers.MarkerKindRecap, req.Duration, true); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Preview, markers.MarkerKindPreview, req.Duration, false); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// pickMarker selects the first usable segment from a TheIntroDB response
|
||||
// array. `requireEnd` is true for segments where the end timestamp is the
|
||||
// load-bearing field (intro, recap) — they're allowed to start at 0 if
|
||||
// `start_ms` is omitted. For trailing segments (credits, preview) the
|
||||
// start is required but the end defaults to the file duration.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
return markers.Marker{Kind: kind, Start: start, End: end, Confidence: 0.9}, true
|
||||
}
|
||||
return markers.Marker{}, false
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Package introdb implements a markers.Provider against the public
|
||||
// TheIntroDB API (https://theintrodb.org). The provider is read-only:
|
||||
// it fetches intro/recap/credits/preview timestamps for episodes and
|
||||
// movies. Submissions are intentionally not supported.
|
||||
package introdb
|
||||
|
||||
// ProviderID is the canonical identifier stored in
|
||||
// media_files.*_markers_provider for markers sourced from TheIntroDB.
|
||||
const ProviderID = "introdb"
|
||||
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// DefaultBaseURL is the production TheIntroDB v3 endpoint. Overridable
|
||||
// in tests via Client.SetBaseURL.
|
||||
const DefaultBaseURL = "https://api.theintrodb.org/v3"
|
||||
|
||||
// mediaResponse mirrors the JSON shape returned by GET /v3/media.
|
||||
// Each segment kind is an array of zero or more entries; absent fields
|
||||
// are decoded as empty slices via Go's zero-value semantics.
|
||||
type mediaResponse struct {
|
||||
TmdbID int `json:"tmdb_id"`
|
||||
Type string `json:"type"`
|
||||
Season *int `json:"season,omitempty"`
|
||||
Episode *int `json:"episode,omitempty"`
|
||||
Intro []segmentTimestamps `json:"intro,omitempty"`
|
||||
Recap []segmentTimestamps `json:"recap,omitempty"`
|
||||
Credits []segmentTimestamps `json:"credits,omitempty"`
|
||||
Preview []segmentTimestamps `json:"preview,omitempty"`
|
||||
}
|
||||
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end).
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/pluginhost"
|
||||
)
|
||||
|
||||
const pluginMarkerAlgorithm = "plugin:marker_provider.v1"
|
||||
|
||||
type pluginMarkerClient interface {
|
||||
FetchMarkers(ctx context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error)
|
||||
SubmitMarker(ctx context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error)
|
||||
GetMarkerProviderStats(ctx context.Context, req *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error)
|
||||
}
|
||||
|
||||
type pluginMarkerResolver interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error)
|
||||
}
|
||||
|
||||
type pluginMarkerClientFactory func(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error)
|
||||
|
||||
type PluginResolverAdapter struct {
|
||||
inner interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
}
|
||||
}
|
||||
|
||||
func NewPluginResolverAdapter(svc interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
}) *PluginResolverAdapter {
|
||||
if svc == nil {
|
||||
return nil
|
||||
}
|
||||
return &PluginResolverAdapter{inner: svc}
|
||||
}
|
||||
|
||||
func (a *PluginResolverAdapter) MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error) {
|
||||
return a.inner.MarkerProviderClient(ctx, installationID, capabilityID)
|
||||
}
|
||||
|
||||
type PluginProviderOptions struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
DisplayName string
|
||||
PluginID string
|
||||
RequiredExternalIDs []string
|
||||
}
|
||||
|
||||
// PluginProvider adapts a marker_provider.v1 plugin capability to Silo's
|
||||
// marker Provider/Submitter interfaces.
|
||||
type PluginProvider struct {
|
||||
installationID int
|
||||
capabilityID string
|
||||
displayName string
|
||||
pluginID string
|
||||
requiredExternalIDs []string
|
||||
clientFactory pluginMarkerClientFactory
|
||||
}
|
||||
|
||||
var _ Submitter = (*PluginProvider)(nil)
|
||||
|
||||
func NewPluginProvider(opts PluginProviderOptions, resolver pluginMarkerResolver) (*PluginProvider, error) {
|
||||
if resolver == nil {
|
||||
return nil, fmt.Errorf("plugin marker resolver is required")
|
||||
}
|
||||
return NewPluginProviderWithClientFactory(opts, resolver.MarkerProviderClient)
|
||||
}
|
||||
|
||||
func NewPluginProviderWithClientFactory(opts PluginProviderOptions, clientFactory pluginMarkerClientFactory) (*PluginProvider, error) {
|
||||
if opts.InstallationID <= 0 {
|
||||
return nil, fmt.Errorf("plugin marker installation id is required")
|
||||
}
|
||||
if strings.TrimSpace(opts.CapabilityID) == "" {
|
||||
return nil, fmt.Errorf("plugin marker capability id is required")
|
||||
}
|
||||
if clientFactory == nil {
|
||||
return nil, fmt.Errorf("plugin marker client factory is required")
|
||||
}
|
||||
displayName := strings.TrimSpace(opts.DisplayName)
|
||||
if displayName == "" {
|
||||
displayName = opts.CapabilityID
|
||||
}
|
||||
requiredIDs := normalizeExternalIDKeys(opts.RequiredExternalIDs)
|
||||
return &PluginProvider{
|
||||
installationID: opts.InstallationID,
|
||||
capabilityID: strings.TrimSpace(opts.CapabilityID),
|
||||
displayName: displayName,
|
||||
pluginID: strings.TrimSpace(opts.PluginID),
|
||||
requiredExternalIDs: requiredIDs,
|
||||
clientFactory: clientFactory,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func PluginProviderID(installationID int, capabilityID string) string {
|
||||
return fmt.Sprintf("plugin:%d:%s", installationID, strings.TrimSpace(capabilityID))
|
||||
}
|
||||
|
||||
func (p *PluginProvider) ID() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return PluginProviderID(p.installationID, p.capabilityID)
|
||||
}
|
||||
|
||||
func (p *PluginProvider) ProviderDescription() ProviderDescriptor {
|
||||
if p == nil {
|
||||
return ProviderDescriptor{}
|
||||
}
|
||||
return ProviderDescriptor{
|
||||
ID: p.ID(),
|
||||
DisplayName: p.displayName,
|
||||
SourceType: ProviderSourcePlugin,
|
||||
PluginID: p.pluginID,
|
||||
PluginInstallationID: p.installationID,
|
||||
CapabilityID: p.capabilityID,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PluginProvider) SubmissionRequirements() SubmissionRequirements {
|
||||
if p == nil || len(p.requiredExternalIDs) == 0 {
|
||||
return SubmissionRequirements{}
|
||||
}
|
||||
return SubmissionRequirements{RequiredExternalIDs: append([]string(nil), p.requiredExternalIDs...)}
|
||||
}
|
||||
|
||||
func (p *PluginProvider) FetchMarkers(ctx context.Context, req Request) (Result, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
response, err := client.FetchMarkers(ctx, &pluginv1.FetchMarkersRequest{
|
||||
ItemType: itemTypeName(req.Kind),
|
||||
ExternalIds: externalIDsProto(req.ExternalIDs),
|
||||
SeasonNumber: int32(req.SeasonNumber),
|
||||
EpisodeNumber: int32(req.EpisodeNumber),
|
||||
DurationSeconds: req.Duration.Seconds(),
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
result := Result{
|
||||
SourceClass: models.MarkerSourcePlugin,
|
||||
ProviderID: p.ID(),
|
||||
Algorithm: pluginMarkerAlgorithm,
|
||||
}
|
||||
for _, segment := range response.GetMarkers() {
|
||||
marker, ok := markerFromPluginSegment(segment, req.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
marker.SourceClass = models.MarkerSourcePlugin
|
||||
marker.ProviderID = p.ID()
|
||||
if marker.Algorithm == "" {
|
||||
marker.Algorithm = pluginMarkerAlgorithm
|
||||
}
|
||||
result.Markers = append(result.Markers, marker)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *PluginProvider) SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return SubmissionResult{}, fmt.Errorf("plugin marker provider not configured")
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return SubmissionResult{}, err
|
||||
}
|
||||
body := &pluginv1.SubmitMarkerRequest{
|
||||
ItemType: itemTypeName(req.Kind),
|
||||
ExternalIds: externalIDsProto(req.ExternalIDs),
|
||||
SeasonNumber: int32(req.SeasonNumber),
|
||||
EpisodeNumber: int32(req.EpisodeNumber),
|
||||
Segment: markerKindName(req.Segment),
|
||||
DurationSeconds: req.Duration.Seconds(),
|
||||
}
|
||||
if req.Start != nil {
|
||||
start := req.Start.Seconds()
|
||||
body.StartSeconds = &start
|
||||
}
|
||||
if req.End != nil {
|
||||
end := req.End.Seconds()
|
||||
body.EndSeconds = &end
|
||||
}
|
||||
response, err := client.SubmitMarker(ctx, body)
|
||||
if err != nil {
|
||||
return SubmissionResult{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return SubmissionResult{Status: SubmissionStatusPending}, nil
|
||||
}
|
||||
status := strings.TrimSpace(response.GetStatus())
|
||||
if status == "" {
|
||||
status = SubmissionStatusPending
|
||||
}
|
||||
return SubmissionResult{ID: response.GetSubmissionId(), Status: status, Weight: response.GetWeight()}, nil
|
||||
}
|
||||
|
||||
func (p *PluginProvider) FetchUserStats(ctx context.Context) (UserStats, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return UserStats{}, fmt.Errorf("plugin marker provider not configured")
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return UserStats{}, err
|
||||
}
|
||||
response, err := client.GetMarkerProviderStats(ctx, &pluginv1.GetMarkerProviderStatsRequest{})
|
||||
if err != nil {
|
||||
return UserStats{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return UserStats{}, nil
|
||||
}
|
||||
return UserStats{
|
||||
Total: int(response.GetTotal()),
|
||||
Accepted: int(response.GetAccepted()),
|
||||
Pending: int(response.GetPending()),
|
||||
Rejected: int(response.GetRejected()),
|
||||
AcceptanceRate: response.GetAcceptanceRate(),
|
||||
CurrentStreak: int(response.GetCurrentStreak()),
|
||||
BestStreak: int(response.GetBestStreak()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func markerFromPluginSegment(segment *pluginv1.MarkerSegment, duration time.Duration) (Marker, bool) {
|
||||
if segment == nil {
|
||||
return Marker{}, false
|
||||
}
|
||||
kind, ok := markerKindFromName(segment.GetSegment())
|
||||
if !ok {
|
||||
return Marker{}, false
|
||||
}
|
||||
start := time.Duration(0)
|
||||
end := duration
|
||||
if segment.StartSeconds != nil {
|
||||
start = secondsDuration(segment.GetStartSeconds())
|
||||
}
|
||||
if segment.EndSeconds != nil {
|
||||
end = secondsDuration(segment.GetEndSeconds())
|
||||
}
|
||||
if start < 0 {
|
||||
return Marker{}, false
|
||||
}
|
||||
if duration > 0 && (start >= duration || end > duration) {
|
||||
return Marker{}, false
|
||||
}
|
||||
if end <= start {
|
||||
return Marker{}, false
|
||||
}
|
||||
return Marker{
|
||||
Kind: kind,
|
||||
Start: start,
|
||||
End: end,
|
||||
Confidence: segment.GetConfidence(),
|
||||
SubmissionCount: int(segment.GetSubmissionCount()),
|
||||
Algorithm: strings.TrimSpace(segment.GetAlgorithm()),
|
||||
}, true
|
||||
}
|
||||
|
||||
func externalIDsProto(ids map[string]string) *pluginv1.MarkerExternalIDs {
|
||||
if len(ids) == 0 {
|
||||
return &pluginv1.MarkerExternalIDs{}
|
||||
}
|
||||
providerIDs := make(map[string]any, len(ids))
|
||||
for key, value := range ids {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
providerIDs[key] = value
|
||||
}
|
||||
}
|
||||
structIDs, _ := structpb.NewStruct(providerIDs)
|
||||
return &pluginv1.MarkerExternalIDs{
|
||||
TmdbId: strings.TrimSpace(ids[ExternalIDKeyTMDB]),
|
||||
ImdbId: strings.TrimSpace(ids[ExternalIDKeyIMDB]),
|
||||
TvdbId: strings.TrimSpace(ids[ExternalIDKeyTVDB]),
|
||||
ProviderIds: structIDs,
|
||||
}
|
||||
}
|
||||
|
||||
func itemTypeName(kind ItemKind) string {
|
||||
switch kind {
|
||||
case ItemKindEpisode:
|
||||
return "episode"
|
||||
case ItemKindMovie:
|
||||
return "movie"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func markerKindName(kind MarkerKind) string {
|
||||
switch kind {
|
||||
case MarkerKindIntro:
|
||||
return "intro"
|
||||
case MarkerKindCredits:
|
||||
return "credits"
|
||||
case MarkerKindRecap:
|
||||
return "recap"
|
||||
case MarkerKindPreview:
|
||||
return "preview"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func markerKindFromName(name string) (MarkerKind, bool) {
|
||||
switch strings.ToLower(strings.TrimSpace(name)) {
|
||||
case "intro":
|
||||
return MarkerKindIntro, true
|
||||
case "credits":
|
||||
return MarkerKindCredits, true
|
||||
case "recap":
|
||||
return MarkerKindRecap, true
|
||||
case "preview":
|
||||
return MarkerKindPreview, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func secondsDuration(seconds float64) time.Duration {
|
||||
return time.Duration(seconds * float64(time.Second))
|
||||
}
|
||||
|
||||
func normalizeExternalIDKeys(keys []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case ExternalIDKeyTMDB, ExternalIDKeyIMDB, ExternalIDKeyTVDB:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pluginProviderError(providerID string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
st, ok := status.FromError(err)
|
||||
if !ok || st.Code() != codes.ResourceExhausted {
|
||||
return err
|
||||
}
|
||||
retryAfter := time.Duration(0)
|
||||
for _, detail := range st.Details() {
|
||||
if retry, ok := detail.(*errdetails.RetryInfo); ok && retry.GetRetryDelay() != nil {
|
||||
retryAfter = retry.GetRetryDelay().AsDuration()
|
||||
break
|
||||
}
|
||||
}
|
||||
return &RetryAfterError{Provider: providerID, RetryAfter: retryAfter, Message: err.Error()}
|
||||
}
|
||||
|
||||
func PluginRequiredExternalIDsFromMetadata(metadata map[string]any) []string {
|
||||
raw, ok := metadata["required_external_ids"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case []string:
|
||||
return normalizeExternalIDKeys(typed)
|
||||
case []any:
|
||||
keys := make([]string, 0, len(typed))
|
||||
for _, value := range typed {
|
||||
if s, ok := value.(string); ok {
|
||||
keys = append(keys, s)
|
||||
}
|
||||
}
|
||||
return normalizeExternalIDKeys(keys)
|
||||
case string:
|
||||
if strings.TrimSpace(typed) == "" {
|
||||
return nil
|
||||
}
|
||||
return normalizeExternalIDKeys(strings.Split(typed, ","))
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func PluginDefaultFetchPriorityFromMetadata(metadata map[string]any) (int, bool) {
|
||||
raw, ok := metadata["default_fetch_priority"]
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case int:
|
||||
return typed, true
|
||||
case int32:
|
||||
return int(typed), true
|
||||
case int64:
|
||||
return int(typed), true
|
||||
case float64:
|
||||
return int(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(strings.TrimSpace(typed))
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakePluginMarkerClient struct {
|
||||
fetchResp *pluginv1.FetchMarkersResponse
|
||||
fetchReq *pluginv1.FetchMarkersRequest
|
||||
submitReq *pluginv1.SubmitMarkerRequest
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) FetchMarkers(_ context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error) {
|
||||
f.fetchReq = req
|
||||
return f.fetchResp, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) SubmitMarker(_ context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error) {
|
||||
f.submitReq = req
|
||||
return &pluginv1.SubmitMarkerResponse{SubmissionId: "sub1", Status: SubmissionStatusPending, Weight: 2}, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) GetMarkerProviderStats(context.Context, *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error) {
|
||||
return &pluginv1.MarkerProviderStatsResponse{Total: 3, Accepted: 2, Pending: 1, AcceptanceRate: 0.66}, nil
|
||||
}
|
||||
|
||||
func TestPluginProviderFetchMapsAllSegments(t *testing.T) {
|
||||
start10, end60 := 10.0, 60.0
|
||||
creditsStart := 1700.0
|
||||
previewStart := 1750.0
|
||||
client := &fakePluginMarkerClient{fetchResp: &pluginv1.FetchMarkersResponse{Markers: []*pluginv1.MarkerSegment{
|
||||
{Segment: "intro", StartSeconds: &start10, EndSeconds: &end60, Confidence: 0.8, SubmissionCount: 2, Algorithm: "intro:v1"},
|
||||
{Segment: "credits", StartSeconds: &creditsStart, Confidence: 0.9, SubmissionCount: 3},
|
||||
{Segment: "recap", EndSeconds: &start10, Confidence: 0.7},
|
||||
{Segment: "preview", StartSeconds: &previewStart, Confidence: 0.6},
|
||||
}}}
|
||||
provider, err := NewPluginProviderWithClientFactory(PluginProviderOptions{
|
||||
InstallationID: 12,
|
||||
CapabilityID: "markers",
|
||||
DisplayName: "Markers",
|
||||
PluginID: "silo.markers",
|
||||
}, func(context.Context, int, string) (pluginMarkerClient, error) {
|
||||
return client, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPluginProviderWithClientFactory: %v", err)
|
||||
}
|
||||
|
||||
res, err := provider.FetchMarkers(context.Background(), Request{
|
||||
Kind: ItemKindEpisode,
|
||||
ExternalIDs: map[string]string{ExternalIDKeyTVDB: "777"},
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 2,
|
||||
Duration: 1800 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if client.fetchReq.GetItemType() != "episode" || client.fetchReq.GetExternalIds().GetTvdbId() != "777" {
|
||||
t.Fatalf("fetch request = %+v", client.fetchReq)
|
||||
}
|
||||
if res.SourceClass != models.MarkerSourcePlugin || res.ProviderID != "plugin:12:markers" {
|
||||
t.Fatalf("result provenance = source %q provider %q", res.SourceClass, res.ProviderID)
|
||||
}
|
||||
byKind := map[MarkerKind]Marker{}
|
||||
for _, marker := range res.Markers {
|
||||
byKind[marker.Kind] = marker
|
||||
if marker.SourceClass != models.MarkerSourcePlugin || marker.ProviderID != "plugin:12:markers" {
|
||||
t.Fatalf("marker provenance = %+v", marker)
|
||||
}
|
||||
}
|
||||
if len(byKind) != 4 {
|
||||
t.Fatalf("mapped %d markers, want 4: %+v", len(byKind), res.Markers)
|
||||
}
|
||||
if got := byKind[MarkerKindCredits]; got.End != 1800*time.Second {
|
||||
t.Fatalf("credits end = %s, want duration default", got.End)
|
||||
}
|
||||
if got := byKind[MarkerKindRecap]; got.Start != 0 {
|
||||
t.Fatalf("recap start = %s, want zero default", got.Start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginProviderRejectsOutOfBoundsSegments(t *testing.T) {
|
||||
negativeStart := -1.0
|
||||
start10, end61 := 10.0, 61.0
|
||||
validStart := 50.0
|
||||
duration := time.Minute
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
segment *pluginv1.MarkerSegment
|
||||
wantOK bool
|
||||
wantEnd time.Duration
|
||||
}{
|
||||
{
|
||||
name: "negative start",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "intro", StartSeconds: &negativeStart},
|
||||
},
|
||||
{
|
||||
name: "end past duration",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "intro", StartSeconds: &start10, EndSeconds: &end61},
|
||||
},
|
||||
{
|
||||
name: "default end uses duration",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "credits", StartSeconds: &validStart},
|
||||
wantOK: true,
|
||||
wantEnd: duration,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
marker, ok := markerFromPluginSegment(tt.segment, duration)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
|
||||
}
|
||||
if tt.wantOK && marker.End != tt.wantEnd {
|
||||
t.Fatalf("end = %s, want %s", marker.End, tt.wantEnd)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginProviderSubmitMapsRequest(t *testing.T) {
|
||||
client := &fakePluginMarkerClient{}
|
||||
provider, err := NewPluginProviderWithClientFactory(PluginProviderOptions{
|
||||
InstallationID: 12,
|
||||
CapabilityID: "markers",
|
||||
}, func(context.Context, int, string) (pluginMarkerClient, error) {
|
||||
return client, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPluginProviderWithClientFactory: %v", err)
|
||||
}
|
||||
start, end := 5*time.Second, 30*time.Second
|
||||
result, err := provider.SubmitMarker(context.Background(), SubmissionRequest{
|
||||
Kind: ItemKindMovie,
|
||||
ExternalIDs: map[string]string{ExternalIDKeyIMDB: "tt1"},
|
||||
Segment: MarkerKindIntro,
|
||||
Start: &start,
|
||||
End: &end,
|
||||
Duration: 90 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SubmitMarker: %v", err)
|
||||
}
|
||||
if result.ID != "sub1" || result.Status != SubmissionStatusPending || result.Weight != 2 {
|
||||
t.Fatalf("submit result = %+v", result)
|
||||
}
|
||||
if client.submitReq.GetItemType() != "movie" || client.submitReq.GetExternalIds().GetImdbId() != "tt1" {
|
||||
t.Fatalf("submit request identity = %+v", client.submitReq)
|
||||
}
|
||||
if client.submitReq.GetSegment() != "intro" || client.submitReq.GetStartSeconds() != 5 || client.submitReq.GetEndSeconds() != 30 {
|
||||
t.Fatalf("submit request segment = %+v", client.submitReq)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ProviderConfig is the per-provider behavior row from marker_provider_config.
|
||||
// fetch_* control multi-source read dispatch; contribute_* gate submission and
|
||||
// default off so contribution is opt-in per provider.
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
const providerConfigColumns = `provider, fetch_enabled, fetch_priority, contribute_enabled, contribute_auto_local, contribute_min_confidence`
|
||||
|
||||
// ProviderConfigStore is a cached read/write facade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; call Reload at startup and after a
|
||||
// settings-changed event. Update writes through and refreshes the snapshot.
|
||||
type ProviderConfigStore struct {
|
||||
pool *pgxpool.Pool
|
||||
mu sync.RWMutex
|
||||
cache map[string]ProviderConfig
|
||||
}
|
||||
|
||||
// NewProviderConfigStore constructs a store backed by the supplied pool.
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore {
|
||||
return &ProviderConfigStore{pool: pool, cache: map[string]ProviderConfig{}}
|
||||
}
|
||||
|
||||
// Reload replaces the in-memory snapshot from the database.
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `SELECT `+providerConfigColumns+` FROM marker_provider_config`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load marker provider config: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
next := make(map[string]ProviderConfig)
|
||||
for rows.Next() {
|
||||
var c ProviderConfig
|
||||
if err := rows.Scan(
|
||||
&c.Provider,
|
||||
&c.FetchEnabled,
|
||||
&c.FetchPriority,
|
||||
&c.ContributeEnabled,
|
||||
&c.ContributeAutoLocal,
|
||||
&c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("scan marker provider config: %w", err)
|
||||
}
|
||||
next[c.Provider] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate marker provider config: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.cache = next
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the config for a provider id from the snapshot.
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
c, ok := s.cache[provider]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
// List returns all configs sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig {
|
||||
s.mu.RLock()
|
||||
out := make([]ProviderConfig, 0, len(s.cache))
|
||||
for _, c := range s.cache {
|
||||
out = append(out, c)
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// EnabledForFetch returns the fetch-enabled providers in priority order.
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig {
|
||||
all := s.List()
|
||||
out := make([]ProviderConfig, 0, len(all))
|
||||
for _, c := range all {
|
||||
if c.FetchEnabled {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Update upserts a provider config row and refreshes the snapshot.
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, c ProviderConfig) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("marker provider config store unavailable")
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_provider_config (
|
||||
provider, fetch_enabled, fetch_priority,
|
||||
contribute_enabled, contribute_auto_local, contribute_min_confidence, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())
|
||||
ON CONFLICT (provider) DO UPDATE SET
|
||||
fetch_enabled = EXCLUDED.fetch_enabled,
|
||||
fetch_priority = EXCLUDED.fetch_priority,
|
||||
contribute_enabled = EXCLUDED.contribute_enabled,
|
||||
contribute_auto_local = EXCLUDED.contribute_auto_local,
|
||||
contribute_min_confidence = EXCLUDED.contribute_min_confidence,
|
||||
updated_at = now()`,
|
||||
c.Provider, c.FetchEnabled, c.FetchPriority,
|
||||
c.ContributeEnabled, c.ContributeAutoLocal, c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("update marker provider config: %w", err)
|
||||
}
|
||||
return s.Reload(ctx)
|
||||
}
|
||||
|
||||
// Ensure inserts a default provider config row if one does not already exist.
|
||||
// Existing rows are left untouched so admin choices survive plugin restarts and
|
||||
// upgrades.
|
||||
func (s *ProviderConfigStore) Ensure(ctx context.Context, c ProviderConfig) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("marker provider config store unavailable")
|
||||
}
|
||||
if _, ok := s.Get(c.Provider); ok {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_provider_config (
|
||||
provider, fetch_enabled, fetch_priority,
|
||||
contribute_enabled, contribute_auto_local, contribute_min_confidence, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())
|
||||
ON CONFLICT (provider) DO NOTHING`,
|
||||
c.Provider, c.FetchEnabled, c.FetchPriority,
|
||||
c.ContributeEnabled, c.ContributeAutoLocal, c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("ensure marker provider config: %w", err)
|
||||
}
|
||||
return s.Reload(ctx)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package markers
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestProviderConfigEnabledForFetchSortsAndFilters(t *testing.T) {
|
||||
s := &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"alpha": {Provider: "alpha", FetchEnabled: true, FetchPriority: 200},
|
||||
"bravo": {Provider: "bravo", FetchEnabled: true, FetchPriority: 100},
|
||||
"charlie": {Provider: "charlie", FetchEnabled: false, FetchPriority: 50},
|
||||
}}
|
||||
|
||||
got := s.EnabledForFetch()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("EnabledForFetch returned %d providers, want 2 (charlie disabled)", len(got))
|
||||
}
|
||||
if got[0].Provider != "bravo" || got[1].Provider != "alpha" {
|
||||
t.Errorf("order = [%s %s], want [bravo alpha] by fetch_priority", got[0].Provider, got[1].Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderConfigGet(t *testing.T) {
|
||||
s := &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"introdb": {Provider: "introdb", FetchEnabled: true, ContributeEnabled: false},
|
||||
}}
|
||||
c, ok := s.Get("introdb")
|
||||
if !ok {
|
||||
t.Fatal("expected introdb config")
|
||||
}
|
||||
if c.ContributeEnabled {
|
||||
t.Error("contribute should default off")
|
||||
}
|
||||
if _, ok := s.Get("missing"); ok {
|
||||
t.Error("unexpected config for unknown provider")
|
||||
}
|
||||
}
|
||||
+299
-7
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -32,6 +33,36 @@ type Provider interface {
|
||||
FetchMarkers(ctx context.Context, req Request) (Result, error)
|
||||
}
|
||||
|
||||
const (
|
||||
ProviderSourceBuiltIn = "built_in"
|
||||
ProviderSourcePlugin = "plugin"
|
||||
)
|
||||
|
||||
// ProviderDescriptor is optional provider metadata surfaced by admin APIs.
|
||||
type ProviderDescriptor struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
SourceType string
|
||||
PluginID string
|
||||
PluginInstallationID int
|
||||
CapabilityID string
|
||||
}
|
||||
|
||||
type DescribedProvider interface {
|
||||
ProviderDescription() ProviderDescriptor
|
||||
}
|
||||
|
||||
// SubmissionRequirements describes IDs the generic contribution path can
|
||||
// validate before calling a provider. Providers may still perform richer
|
||||
// validation inside SubmitMarker.
|
||||
type SubmissionRequirements struct {
|
||||
RequiredExternalIDs []string
|
||||
}
|
||||
|
||||
type SubmissionRequirementProvider interface {
|
||||
SubmissionRequirements() SubmissionRequirements
|
||||
}
|
||||
|
||||
type ItemKind int
|
||||
|
||||
const (
|
||||
@@ -72,15 +103,104 @@ type Result struct {
|
||||
}
|
||||
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
SourceClass string
|
||||
// ProviderID and Algorithm identify the source of this individual marker.
|
||||
// They are usually empty for a single-provider Result (the Result-level
|
||||
// SourceClass/ProviderID/Algorithm apply); FetchMerged sets them per marker
|
||||
// so a merged result records correct per-segment provenance.
|
||||
ProviderID string
|
||||
Algorithm string
|
||||
}
|
||||
|
||||
// SubmissionRequest is a single-segment contribution to an online marker
|
||||
// source. Start is nil when the segment begins at the start of the file
|
||||
// (intro/recap); End is nil when it runs to the end (credits/preview).
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start *time.Duration
|
||||
End *time.Duration
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// SubmissionStatus values returned by a Submitter.
|
||||
const (
|
||||
SubmissionStatusPending = "pending"
|
||||
SubmissionStatusAccepted = "accepted"
|
||||
SubmissionStatusRejected = "rejected"
|
||||
)
|
||||
|
||||
// SubmissionResult is the provider's acknowledgement of a submitted segment.
|
||||
type SubmissionResult struct {
|
||||
ID string
|
||||
Status string
|
||||
Weight float64
|
||||
}
|
||||
|
||||
// UserStats is a contribution-account summary used to validate a key and show
|
||||
// contribution totals in the admin UI.
|
||||
type UserStats struct {
|
||||
Total int
|
||||
Accepted int
|
||||
Pending int
|
||||
Rejected int
|
||||
AcceptanceRate float64
|
||||
CurrentStreak int
|
||||
BestStreak int
|
||||
}
|
||||
|
||||
// RetryAfterError marks provider errors that should pause contribution work
|
||||
// until RetryAfter has elapsed, such as TheIntroDB usage-limit responses.
|
||||
type RetryAfterError struct {
|
||||
Provider string
|
||||
RetryAfter time.Duration
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *RetryAfterError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
if e.Message != "" {
|
||||
return e.Message
|
||||
}
|
||||
if e.Provider != "" {
|
||||
return fmt.Sprintf("%s: retry after %s", e.Provider, e.RetryAfter)
|
||||
}
|
||||
return fmt.Sprintf("retry after %s", e.RetryAfter)
|
||||
}
|
||||
|
||||
// RetryAfter extracts a provider backoff duration from err.
|
||||
func RetryAfter(err error) (time.Duration, bool) {
|
||||
var retryErr *RetryAfterError
|
||||
if errors.As(err, &retryErr) && retryErr != nil {
|
||||
return retryErr.RetryAfter, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Submitter is an optional capability implemented by providers that accept
|
||||
// marker contributions. The contribution service type-asserts a Provider to
|
||||
// Submitter; non-contributing providers simply don't implement it.
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
providers []Provider
|
||||
logger *slog.Logger
|
||||
config *ProviderConfigStore
|
||||
}
|
||||
|
||||
func NewRegistry(logger *slog.Logger) *Registry {
|
||||
@@ -98,6 +218,8 @@ func (r *Registry) Register(provider Provider) error {
|
||||
if id == "" {
|
||||
return fmt.Errorf("marker provider ID is required")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, existing := range r.providers {
|
||||
if existing.ID() == id {
|
||||
return fmt.Errorf("marker provider %q already registered", id)
|
||||
@@ -108,7 +230,12 @@ func (r *Registry) Register(provider Provider) error {
|
||||
}
|
||||
|
||||
func (r *Registry) Providers() []Provider {
|
||||
if r == nil || len(r.providers) == 0 {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if len(r.providers) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]Provider, len(r.providers))
|
||||
@@ -116,13 +243,43 @@ func (r *Registry) Providers() []Provider {
|
||||
return out
|
||||
}
|
||||
|
||||
// SetProviders atomically replaces the registered provider set. It is used when
|
||||
// plugin lifecycle changes require rebuilding the marker-provider list.
|
||||
func (r *Registry) SetProviders(providers []Provider) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
seen := make(map[string]struct{}, len(providers))
|
||||
next := make([]Provider, 0, len(providers))
|
||||
for _, provider := range providers {
|
||||
if provider == nil {
|
||||
return fmt.Errorf("marker provider is nil")
|
||||
}
|
||||
id := strings.TrimSpace(provider.ID())
|
||||
if id == "" {
|
||||
return fmt.Errorf("marker provider ID is required")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return fmt.Errorf("marker provider %q already registered", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
next = append(next, provider)
|
||||
}
|
||||
r.providers = next
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) FetchFirstHit(ctx context.Context, req Request) (Result, bool, error) {
|
||||
if r == nil || len(r.providers) == 0 {
|
||||
providers := r.Providers()
|
||||
if len(providers) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, provider := range r.providers {
|
||||
for _, provider := range providers {
|
||||
result, err := provider.FetchMarkers(ctx, req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
@@ -144,6 +301,141 @@ func (r *Registry) FetchFirstHit(ctx context.Context, req Request) (Result, bool
|
||||
return Result{}, false, lastErr
|
||||
}
|
||||
|
||||
// UseConfigStore attaches a per-provider config store so FetchMerged consults
|
||||
// fetch_enabled / fetch_priority. Without one, all registered providers
|
||||
// participate in registration order.
|
||||
func (r *Registry) UseConfigStore(store *ProviderConfigStore) {
|
||||
if r != nil {
|
||||
r.config = store
|
||||
}
|
||||
}
|
||||
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the best candidate — ranked by provider fetch priority (lower
|
||||
// preferred), then submission count, then confidence, then provider id for
|
||||
// deterministic output. Submission count and confidence are provider-local
|
||||
// quality signals; explicit admin priority is the cross-provider winner rule.
|
||||
// The winning markers are stamped with source/provider/algorithm so the write
|
||||
// path records correct per-segment provenance. With a single enabled provider
|
||||
// this returns the same result as FetchFirstHit.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error) {
|
||||
if r == nil || len(r.Providers()) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
entries := r.fetchEntries()
|
||||
if len(entries) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
type fetched struct {
|
||||
entry fetchEntry
|
||||
result Result
|
||||
err error
|
||||
}
|
||||
out := make([]fetched, len(entries))
|
||||
var wg sync.WaitGroup
|
||||
for i, e := range entries {
|
||||
wg.Add(1)
|
||||
go func(i int, e fetchEntry) {
|
||||
defer wg.Done()
|
||||
res, err := e.provider.FetchMarkers(ctx, req)
|
||||
out[i] = fetched{entry: e, result: res, err: err}
|
||||
}(i, e)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
best := make(map[MarkerKind]mergeCandidate)
|
||||
var lastErr error
|
||||
for _, f := range out {
|
||||
if f.err != nil {
|
||||
lastErr = f.err
|
||||
r.logProviderError(f.entry.provider.ID(), req, f.err)
|
||||
continue
|
||||
}
|
||||
for _, m := range f.result.Markers {
|
||||
m.SourceClass = firstNonEmpty(m.SourceClass, f.result.SourceClass, models.MarkerSourceOnline)
|
||||
m.ProviderID = firstNonEmpty(m.ProviderID, f.result.ProviderID, f.entry.provider.ID())
|
||||
m.Algorithm = firstNonEmpty(m.Algorithm, f.result.Algorithm)
|
||||
cand := mergeCandidate{marker: m, priority: f.entry.priority}
|
||||
if cur, ok := best[m.Kind]; !ok || cand.better(cur) {
|
||||
best[m.Kind] = cand
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(best) == 0 {
|
||||
return Result{}, false, lastErr
|
||||
}
|
||||
|
||||
merged := Result{SourceClass: models.MarkerSourceOnline}
|
||||
for _, kind := range []MarkerKind{MarkerKindIntro, MarkerKindCredits, MarkerKindRecap, MarkerKindPreview} {
|
||||
if cand, ok := best[kind]; ok {
|
||||
merged.Markers = append(merged.Markers, cand.marker)
|
||||
}
|
||||
}
|
||||
return merged, true, nil
|
||||
}
|
||||
|
||||
type fetchEntry struct {
|
||||
provider Provider
|
||||
priority int
|
||||
}
|
||||
|
||||
// fetchEntries returns the providers to query with their priorities. When a
|
||||
// config store is set only fetch-enabled providers participate, ordered by
|
||||
// fetch_priority; otherwise all providers participate in registration order.
|
||||
func (r *Registry) fetchEntries() []fetchEntry {
|
||||
providers := r.Providers()
|
||||
if r.config == nil {
|
||||
entries := make([]fetchEntry, 0, len(providers))
|
||||
for i, p := range providers {
|
||||
entries = append(entries, fetchEntry{provider: p, priority: i})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
priority := make(map[string]int)
|
||||
for _, c := range r.config.EnabledForFetch() {
|
||||
priority[c.Provider] = c.FetchPriority
|
||||
}
|
||||
entries := make([]fetchEntry, 0, len(providers))
|
||||
for _, p := range providers {
|
||||
if prio, ok := priority[p.ID()]; ok {
|
||||
entries = append(entries, fetchEntry{provider: p, priority: prio})
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
type mergeCandidate struct {
|
||||
marker Marker
|
||||
priority int
|
||||
}
|
||||
|
||||
// better reports whether a should win over b for the same segment kind: more
|
||||
// preferred fetch priority, then more submissions, then higher confidence, then
|
||||
// deterministic provider id ordering.
|
||||
func (a mergeCandidate) better(b mergeCandidate) bool {
|
||||
if a.priority != b.priority {
|
||||
return a.priority < b.priority
|
||||
}
|
||||
if a.marker.SubmissionCount != b.marker.SubmissionCount {
|
||||
return a.marker.SubmissionCount > b.marker.SubmissionCount
|
||||
}
|
||||
if a.marker.Confidence != b.marker.Confidence {
|
||||
return a.marker.Confidence > b.marker.Confidence
|
||||
}
|
||||
return a.marker.ProviderID < b.marker.ProviderID
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r *Registry) logProviderError(providerID string, req Request, err error) {
|
||||
logger := slog.Default()
|
||||
if r != nil && r.logger != nil {
|
||||
|
||||
@@ -84,6 +84,148 @@ func TestRegistryFetchFirstHitReturnsLastProviderErrorAsMiss(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedKeepsBestPerSegmentByProviderPriority(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Algorithm: "a:v1", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.7, SubmissionCount: 1},
|
||||
{Kind: MarkerKindCredits, Start: 100 * time.Second, End: 110 * time.Second, Confidence: 0.5, SubmissionCount: 1},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Algorithm: "b:v1", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.9, SubmissionCount: 500},
|
||||
{Kind: MarkerKindCredits, Start: 105 * time.Second, End: 120 * time.Second, Confidence: 0.6, SubmissionCount: 400},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
if err := registry.Register(a); err != nil {
|
||||
t.Fatalf("register a: %v", err)
|
||||
}
|
||||
if err := registry.Register(b); err != nil {
|
||||
t.Fatalf("register b: %v", err)
|
||||
}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected hit")
|
||||
}
|
||||
|
||||
byKind := map[MarkerKind]Marker{}
|
||||
for _, m := range res.Markers {
|
||||
byKind[m.Kind] = m
|
||||
}
|
||||
if intro := byKind[MarkerKindIntro]; intro.ProviderID != "a" || intro.End != 30*time.Second {
|
||||
t.Errorf("intro = %+v, want priority provider a end 30s", intro)
|
||||
}
|
||||
if credits := byKind[MarkerKindCredits]; credits.ProviderID != "a" || credits.End != 110*time.Second {
|
||||
t.Errorf("credits = %+v, want priority provider a end 110s", credits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedUsesConfiguredProviderPriority(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.9, SubmissionCount: 500},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.6, SubmissionCount: 1},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(a)
|
||||
_ = registry.Register(b)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"a": {Provider: "a", FetchEnabled: true, FetchPriority: 20},
|
||||
"b": {Provider: "b", FetchEnabled: true, FetchPriority: 10},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "b" || got.End != 25*time.Second {
|
||||
t.Fatalf("winner = %+v, want lower fetch_priority provider b", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedFallsBackWhenHigherPriorityProviderErrors(t *testing.T) {
|
||||
higher := &fakeProvider{id: "higher", err: errors.New("temporary provider failure")}
|
||||
lower := &fakeProvider{id: "lower", result: Result{ProviderID: "lower", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.6},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(higher)
|
||||
_ = registry.Register(lower)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"higher": {Provider: "higher", FetchEnabled: true, FetchPriority: 10},
|
||||
"lower": {Provider: "lower", FetchEnabled: true, FetchPriority: 20},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged returned error despite fallback marker: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "lower" {
|
||||
t.Fatalf("winner provider = %q, want lower", got.ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedTieBreaksSamePriorityByQuality(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.7, SubmissionCount: 1},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.9, SubmissionCount: 2},
|
||||
}}}
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(a)
|
||||
_ = registry.Register(b)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"a": {Provider: "a", FetchEnabled: true, FetchPriority: 10},
|
||||
"b": {Provider: "b", FetchEnabled: true, FetchPriority: 10},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "b" {
|
||||
t.Fatalf("winner provider = %q, want b by same-priority submission count", got.ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedSingleProviderParity(t *testing.T) {
|
||||
only := &fakeProvider{id: "introdb", result: Result{
|
||||
ProviderID: "introdb",
|
||||
SourceClass: "online",
|
||||
Algorithm: "introdb:v3",
|
||||
Markers: []Marker{{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.9}},
|
||||
}}
|
||||
registry := NewRegistry(nil)
|
||||
if err := registry.Register(only); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 || res.Markers[0].Kind != MarkerKindIntro {
|
||||
t.Fatalf("single-provider merge = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if res.Markers[0].ProviderID != "introdb" {
|
||||
t.Errorf("provider = %q, want introdb (stamped from result)", res.Markers[0].ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSetting(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
+138
-49
@@ -11,6 +11,12 @@ import (
|
||||
// wins; an equal-priority source wins only if its confidence is strictly
|
||||
// higher than what's already stored. Unknown/empty existing source is treated
|
||||
// as priority zero so any defined source can replace it.
|
||||
//
|
||||
// A manual write is the exception: an explicit human edit is authoritative and
|
||||
// always applies at equal-or-higher priority (last-writer-wins). Without this,
|
||||
// correcting a previously hand-set marker — whose confidence is fixed at 1.0 —
|
||||
// would be silently rejected by the strictly-higher-confidence rule, leaving
|
||||
// users unable to fix their own (or each other's) edits.
|
||||
func CanWriteMarker(existingSource *string, existingConfidence *float64, newSource string, newConfidence *float64) bool {
|
||||
currentSource := ""
|
||||
if existingSource != nil {
|
||||
@@ -24,59 +30,97 @@ func CanWriteMarker(existingSource *string, existingConfidence *float64, newSour
|
||||
if newPriority < existingPriority {
|
||||
return false
|
||||
}
|
||||
if newSource == models.MarkerSourceManual {
|
||||
return true
|
||||
}
|
||||
if existingConfidence != nil && newConfidence != nil {
|
||||
return *newConfidence > *existingConfidence
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SegmentPayload is the storage-agnostic per-segment marker write: its bounds
|
||||
// plus the provenance (provider/confidence/algorithm) of the marker that
|
||||
// produced it. A merged multi-provider result carries a different provider per
|
||||
// segment; a single-source result repeats the same provider across segments.
|
||||
type SegmentPayload struct {
|
||||
Start, End *float64
|
||||
Source string
|
||||
Provider *string
|
||||
Confidence *float64
|
||||
Algorithm string
|
||||
}
|
||||
|
||||
// Present reports whether the segment carries a bound to write.
|
||||
func (s SegmentPayload) Present() bool { return s.Start != nil || s.End != nil }
|
||||
|
||||
// MarkerUpdatePayload is the storage-agnostic shape produced from a provider
|
||||
// Result. Repositories convert it into their concrete write column set.
|
||||
// Pointer fields are nil when the result didn't carry that segment kind.
|
||||
// Result. Repositories convert it into their concrete write column set. Source
|
||||
// is the shared source class (online/scanner/manual/...); each SegmentPayload
|
||||
// carries its own provider/confidence/algorithm so a merged multi-provider
|
||||
// result records correct per-segment provenance.
|
||||
type MarkerUpdatePayload struct {
|
||||
IntroStart, IntroEnd *float64
|
||||
CreditsStart, CreditsEnd *float64
|
||||
RecapStart, RecapEnd *float64
|
||||
PreviewStart, PreviewEnd *float64
|
||||
Source string
|
||||
Provider *string
|
||||
Confidence *float64
|
||||
Algorithm string
|
||||
Intro SegmentPayload
|
||||
Credits SegmentPayload
|
||||
Recap SegmentPayload
|
||||
Preview SegmentPayload
|
||||
Source string
|
||||
}
|
||||
|
||||
// HasAnySegment reports whether the payload carries at least one segment
|
||||
// range. Callers can short-circuit empty writes without touching the DB.
|
||||
func (p MarkerUpdatePayload) HasAnySegment() bool {
|
||||
return p.IntroStart != nil || p.IntroEnd != nil ||
|
||||
p.CreditsStart != nil || p.CreditsEnd != nil ||
|
||||
p.RecapStart != nil || p.RecapEnd != nil ||
|
||||
p.PreviewStart != nil || p.PreviewEnd != nil
|
||||
return p.Intro.Present() || p.Credits.Present() || p.Recap.Present() || p.Preview.Present()
|
||||
}
|
||||
|
||||
// BuildUpdatePayload converts a provider Result into the storage payload.
|
||||
// All four segment kinds in the result map to the corresponding *Start/*End
|
||||
// pointer pair; absent kinds remain nil. The Result's Algorithm field is
|
||||
// authoritative; the falls back to `external:<source>` if absent so writes
|
||||
// always carry an algorithm tag for provenance.
|
||||
//
|
||||
// Confidence: the highest per-marker confidence in the result is promoted to
|
||||
// the shared write-time confidence. Per-segment confidence values are not
|
||||
// preserved individually — the write path uses a single confidence per
|
||||
// upsert, and providers today return uniform confidence across all segments
|
||||
// of a single fetch.
|
||||
func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
payload := MarkerUpdatePayload{
|
||||
Source: result.SourceClass,
|
||||
Algorithm: result.Algorithm,
|
||||
// SummaryConfidence returns the highest per-segment confidence present, for the
|
||||
// legacy shared markers_confidence column. Returns nil when no segment carries
|
||||
// a confidence.
|
||||
func (p MarkerUpdatePayload) SummaryConfidence() *float64 {
|
||||
var max float64
|
||||
found := false
|
||||
for _, s := range []SegmentPayload{p.Intro, p.Credits, p.Recap, p.Preview} {
|
||||
if s.Confidence != nil && (!found || *s.Confidence > max) {
|
||||
max = *s.Confidence
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if payload.Algorithm == "" && payload.Source != "" {
|
||||
payload.Algorithm = "external:" + payload.Source
|
||||
}
|
||||
if provider := strings.TrimSpace(result.ProviderID); provider != "" {
|
||||
payload.Provider = &provider
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return &max
|
||||
}
|
||||
|
||||
var maxConfidence float64
|
||||
// SummarySource returns the highest-priority source class present in the
|
||||
// per-segment payloads, using confidence as the tie-breaker. This keeps the
|
||||
// legacy shared markers_source column coherent even when individual segment
|
||||
// provenance differs.
|
||||
func (p MarkerUpdatePayload) SummarySource() string {
|
||||
source := strings.TrimSpace(p.Source)
|
||||
confidence := p.SummaryConfidence()
|
||||
found := source != ""
|
||||
for _, s := range []SegmentPayload{p.Intro, p.Credits, p.Recap, p.Preview} {
|
||||
if !s.Present() || strings.TrimSpace(s.Source) == "" {
|
||||
continue
|
||||
}
|
||||
if !found ||
|
||||
models.MarkerSourcePriority(s.Source) > models.MarkerSourcePriority(source) ||
|
||||
(models.MarkerSourcePriority(s.Source) == models.MarkerSourcePriority(source) && confidenceGreater(s.Confidence, confidence)) {
|
||||
source = s.Source
|
||||
confidence = s.Confidence
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
// BuildUpdatePayload converts a provider Result into the storage payload. Each
|
||||
// marker maps to its segment with its own provenance: ProviderID/Algorithm fall
|
||||
// back to the Result-level values (used for single-provider results), and the
|
||||
// algorithm finally falls back to external:<source> so every write carries an
|
||||
// algorithm tag.
|
||||
func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
payload := MarkerUpdatePayload{Source: result.SourceClass}
|
||||
for _, m := range result.Markers {
|
||||
start := m.Start.Seconds()
|
||||
end := m.End.Seconds()
|
||||
@@ -84,26 +128,71 @@ func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
continue
|
||||
}
|
||||
startPtr, endPtr := start, end
|
||||
source := markerSource(m, result)
|
||||
seg := SegmentPayload{
|
||||
Start: &startPtr,
|
||||
End: &endPtr,
|
||||
Source: source,
|
||||
Provider: markerProvider(m, result),
|
||||
Algorithm: markerAlgorithm(m, result.Algorithm, source),
|
||||
}
|
||||
if m.Confidence > 0 {
|
||||
conf := m.Confidence
|
||||
seg.Confidence = &conf
|
||||
}
|
||||
switch m.Kind {
|
||||
case MarkerKindIntro:
|
||||
payload.IntroStart = &startPtr
|
||||
payload.IntroEnd = &endPtr
|
||||
payload.Intro = seg
|
||||
case MarkerKindCredits:
|
||||
payload.CreditsStart = &startPtr
|
||||
payload.CreditsEnd = &endPtr
|
||||
payload.Credits = seg
|
||||
case MarkerKindRecap:
|
||||
payload.RecapStart = &startPtr
|
||||
payload.RecapEnd = &endPtr
|
||||
payload.Recap = seg
|
||||
case MarkerKindPreview:
|
||||
payload.PreviewStart = &startPtr
|
||||
payload.PreviewEnd = &endPtr
|
||||
payload.Preview = seg
|
||||
}
|
||||
if m.Confidence > maxConfidence {
|
||||
maxConfidence = m.Confidence
|
||||
}
|
||||
}
|
||||
if maxConfidence > 0 {
|
||||
payload.Confidence = &maxConfidence
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func markerSource(m Marker, result Result) string {
|
||||
source := strings.TrimSpace(m.SourceClass)
|
||||
if source == "" {
|
||||
source = strings.TrimSpace(result.SourceClass)
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
// markerProvider returns the per-marker provider, falling back to the
|
||||
// Result-level provider (single-provider results), or nil when neither is set.
|
||||
func markerProvider(m Marker, result Result) *string {
|
||||
provider := strings.TrimSpace(m.ProviderID)
|
||||
if provider == "" {
|
||||
provider = strings.TrimSpace(result.ProviderID)
|
||||
}
|
||||
if provider == "" {
|
||||
return nil
|
||||
}
|
||||
return &provider
|
||||
}
|
||||
|
||||
// markerAlgorithm returns the per-marker algorithm, falling back to the
|
||||
// already-resolved Result-level algorithm.
|
||||
func markerAlgorithm(m Marker, resultAlgorithm, source string) string {
|
||||
if a := strings.TrimSpace(m.Algorithm); a != "" {
|
||||
return a
|
||||
}
|
||||
if strings.TrimSpace(resultAlgorithm) == "" && strings.TrimSpace(source) != "" {
|
||||
return "external:" + source
|
||||
}
|
||||
return resultAlgorithm
|
||||
}
|
||||
|
||||
func confidenceGreater(a, b *float64) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
return *a > *b
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestBuildUpdatePayloadAggregatesConfidence(t *testing.T) {
|
||||
func TestBuildUpdatePayloadPreservesPerSegmentConfidence(t *testing.T) {
|
||||
result := Result{
|
||||
ProviderID: "introdb",
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
@@ -21,35 +21,71 @@ func TestBuildUpdatePayloadAggregatesConfidence(t *testing.T) {
|
||||
|
||||
payload := BuildUpdatePayload(result)
|
||||
|
||||
if payload.Confidence == nil {
|
||||
t.Fatal("confidence should be populated")
|
||||
if payload.Intro.Confidence == nil || *payload.Intro.Confidence != 0.7 {
|
||||
t.Errorf("intro confidence = %v, want 0.7", payload.Intro.Confidence)
|
||||
}
|
||||
if *payload.Confidence != 0.9 {
|
||||
t.Errorf("confidence = %v, want 0.9 (max across markers)", *payload.Confidence)
|
||||
if payload.Credits.Confidence == nil || *payload.Credits.Confidence != 0.9 {
|
||||
t.Errorf("credits confidence = %v, want 0.9", payload.Credits.Confidence)
|
||||
}
|
||||
if payload.Algorithm != "introdb:v3" {
|
||||
t.Errorf("algorithm = %q, want introdb:v3", payload.Algorithm)
|
||||
if payload.Recap.Confidence == nil || *payload.Recap.Confidence != 0.5 {
|
||||
t.Errorf("recap confidence = %v, want 0.5", payload.Recap.Confidence)
|
||||
}
|
||||
if payload.IntroStart == nil || *payload.IntroStart != 10 {
|
||||
t.Errorf("intro start = %v, want 10", payload.IntroStart)
|
||||
if sc := payload.SummaryConfidence(); sc == nil || *sc != 0.9 {
|
||||
t.Errorf("summary confidence = %v, want 0.9 (max)", sc)
|
||||
}
|
||||
if payload.RecapStart == nil || *payload.RecapStart != 0 {
|
||||
t.Errorf("recap start = %v, want 0", payload.RecapStart)
|
||||
if payload.Intro.Algorithm != "introdb:v3" {
|
||||
t.Errorf("intro algorithm = %q, want introdb:v3", payload.Intro.Algorithm)
|
||||
}
|
||||
if payload.PreviewStart != nil {
|
||||
t.Errorf("preview start = %v, want nil (no preview marker)", payload.PreviewStart)
|
||||
if payload.Intro.Start == nil || *payload.Intro.Start != 10 {
|
||||
t.Errorf("intro start = %v, want 10", payload.Intro.Start)
|
||||
}
|
||||
if payload.Recap.Start == nil || *payload.Recap.Start != 0 {
|
||||
t.Errorf("recap start = %v, want 0", payload.Recap.Start)
|
||||
}
|
||||
if payload.Preview.Present() {
|
||||
t.Errorf("preview should be absent (no preview marker)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildUpdatePayloadFallsBackAlgorithm(t *testing.T) {
|
||||
func TestBuildUpdatePayloadPerMarkerProvider(t *testing.T) {
|
||||
// A merged result: each marker carries its own provider/algorithm.
|
||||
result := Result{
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.8, ProviderID: "introdb", Algorithm: "introdb:v3"},
|
||||
{Kind: MarkerKindCredits, Start: 100 * time.Second, End: 120 * time.Second, Confidence: 0.7, SourceClass: models.MarkerSourcePlugin, ProviderID: "plugin:1:markers", Algorithm: "other:v1"},
|
||||
},
|
||||
}
|
||||
payload := BuildUpdatePayload(result)
|
||||
if payload.Intro.Provider == nil || *payload.Intro.Provider != "introdb" {
|
||||
t.Errorf("intro provider = %v, want introdb", payload.Intro.Provider)
|
||||
}
|
||||
if payload.Credits.Provider == nil || *payload.Credits.Provider != "plugin:1:markers" {
|
||||
t.Errorf("credits provider = %v, want plugin:1:markers", payload.Credits.Provider)
|
||||
}
|
||||
if payload.Credits.Source != models.MarkerSourcePlugin {
|
||||
t.Errorf("credits source = %q, want plugin", payload.Credits.Source)
|
||||
}
|
||||
if payload.Credits.Algorithm != "other:v1" {
|
||||
t.Errorf("credits algorithm = %q, want other:v1", payload.Credits.Algorithm)
|
||||
}
|
||||
if payload.Intro.Source != models.MarkerSourceOnline {
|
||||
t.Errorf("intro source = %q, want online fallback", payload.Intro.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildUpdatePayloadFallsBackAlgorithmAndProvider(t *testing.T) {
|
||||
result := Result{
|
||||
ProviderID: "custom",
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Markers: []Marker{{Kind: MarkerKindIntro, Start: 0, End: 10 * time.Second}},
|
||||
}
|
||||
payload := BuildUpdatePayload(result)
|
||||
if payload.Algorithm != "external:online" {
|
||||
t.Errorf("algorithm = %q, want external:online fallback", payload.Algorithm)
|
||||
if payload.Intro.Algorithm != "external:online" {
|
||||
t.Errorf("intro algorithm = %q, want external:online fallback", payload.Intro.Algorithm)
|
||||
}
|
||||
if payload.Intro.Provider == nil || *payload.Intro.Provider != "custom" {
|
||||
t.Errorf("intro provider = %v, want custom (result-level fallback)", payload.Intro.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,3 +114,24 @@ func TestCanWriteMarkerHigherPriorityWinsRegardless(t *testing.T) {
|
||||
t.Error("lower priority should not overwrite higher")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanWriteMarkerManualAlwaysWinsLastWriter(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
conf := 1.0
|
||||
|
||||
// A manual edit must overwrite an existing manual marker even though both
|
||||
// share priority 4 and confidence 1.0 — otherwise corrections silently fail.
|
||||
if !CanWriteMarker(&manual, &conf, models.MarkerSourceManual, &conf) {
|
||||
t.Error("manual edit should overwrite an existing manual marker (last-writer-wins)")
|
||||
}
|
||||
// Manual still wins over lower-priority sources.
|
||||
online := models.MarkerSourceOnline
|
||||
highConf := 0.99
|
||||
if !CanWriteMarker(&online, &highConf, models.MarkerSourceManual, &conf) {
|
||||
t.Error("manual edit should overwrite a high-confidence online marker")
|
||||
}
|
||||
// A non-manual source still cannot displace a manual marker.
|
||||
if CanWriteMarker(&manual, &conf, models.MarkerSourceOnline, &highConf) {
|
||||
t.Error("online source should never overwrite a manual marker")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,10 @@ import (
|
||||
|
||||
const EpisodeRecentStillWindow = 45 * 24 * time.Hour
|
||||
|
||||
var episodePlaceholderTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd|episode\s+\d+)$`)
|
||||
var (
|
||||
episodePlaceholderTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd|episode\s+\d+)$`)
|
||||
episodeProvisionalTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd)$`)
|
||||
)
|
||||
|
||||
func IsEpisodePlaceholderTitle(title string) bool {
|
||||
title = strings.TrimSpace(title)
|
||||
@@ -20,6 +23,23 @@ func IsEpisodePlaceholderTitle(title string) bool {
|
||||
return episodePlaceholderTitlePattern.MatchString(title)
|
||||
}
|
||||
|
||||
func isEpisodeProvisionalTitle(title string) bool {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" {
|
||||
return false
|
||||
}
|
||||
return episodeProvisionalTitlePattern.MatchString(title)
|
||||
}
|
||||
|
||||
func episodeHasProviderMatch(ep *models.Episode) bool {
|
||||
if ep == nil || strings.EqualFold(strings.TrimSpace(ep.MetadataSource), "scanner_fallback") {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(ep.TmdbID) != "" ||
|
||||
strings.TrimSpace(ep.TvdbID) != "" ||
|
||||
strings.TrimSpace(ep.ImdbID) != ""
|
||||
}
|
||||
|
||||
func EpisodeHasIncompleteMetadata(ep *models.Episode, now time.Time) bool {
|
||||
if ep == nil {
|
||||
return false
|
||||
@@ -43,3 +63,29 @@ func EpisodeHasIncompleteMetadata(ep *models.Episode, now time.Time) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EpisodeHasActionableMetadataDebt returns true when another provider refresh
|
||||
// can reasonably be expected to improve the episode metadata.
|
||||
func EpisodeHasActionableMetadataDebt(ep *models.Episode, now time.Time) bool {
|
||||
if ep == nil {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(ep.Title) == "" {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(ep.MetadataSource), "scanner_fallback") {
|
||||
return true
|
||||
}
|
||||
if !episodeHasProviderMatch(ep) {
|
||||
return true
|
||||
}
|
||||
if isEpisodeProvisionalTitle(ep.Title) {
|
||||
return true
|
||||
}
|
||||
if ep.AirDate != nil &&
|
||||
!ep.AirDate.Before(now.Add(-EpisodeRecentStillWindow)) &&
|
||||
strings.TrimSpace(ep.StillPath) == "" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -46,4 +46,110 @@ func TestEpisodeHasIncompleteMetadata(t *testing.T) {
|
||||
if !EpisodeHasIncompleteMetadata(incomplete, now) {
|
||||
t.Fatal("expected fallback placeholder episode to be treated as incomplete")
|
||||
}
|
||||
|
||||
providerNumbered := &models.Episode{
|
||||
Title: "Episode 1",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
}
|
||||
if !EpisodeHasIncompleteMetadata(providerNumbered, now) {
|
||||
t.Fatal("expected provider numbered title to remain quality-incomplete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpisodeHasActionableMetadataDebt(t *testing.T) {
|
||||
now := time.Date(2026, 4, 14, 12, 0, 0, 0, time.UTC)
|
||||
recent := now.Add(-7 * 24 * time.Hour)
|
||||
old := now.Add(-90 * 24 * time.Hour)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
ep *models.Episode
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "provider numbered title with overview",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "provider numbered title with empty overview",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
TvdbID: "9607609",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "provider provisional title",
|
||||
ep: &models.Episode{
|
||||
Title: "TBD",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "scanner fallback numbered title",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
MetadataSource: "scanner_fallback",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "missing title",
|
||||
ep: &models.Episode{
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "missing provider id",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "recent provider episode missing still",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
AirDate: &recent,
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "old provider episode missing still",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
AirDate: &old,
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := EpisodeHasActionableMetadataDebt(tc.ep, now); got != tc.want {
|
||||
t.Fatalf("EpisodeHasActionableMetadataDebt() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// isPgConstraintViolation reports whether err (or any error it wraps) is a
|
||||
// Postgres error with the given SQLSTATE code raised by the named constraint.
|
||||
// Centralizes the errors.As(&pgconn.PgError) + Code/ConstraintName boilerplate
|
||||
// that several constraint-specific predicates in this package share.
|
||||
func isPgConstraintViolation(err error, code, constraint string) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) {
|
||||
return false
|
||||
}
|
||||
return pgErr.Code == code && pgErr.ConstraintName == constraint
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHandleChildProvider404KeepsProviderID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ids := map[string]string{
|
||||
"tmdb": "32843",
|
||||
"tvdb": "164951",
|
||||
}
|
||||
|
||||
if !handleChildProvider404("tmdb", ids, errors.New("tmdb: HTTP 404: not found"), "season", 0) {
|
||||
t.Fatal("handleChildProvider404() = false, want true")
|
||||
}
|
||||
if ids["tmdb"] != "32843" {
|
||||
t.Fatalf("tmdb id = %q, want preserved", ids["tmdb"])
|
||||
}
|
||||
if ids["tvdb"] != "164951" {
|
||||
t.Fatalf("tvdb id = %q, want preserved", ids["tvdb"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleProvider404DropsProviderID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ids := map[string]string{
|
||||
"tmdb": "32843",
|
||||
"tvdb": "164951",
|
||||
}
|
||||
|
||||
if !handleProvider404(nil, ids, "tmdb", errors.New("tmdb: HTTP 404: not found")) {
|
||||
t.Fatal("handleProvider404() = false, want true")
|
||||
}
|
||||
if _, ok := ids["tmdb"]; ok {
|
||||
t.Fatalf("tmdb id was not dropped: %v", ids)
|
||||
}
|
||||
if ids["tvdb"] != "164951" {
|
||||
t.Fatalf("tvdb id = %q, want preserved", ids["tvdb"])
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -577,25 +579,59 @@ func betterCanonicalCandidate(left, right canonicalCandidate) bool {
|
||||
return left.ContentID < right.ContentID
|
||||
}
|
||||
|
||||
func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID, canonicalID string) error {
|
||||
if sourceID == "" || canonicalID == "" || sourceID == canonicalID {
|
||||
return nil
|
||||
// mediaItemMergeStep is one statement in the ordered media-item merge sequence
|
||||
// run by canonicalizeMediaItemReferencesTx. Every step references $1 (the
|
||||
// source content ID); steps that move data onto the canonical row also
|
||||
// reference $2 (the canonical content ID).
|
||||
type mediaItemMergeStep struct {
|
||||
name string
|
||||
sql string
|
||||
}
|
||||
|
||||
// mergeStepPlaceholderRe matches positional query placeholders ($1, $2, …).
|
||||
var mergeStepPlaceholderRe = regexp.MustCompile(`\$(\d+)`)
|
||||
|
||||
// maxPlaceholder returns the highest positional placeholder number referenced
|
||||
// in sql (0 if none). It reads the full placeholder number, so "$20" is 20 and
|
||||
// is never confused with "$2" — unlike a substring check.
|
||||
func maxPlaceholder(sql string) int {
|
||||
highest := 0
|
||||
for _, m := range mergeStepPlaceholderRe.FindAllStringSubmatch(sql, -1) {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil && n > highest {
|
||||
highest = n
|
||||
}
|
||||
}
|
||||
if err := ensureSeriesCanMove(ctx, tx, sourceID, canonicalID); err != nil {
|
||||
return err
|
||||
return highest
|
||||
}
|
||||
|
||||
// mergeStepArgs returns the positional arguments for a merge step, sized to the
|
||||
// placeholders the SQL actually binds. Passing too many args (e.g. an unused
|
||||
// $2) makes pgx reject the Exec with "mismatched param and argument count"
|
||||
// under the default QueryExecModeCacheStatement, aborting the whole merge
|
||||
// transaction. Every merge step binds $1 (source) and optionally $2
|
||||
// (canonical); TestMergeStepPlaceholdersAreBounded enforces that contract, so
|
||||
// the default branch is unreachable for the defined steps and fails loud if a
|
||||
// future step violates it.
|
||||
func mergeStepArgs(sql, sourceID, canonicalID string) []any {
|
||||
switch n := maxPlaceholder(sql); n {
|
||||
case 1:
|
||||
return []any{sourceID}
|
||||
case 2:
|
||||
return []any{sourceID, canonicalID}
|
||||
default:
|
||||
panic(fmt.Sprintf("merge step binds unsupported placeholder count %d: %q", n, sql))
|
||||
}
|
||||
steps := []struct {
|
||||
name string
|
||||
sql string
|
||||
}{
|
||||
{"merge provider ids", `
|
||||
}
|
||||
|
||||
var mediaItemMergeSteps = []mediaItemMergeStep{
|
||||
{"merge provider ids", `
|
||||
INSERT INTO media_item_provider_ids (content_id, item_type, provider, provider_id, created_at, updated_at)
|
||||
SELECT $2, item_type, provider, provider_id, created_at, NOW()
|
||||
FROM media_item_provider_ids
|
||||
WHERE content_id = $1
|
||||
ON CONFLICT DO NOTHING`},
|
||||
{"drop source provider ids", `DELETE FROM media_item_provider_ids WHERE content_id = $1`},
|
||||
{"merge legacy provider columns", `
|
||||
{"drop source provider ids", `DELETE FROM media_item_provider_ids WHERE content_id = $1`},
|
||||
{"merge legacy provider columns", `
|
||||
UPDATE media_items dest
|
||||
SET imdb_id = COALESCE(NULLIF(dest.imdb_id, ''), src.imdb_id),
|
||||
tmdb_id = COALESCE(NULLIF(dest.tmdb_id, ''), src.tmdb_id),
|
||||
@@ -603,10 +639,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
updated_at = NOW()
|
||||
FROM media_items src
|
||||
WHERE dest.content_id = $2 AND src.content_id = $1`},
|
||||
{"move media files", `UPDATE media_files SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"move seasons", `UPDATE seasons SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"move episodes", `UPDATE episodes SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"merge library memberships", `
|
||||
{"move media files", `UPDATE media_files SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"move seasons", `UPDATE seasons SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"move episodes", `UPDATE episodes SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"merge library memberships", `
|
||||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||||
SELECT $2, media_folder_id, MIN(first_seen_at)
|
||||
FROM media_item_libraries
|
||||
@@ -614,10 +650,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY media_folder_id
|
||||
ON CONFLICT (content_id, media_folder_id) DO UPDATE
|
||||
SET first_seen_at = LEAST(media_item_libraries.first_seen_at, EXCLUDED.first_seen_at)`},
|
||||
{"delete source library memberships", `DELETE FROM media_item_libraries WHERE content_id = $1`},
|
||||
{"move root claims", `UPDATE media_item_roots SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"move group claims", `UPDATE media_item_groups SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"merge collection items", `
|
||||
{"delete source library memberships", `DELETE FROM media_item_libraries WHERE content_id = $1`},
|
||||
{"move root claims", `UPDATE media_item_roots SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"move group claims", `UPDATE media_item_groups SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"merge collection items", `
|
||||
INSERT INTO library_collection_items (collection_id, media_item_id, position, source_rank, created_at, updated_at)
|
||||
SELECT collection_id, $2, MIN(position), MIN(source_rank), MIN(created_at), NOW()
|
||||
FROM library_collection_items
|
||||
@@ -627,8 +663,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
SET position = LEAST(library_collection_items.position, EXCLUDED.position),
|
||||
source_rank = LEAST(library_collection_items.source_rank, EXCLUDED.source_rank),
|
||||
updated_at = NOW()`},
|
||||
{"delete source collection items", `DELETE FROM library_collection_items WHERE media_item_id = $1`},
|
||||
{"merge personal collection items", `
|
||||
{"delete source collection items", `DELETE FROM library_collection_items WHERE media_item_id = $1`},
|
||||
{"merge personal collection items", `
|
||||
INSERT INTO user_personal_collection_items (user_id, collection_id, media_item_id, position, added_at)
|
||||
SELECT user_id, collection_id, $2, MIN(position), MIN(added_at)
|
||||
FROM user_personal_collection_items
|
||||
@@ -637,8 +673,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, collection_id, media_item_id) DO UPDATE
|
||||
SET position = LEAST(user_personal_collection_items.position, EXCLUDED.position),
|
||||
added_at = LEAST(user_personal_collection_items.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source personal collection items", `DELETE FROM user_personal_collection_items WHERE media_item_id = $1`},
|
||||
{"merge favorites", `
|
||||
{"delete source personal collection items", `DELETE FROM user_personal_collection_items WHERE media_item_id = $1`},
|
||||
{"merge favorites", `
|
||||
INSERT INTO user_favorites (user_id, profile_id, media_item_id, added_at)
|
||||
SELECT user_id, profile_id, $2, MIN(added_at)
|
||||
FROM user_favorites
|
||||
@@ -646,8 +682,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY user_id, profile_id
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET added_at = LEAST(user_favorites.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source favorites", `DELETE FROM user_favorites WHERE media_item_id = $1`},
|
||||
{"merge watchlist", `
|
||||
{"delete source favorites", `DELETE FROM user_favorites WHERE media_item_id = $1`},
|
||||
{"merge watchlist", `
|
||||
INSERT INTO user_watchlist (user_id, profile_id, media_item_id, added_at)
|
||||
SELECT user_id, profile_id, $2, MIN(added_at)
|
||||
FROM user_watchlist
|
||||
@@ -655,8 +691,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY user_id, profile_id
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET added_at = LEAST(user_watchlist.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source watchlist", `DELETE FROM user_watchlist WHERE media_item_id = $1`},
|
||||
{"merge ratings", `
|
||||
{"delete source watchlist", `DELETE FROM user_watchlist WHERE media_item_id = $1`},
|
||||
{"merge ratings", `
|
||||
INSERT INTO user_ratings (user_id, profile_id, media_item_id, rating, rated_at)
|
||||
SELECT DISTINCT ON (user_id, profile_id) user_id, profile_id, $2, rating, rated_at
|
||||
FROM user_ratings
|
||||
@@ -665,8 +701,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET rating = CASE WHEN EXCLUDED.rated_at >= user_ratings.rated_at THEN EXCLUDED.rating ELSE user_ratings.rating END,
|
||||
rated_at = GREATEST(user_ratings.rated_at, EXCLUDED.rated_at)`},
|
||||
{"delete source ratings", `DELETE FROM user_ratings WHERE media_item_id = $1`},
|
||||
{"merge progress", `
|
||||
{"delete source ratings", `DELETE FROM user_ratings WHERE media_item_id = $1`},
|
||||
{"merge progress", `
|
||||
INSERT INTO user_watch_progress (
|
||||
user_id, profile_id, media_item_id, position_seconds, duration_seconds, completed,
|
||||
updated_at, last_file_id, last_resolution, last_hdr, last_codec_video, last_edition_key
|
||||
@@ -685,9 +721,9 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
last_hdr = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_hdr ELSE user_watch_progress.last_hdr END,
|
||||
last_codec_video = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_codec_video ELSE user_watch_progress.last_codec_video END,
|
||||
last_edition_key = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_edition_key ELSE user_watch_progress.last_edition_key END`},
|
||||
{"delete source progress", `DELETE FROM user_watch_progress WHERE media_item_id = $1`},
|
||||
{"move history", `UPDATE user_watch_history SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"merge hidden history", `
|
||||
{"delete source progress", `DELETE FROM user_watch_progress WHERE media_item_id = $1`},
|
||||
{"move history", `UPDATE user_watch_history SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"merge hidden history", `
|
||||
INSERT INTO user_history_hidden_items (user_id, profile_id, media_item_id, hidden_before, updated_at)
|
||||
SELECT user_id, profile_id, $2, MAX(hidden_before), MAX(updated_at)
|
||||
FROM user_history_hidden_items
|
||||
@@ -696,8 +732,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET hidden_before = GREATEST(user_history_hidden_items.hidden_before, EXCLUDED.hidden_before),
|
||||
updated_at = GREATEST(user_history_hidden_items.updated_at, EXCLUDED.updated_at)`},
|
||||
{"delete source hidden history", `DELETE FROM user_history_hidden_items WHERE media_item_id = $1`},
|
||||
{"delete duplicate home item dismissals", `
|
||||
{"delete source hidden history", `DELETE FROM user_history_hidden_items WHERE media_item_id = $1`},
|
||||
{"delete duplicate home item dismissals", `
|
||||
DELETE FROM user_home_item_dismissals src
|
||||
USING user_home_item_dismissals dest
|
||||
WHERE src.media_item_id = $1
|
||||
@@ -705,33 +741,33 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id
|
||||
AND dest.surface = src.surface`},
|
||||
{"move home item dismissals", `UPDATE user_home_item_dismissals SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"remap home item dismissals series", `UPDATE user_home_item_dismissals SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate audio preferences", `
|
||||
{"move home item dismissals", `UPDATE user_home_item_dismissals SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"remap home item dismissals series", `UPDATE user_home_item_dismissals SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate audio preferences", `
|
||||
DELETE FROM user_audio_preferences src
|
||||
USING user_audio_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move audio preferences", `UPDATE user_audio_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate subtitle preferences", `
|
||||
{"move audio preferences", `UPDATE user_audio_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate subtitle preferences", `
|
||||
DELETE FROM user_subtitle_preferences src
|
||||
USING user_subtitle_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move subtitle preferences", `UPDATE user_subtitle_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate series playback preferences", `
|
||||
{"move subtitle preferences", `UPDATE user_subtitle_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate series playback preferences", `
|
||||
DELETE FROM user_series_playback_preferences src
|
||||
USING user_series_playback_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move series playback preferences", `UPDATE user_series_playback_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"merge plex sync item bindings timestamps", `
|
||||
{"move series playback preferences", `UPDATE user_series_playback_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"merge plex sync item bindings timestamps", `
|
||||
UPDATE plex_sync_item_bindings dest
|
||||
SET last_seen_at = GREATEST(dest.last_seen_at, src.last_seen_at),
|
||||
updated_at = NOW()
|
||||
@@ -740,35 +776,35 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.connection_id = src.connection_id
|
||||
AND (dest.media_item_id = $2 OR dest.plex_rating_key = src.plex_rating_key)
|
||||
AND dest.media_item_id <> src.media_item_id`},
|
||||
{"delete duplicate plex sync item bindings", `
|
||||
{"delete duplicate plex sync item bindings", `
|
||||
DELETE FROM plex_sync_item_bindings src
|
||||
USING plex_sync_item_bindings dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.connection_id = src.connection_id
|
||||
AND (dest.media_item_id = $2 OR dest.plex_rating_key = src.plex_rating_key)
|
||||
AND dest.media_item_id <> src.media_item_id`},
|
||||
{"move remaining plex sync item bindings", `UPDATE plex_sync_item_bindings SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"delete duplicate plex sync item state", `
|
||||
{"move remaining plex sync item bindings", `UPDATE plex_sync_item_bindings SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"delete duplicate plex sync item state", `
|
||||
DELETE FROM plex_sync_item_state src
|
||||
USING plex_sync_item_state dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND dest.mapping_id = src.mapping_id`},
|
||||
{"move plex sync item state", `UPDATE plex_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move webhook sync item state", `UPDATE webhook_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch together rooms selected", `UPDATE watch_together_rooms SET selected_content_id = $2 WHERE selected_content_id = $1`},
|
||||
{"move watch together suggestions", `UPDATE watch_together_suggestions SET content_id = $2 WHERE content_id = $1`},
|
||||
{"move admin playback history", `UPDATE playback_history_admin SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move user downloads", `UPDATE user_downloads SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move downloads", `UPDATE downloads SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"merge embeddings", `
|
||||
{"move plex sync item state", `UPDATE plex_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move webhook sync item state", `UPDATE webhook_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch together rooms selected", `UPDATE watch_together_rooms SET selected_content_id = $2 WHERE selected_content_id = $1`},
|
||||
{"move watch together suggestions", `UPDATE watch_together_suggestions SET content_id = $2 WHERE content_id = $1`},
|
||||
{"move admin playback history", `UPDATE playback_history_admin SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move user downloads", `UPDATE user_downloads SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move downloads", `UPDATE downloads SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"merge embeddings", `
|
||||
INSERT INTO media_item_embeddings (media_item_id, embedding, model, canonical_text, created_at, updated_at)
|
||||
SELECT $2, embedding, model, canonical_text, created_at, updated_at
|
||||
FROM media_item_embeddings
|
||||
WHERE media_item_id = $1
|
||||
ON CONFLICT (media_item_id) DO NOTHING`},
|
||||
{"delete source embeddings", `DELETE FROM media_item_embeddings WHERE media_item_id = $1`},
|
||||
{"merge item localizations", `
|
||||
{"delete source embeddings", `DELETE FROM media_item_embeddings WHERE media_item_id = $1`},
|
||||
{"merge item localizations", `
|
||||
INSERT INTO media_item_localizations (
|
||||
content_id, language, title, sort_title, overview, tagline, poster_path, poster_thumbhash,
|
||||
backdrop_path, backdrop_thumbhash, logo_path, created_at, updated_at
|
||||
@@ -778,8 +814,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
FROM media_item_localizations
|
||||
WHERE content_id = $1
|
||||
ON CONFLICT (content_id, language) DO NOTHING`},
|
||||
{"delete source localizations", `DELETE FROM media_item_localizations WHERE content_id = $1`},
|
||||
{"dedupe people", `
|
||||
{"delete source localizations", `DELETE FROM media_item_localizations WHERE content_id = $1`},
|
||||
{"dedupe people", `
|
||||
DELETE FROM item_people src
|
||||
USING item_people dest
|
||||
WHERE src.content_id = $1
|
||||
@@ -787,8 +823,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.person_id = src.person_id
|
||||
AND dest.kind = src.kind
|
||||
AND dest.character = src.character`},
|
||||
{"move people", `UPDATE item_people SET content_id = $2 WHERE content_id = $1`},
|
||||
{"merge refresh debt", `
|
||||
{"move people", `UPDATE item_people SET content_id = $2 WHERE content_id = $1`},
|
||||
{"merge refresh debt", `
|
||||
INSERT INTO metadata_refresh_debt (
|
||||
target_type, content_id, priority, reason_mask, next_refresh_at,
|
||||
claimed_at, lease_expires_at, last_attempt_at, last_success_at,
|
||||
@@ -806,8 +842,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
attempt_count = GREATEST(metadata_refresh_debt.attempt_count, EXCLUDED.attempt_count),
|
||||
last_error = COALESCE(NULLIF(metadata_refresh_debt.last_error, ''), EXCLUDED.last_error),
|
||||
updated_at = NOW()`},
|
||||
{"delete source refresh debt", `DELETE FROM metadata_refresh_debt WHERE target_type = 'item' AND content_id = $1`},
|
||||
{"merge stale media ids", `
|
||||
{"delete source refresh debt", `DELETE FROM metadata_refresh_debt WHERE target_type = 'item' AND content_id = $1`},
|
||||
{"merge stale media ids", `
|
||||
INSERT INTO stale_media_ids (content_id, provider, provider_id, first_seen_at, last_seen_at)
|
||||
SELECT $2, provider, provider_id, first_seen_at, last_seen_at
|
||||
FROM stale_media_ids
|
||||
@@ -816,10 +852,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
SET provider_id = COALESCE(NULLIF(stale_media_ids.provider_id, ''), EXCLUDED.provider_id),
|
||||
first_seen_at = LEAST(stale_media_ids.first_seen_at, EXCLUDED.first_seen_at),
|
||||
last_seen_at = GREATEST(stale_media_ids.last_seen_at, EXCLUDED.last_seen_at)`},
|
||||
{"delete source stale media ids", `DELETE FROM stale_media_ids WHERE content_id = $1`},
|
||||
{"move watch provider history exports", `UPDATE watch_provider_history_exports SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch provider scrobble sessions", `UPDATE watch_provider_scrobble_sessions SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"merge watch provider favorites by provider key", `
|
||||
{"delete source stale media ids", `DELETE FROM stale_media_ids WHERE content_id = $1`},
|
||||
{"move watch provider history exports", `UPDATE watch_provider_history_exports SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch provider scrobble sessions", `UPDATE watch_provider_scrobble_sessions SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"merge watch provider favorites by provider key", `
|
||||
UPDATE watch_provider_favorite_items dest
|
||||
SET remote_present = dest.remote_present OR src.remote_present,
|
||||
local_present = dest.local_present OR src.local_present,
|
||||
@@ -834,7 +870,7 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND src.connection_id = dest.connection_id
|
||||
AND src.provider_item_key <> ''
|
||||
AND src.provider_item_key = dest.provider_item_key`},
|
||||
{"merge watch provider favorites by connection", `
|
||||
{"merge watch provider favorites by connection", `
|
||||
UPDATE watch_provider_favorite_items dest
|
||||
SET provider_item_key = CASE
|
||||
WHEN dest.provider_item_key = '' THEN src.provider_item_key
|
||||
@@ -854,17 +890,25 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND src.connection_id = dest.connection_id`},
|
||||
{"delete duplicate watch provider favorites", `
|
||||
{"delete duplicate watch provider favorites", `
|
||||
DELETE FROM watch_provider_favorite_items src
|
||||
USING watch_provider_favorite_items dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND src.connection_id = dest.connection_id
|
||||
AND (src.provider_item_key = dest.provider_item_key OR dest.media_item_id = $2)`},
|
||||
{"move remaining watch provider favorites", `UPDATE watch_provider_favorite_items SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move remaining watch provider favorites", `UPDATE watch_provider_favorite_items SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
}
|
||||
|
||||
func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID, canonicalID string) error {
|
||||
if sourceID == "" || canonicalID == "" || sourceID == canonicalID {
|
||||
return nil
|
||||
}
|
||||
for _, step := range steps {
|
||||
if _, err := tx.Exec(ctx, step.sql, sourceID, canonicalID); err != nil {
|
||||
if err := ensureSeriesCanMove(ctx, tx, sourceID, canonicalID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, step := range mediaItemMergeSteps {
|
||||
if _, err := tx.Exec(ctx, step.sql, mergeStepArgs(step.sql, sourceID, canonicalID)...); err != nil {
|
||||
return fmt.Errorf("%s: %w", step.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -42,6 +43,73 @@ func TestProviderEntriesFromDriftUsesDurableProvidersOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxPlaceholder(t *testing.T) {
|
||||
// maxPlaceholder must read the FULL placeholder number, not a substring:
|
||||
// "$20" is placeholder 20, never "$2". This is the property the old
|
||||
// strings.Contains(sql, "$2") heuristic got wrong.
|
||||
cases := []struct {
|
||||
sql string
|
||||
want int
|
||||
}{
|
||||
{`DELETE FROM media_item_provider_ids WHERE content_id = $1`, 1},
|
||||
{`UPDATE media_files SET content_id = $2 WHERE content_id = $1`, 2},
|
||||
{`INSERT INTO t SELECT $2 FROM s WHERE content_id = $1`, 2},
|
||||
{`SELECT $1, $2, $1`, 2}, // repeated placeholders count once
|
||||
{`SELECT $20 FROM t`, 20},
|
||||
{`SELECT $12 FROM t`, 12},
|
||||
{`DELETE FROM t WHERE x = 'literal'`, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := maxPlaceholder(tc.sql); got != tc.want {
|
||||
t.Errorf("maxPlaceholder(%q) = %d, want %d", tc.sql, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStepArgsMatchesPlaceholderArity(t *testing.T) {
|
||||
const sourceID = "src"
|
||||
const canonicalID = "canon"
|
||||
|
||||
// A step that only references $1 must receive exactly one argument.
|
||||
// Passing canonicalID as an unused $2 makes pgx reject the Exec with
|
||||
// "mismatched param and argument count" under the default
|
||||
// QueryExecModeCacheStatement, aborting the whole merge transaction.
|
||||
onlySource := mergeStepArgs(`DELETE FROM media_item_provider_ids WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(onlySource) != 1 || onlySource[0] != sourceID {
|
||||
t.Fatalf("mergeStepArgs($1-only) = %#v, want [%q]", onlySource, sourceID)
|
||||
}
|
||||
|
||||
// A step that references $2 must receive both arguments in order.
|
||||
both := mergeStepArgs(`UPDATE media_files SET content_id = $2 WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(both) != 2 || both[0] != sourceID || both[1] != canonicalID {
|
||||
t.Fatalf("mergeStepArgs($1+$2) = %#v, want [%q %q]", both, sourceID, canonicalID)
|
||||
}
|
||||
|
||||
// $2 may appear before $1 (e.g. INSERT ... SELECT $2 ...). Args are still
|
||||
// positional [source, canonical] regardless of textual order.
|
||||
reversed := mergeStepArgs(`INSERT INTO t SELECT $2 FROM s WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(reversed) != 2 || reversed[0] != sourceID || reversed[1] != canonicalID {
|
||||
t.Fatalf("mergeStepArgs(reversed) = %#v, want [%q %q]", reversed, sourceID, canonicalID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStepPlaceholdersAreBounded(t *testing.T) {
|
||||
// Every merge step must bind $1 (the source) and may bind $2 (the
|
||||
// canonical). A step binding $3+ (or none) cannot be satisfied by the two
|
||||
// IDs the loop passes and would abort the merge at runtime; this guard —
|
||||
// checking the actual placeholder bound rather than re-deriving the arg
|
||||
// count from the same predicate the helper uses — catches such a step.
|
||||
for _, step := range mediaItemMergeSteps {
|
||||
n := maxPlaceholder(step.sql)
|
||||
if n < 1 || n > 2 {
|
||||
t.Errorf("merge step %q binds %d placeholders; merge supports only $1 and $2", step.name, n)
|
||||
}
|
||||
if !strings.Contains(step.sql, "$1") {
|
||||
t.Errorf("merge step %q does not bind $1 (source)", step.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBothCandidatesHaveContentDetectsRealUserData(t *testing.T) {
|
||||
withFiles := canonicalCandidate{ContentID: "a", ActiveFiles: 2}
|
||||
withLibraries := canonicalCandidate{ContentID: "b", LibraryCount: 1}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package metadata
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRefreshFollowUpContentID(t *testing.T) {
|
||||
// After a refresh, follow-up writes (stale-ID clear/record, debt sync) must
|
||||
// target the canonical content ID the item was persisted/merged into, not
|
||||
// the requested ID — which may have been deleted when the item was
|
||||
// canonicalized into an existing one during mergeAndPersist.
|
||||
if got := refreshFollowUpContentID("req", nil); got != "req" {
|
||||
t.Fatalf("nil result: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: ""}); got != "req" {
|
||||
t.Fatalf("empty result id: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: " "}); got != "req" {
|
||||
t.Fatalf("blank result id: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: "canonical"}); got != "canonical" {
|
||||
t.Fatalf("canonicalized result id: got %q, want %q", got, "canonical")
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/naming"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
@@ -242,6 +241,28 @@ func handleProvider404(
|
||||
return true
|
||||
}
|
||||
|
||||
func handleChildProvider404(
|
||||
provider string,
|
||||
providerIDs map[string]string,
|
||||
err error,
|
||||
attrs ...any,
|
||||
) bool {
|
||||
if !isProvider404(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
logAttrs := append([]any{"provider", provider}, attrs...)
|
||||
if provider != "" && providerIDs != nil {
|
||||
if providerID := strings.TrimSpace(providerIDs[provider]); providerID != "" {
|
||||
logAttrs = append(logAttrs, "provider_id", providerID)
|
||||
}
|
||||
}
|
||||
logAttrs = append(logAttrs, "error", err)
|
||||
slog.Info("metadata: provider returned 404 for unavailable child metadata", logAttrs...)
|
||||
return true
|
||||
}
|
||||
|
||||
// MetadataService is the unified pipeline orchestrator.
|
||||
type MetadataService struct {
|
||||
chainRepo *ChainRepository
|
||||
@@ -1137,7 +1158,10 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
Language: req.Language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
|
||||
// Pass nil for provider404s so this refresh can drop the
|
||||
// provider from the in-memory merge without recording a durable
|
||||
// stale item ID from the season chain.
|
||||
if handleProvider404(nil, accumulatedIDs, p.Slug(), err,
|
||||
"content_id", req.ContentID,
|
||||
"season", 0,
|
||||
) {
|
||||
@@ -1170,7 +1194,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
Language: req.Language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
|
||||
if handleChildProvider404(p.Slug(), accumulatedIDs, err,
|
||||
"content_id", req.ContentID,
|
||||
"season", season.SeasonNumber,
|
||||
) {
|
||||
@@ -1210,18 +1234,28 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
|
||||
// Refresh stale ID records on successful refresh: clear anything resolved,
|
||||
// then keep only the providers that still 404ed during this run.
|
||||
if s.staleIDRepo != nil && req.ContentID != "" {
|
||||
if delErr := s.staleIDRepo.DeleteByContentID(ctx, req.ContentID); delErr != nil {
|
||||
// Stale-ID follow-up targets the canonical content ID the item was
|
||||
// persisted/merged into. When mergeAndPersist canonicalizes this item into
|
||||
// an existing one, req.ContentID is the now-deleted source: clearing or
|
||||
// recording stale IDs against it would touch nothing (or FK-violate), so
|
||||
// the still-404ing providers must be recorded on result.ContentID instead.
|
||||
// provider404s is only allocated when req.ContentID was set (the refresh
|
||||
// targeted a known item). Guarding on it preserves the original gating so a
|
||||
// content-id-less refresh that canonicalizes into an existing item does not
|
||||
// wipe that item's stale rows without re-recording any.
|
||||
followUpContentID := refreshFollowUpContentID(req.ContentID, result)
|
||||
if s.staleIDRepo != nil && followUpContentID != "" && provider404s != nil {
|
||||
if delErr := s.staleIDRepo.DeleteByContentID(ctx, followUpContentID); delErr != nil {
|
||||
slog.Warn("metadata: failed to clear stale IDs after refresh",
|
||||
"content_id", req.ContentID, "error", delErr)
|
||||
"content_id", followUpContentID, "error", delErr)
|
||||
}
|
||||
for slug, providerID := range provider404s {
|
||||
if providerID == "" {
|
||||
continue
|
||||
}
|
||||
if upsertErr := s.staleIDRepo.Upsert(ctx, req.ContentID, slug, providerID); upsertErr != nil {
|
||||
if upsertErr := s.staleIDRepo.Upsert(ctx, followUpContentID, slug, providerID); upsertErr != nil {
|
||||
slog.Warn("metadata: failed to persist stale provider ID after partial refresh",
|
||||
"content_id", req.ContentID,
|
||||
"content_id", followUpContentID,
|
||||
"provider", slug,
|
||||
"provider_id", providerID,
|
||||
"error", upsertErr)
|
||||
@@ -1239,6 +1273,19 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// refreshFollowUpContentID returns the content ID that a completed refresh's
|
||||
// follow-up writes (stale-ID clear/record, debt sync) should target: the
|
||||
// canonical ID the item was persisted or merged into when available, falling
|
||||
// back to the requested ID. mergeAndPersist can canonicalize an item into an
|
||||
// existing one, deleting the requested ID, so follow-up writes keyed on
|
||||
// req.ContentID would miss the surviving item.
|
||||
func refreshFollowUpContentID(reqContentID string, result *ProcessResult) string {
|
||||
if result != nil && strings.TrimSpace(result.ContentID) != "" {
|
||||
return result.ContentID
|
||||
}
|
||||
return reqContentID
|
||||
}
|
||||
|
||||
// mergeAndPersist handles the final merge into existing item and DB persistence.
|
||||
func (s *MetadataService) mergeAndPersist(
|
||||
ctx context.Context,
|
||||
@@ -1917,7 +1964,7 @@ func (s *MetadataService) syncRefreshDebtForSeason(ctx context.Context, seasonID
|
||||
reasonMask := int64(0)
|
||||
now := time.Now().UTC()
|
||||
for _, episode := range episodes {
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
reasonMask = RefreshDebtReasonEpisodeIncomplete
|
||||
break
|
||||
}
|
||||
@@ -1952,7 +1999,7 @@ func (s *MetadataService) syncRefreshDebtForEpisode(ctx context.Context, episode
|
||||
}
|
||||
reasonMask := int64(0)
|
||||
now := time.Now().UTC()
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
reasonMask = RefreshDebtReasonEpisodeIncomplete
|
||||
}
|
||||
if reasonMask == 0 {
|
||||
@@ -2336,7 +2383,7 @@ func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, provide
|
||||
Language: language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(nil, providerIDs, p.Slug(), err, "season", seasonNumber) {
|
||||
if handleChildProvider404(p.Slug(), providerIDs, err, "season", seasonNumber) {
|
||||
continue
|
||||
}
|
||||
slog.Warn("metadata: target episode provider error",
|
||||
@@ -3468,7 +3515,7 @@ func (s *MetadataService) refreshSeriesEpisodeMetadataState(ctx context.Context,
|
||||
|
||||
incomplete := false
|
||||
for _, episode := range episodes {
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
incomplete = true
|
||||
}
|
||||
if err := s.syncVisibleEpisodeRefreshDebt(ctx, episode, now); err != nil {
|
||||
@@ -3487,7 +3534,7 @@ func (s *MetadataService) syncVisibleEpisodeRefreshDebt(ctx context.Context, epi
|
||||
if s == nil || s.refreshDebtRepo == nil || episode == nil || strings.TrimSpace(episode.ContentID) == "" {
|
||||
return nil
|
||||
}
|
||||
if !EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if !EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
return s.refreshDebtRepo.DeleteTargetDebt(ctx, RefreshTargetEpisode, episode.ContentID)
|
||||
}
|
||||
reasonMask := RefreshDebtReasonEpisodeIncomplete
|
||||
@@ -4422,14 +4469,7 @@ func rebindDeletableStatuses(allowMatchedSource bool) []string {
|
||||
}
|
||||
|
||||
func isProviderIDUniqueConflict(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) {
|
||||
return false
|
||||
}
|
||||
return pgErr.Code == "23505" && pgErr.ConstraintName == "media_item_provider_ids_provider_provider_id_item_type_key"
|
||||
return isPgConstraintViolation(err, "23505", "media_item_provider_ids_provider_provider_id_item_type_key")
|
||||
}
|
||||
|
||||
func (s *MetadataService) repairMatchedDuplicateProviderOwnersByFolderAndPathPrefix(ctx context.Context, folderID int, pathPrefix string) (int, error) {
|
||||
|
||||
@@ -986,6 +986,98 @@ func TestSyncRefreshDebtForItemPreservesRequestedEpisodeDebt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshSeriesEpisodeMetadataStateUsesActionableDebt(t *testing.T) {
|
||||
now := time.Date(2026, 4, 14, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("provider numbered episode clears stale debt", func(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
episodes := newFakeEpisodeRepo()
|
||||
debts := newFakeRefreshDebtRepo()
|
||||
h.service.episodeRepo = episodes
|
||||
h.service.refreshDebtRepo = debts
|
||||
h.itemRepo.items["series-provider"] = &models.MediaItem{
|
||||
ContentID: "series-provider",
|
||||
Type: "series",
|
||||
EpisodeMetadataIncomplete: true,
|
||||
}
|
||||
if err := episodes.Upsert(ctx, &models.Episode{
|
||||
ContentID: "episode-provider",
|
||||
SeriesID: "series-provider",
|
||||
SeasonID: "season-provider",
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Title: "Episode 1",
|
||||
TvdbID: "9607609",
|
||||
MetadataSource: "provider",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed provider episode: %v", err)
|
||||
}
|
||||
debts.debts[fakeRefreshDebtKey(RefreshTargetEpisode, "episode-provider")] = &models.MetadataRefreshDebt{
|
||||
TargetType: RefreshTargetEpisode,
|
||||
ContentID: "episode-provider",
|
||||
ReasonMask: RefreshDebtReasonEpisodeIncomplete,
|
||||
}
|
||||
|
||||
h.service.refreshSeriesEpisodeMetadataState(ctx, "series-provider", now)
|
||||
|
||||
item, err := h.itemRepo.GetByID(ctx, "series-provider")
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID: %v", err)
|
||||
}
|
||||
if item.EpisodeMetadataIncomplete {
|
||||
t.Fatal("expected provider numbered episode to clear series incomplete flag")
|
||||
}
|
||||
if item.EpisodeMetadataLastCheckedAt == nil || !item.EpisodeMetadataLastCheckedAt.Equal(now) {
|
||||
t.Fatalf("last checked at = %v, want %v", item.EpisodeMetadataLastCheckedAt, now)
|
||||
}
|
||||
if _, err := debts.GetTarget(ctx, RefreshTargetEpisode, "episode-provider"); !errors.Is(err, ErrRefreshDebtNotFound) {
|
||||
t.Fatalf("provider episode debt after refresh = %v, want ErrRefreshDebtNotFound", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("scanner fallback episode creates debt", func(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
episodes := newFakeEpisodeRepo()
|
||||
debts := newFakeRefreshDebtRepo()
|
||||
h.service.episodeRepo = episodes
|
||||
h.service.refreshDebtRepo = debts
|
||||
h.itemRepo.items["series-fallback"] = &models.MediaItem{
|
||||
ContentID: "series-fallback",
|
||||
Type: "series",
|
||||
}
|
||||
if err := episodes.Upsert(ctx, &models.Episode{
|
||||
ContentID: "episode-fallback",
|
||||
SeriesID: "series-fallback",
|
||||
SeasonID: "season-fallback",
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Title: "Episode 1",
|
||||
MetadataSource: "scanner_fallback",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fallback episode: %v", err)
|
||||
}
|
||||
|
||||
h.service.refreshSeriesEpisodeMetadataState(ctx, "series-fallback", now)
|
||||
|
||||
item, err := h.itemRepo.GetByID(ctx, "series-fallback")
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID: %v", err)
|
||||
}
|
||||
if !item.EpisodeMetadataIncomplete {
|
||||
t.Fatal("expected fallback episode to keep series incomplete flag")
|
||||
}
|
||||
debt, err := debts.GetTarget(ctx, RefreshTargetEpisode, "episode-fallback")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTarget fallback debt: %v", err)
|
||||
}
|
||||
if !hasRefreshDebtReason(debt.ReasonMask, RefreshDebtReasonEpisodeIncomplete) {
|
||||
t.Fatalf("fallback debt reason mask = %d, want episode incomplete", debt.ReasonMask)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestStaleMetadataRefreshStartsOnDemandRefreshOnce(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -3,6 +3,7 @@ package metadata
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -55,10 +56,39 @@ func (r *StaleMediaIDRepository) Upsert(ctx context.Context, contentID, provider
|
||||
SET provider_id = EXCLUDED.provider_id,
|
||||
last_seen_at = NOW()
|
||||
`, contentID, provider, providerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upserting stale media ID: %w", err)
|
||||
return resolveStaleUpsertError(err, contentID, provider)
|
||||
}
|
||||
|
||||
// staleMediaIDContentFKConstraint is the Postgres-auto-generated name of the
|
||||
// stale_media_ids.content_id → media_items(content_id) foreign key, declared
|
||||
// inline in migrations/sql/036_stale_media_ids.sql (hence "{table}_{column}_fkey").
|
||||
const staleMediaIDContentFKConstraint = "stale_media_ids_content_id_fkey"
|
||||
|
||||
// resolveStaleUpsertError maps an Upsert error to the value Upsert should
|
||||
// return. The referenced media item can be deleted or merged away (e.g. by
|
||||
// provider-ID canonicalization) between a metadata refresh starting and its
|
||||
// 404 landing here; the parent row is then gone and there is nothing left to
|
||||
// track, so the resulting foreign-key violation is a logged no-op. Every other
|
||||
// error is wrapped and propagated.
|
||||
func resolveStaleUpsertError(err error, contentID, provider string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
if isMissingContentForeignKeyViolation(err) {
|
||||
slog.Info("metadata: skipping stale media ID for missing content item",
|
||||
"content_id", contentID,
|
||||
"provider", provider,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("upserting stale media ID: %w", err)
|
||||
}
|
||||
|
||||
// isMissingContentForeignKeyViolation reports whether err is a Postgres
|
||||
// foreign-key violation against stale_media_ids.content_id — i.e. the media
|
||||
// item the stale ID would reference no longer exists.
|
||||
func isMissingContentForeignKeyViolation(err error) bool {
|
||||
return isPgConstraintViolation(err, "23503", staleMediaIDContentFKConstraint)
|
||||
}
|
||||
|
||||
// GetByContentID loads all stale external ID records for a single item.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func TestIsMissingContentForeignKeyViolation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"generic", errors.New("boom"), false},
|
||||
{
|
||||
"unique violation",
|
||||
&pgconn.PgError{Code: "23505", ConstraintName: "stale_media_ids_pkey"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"fk violation on a different constraint",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "some_other_fkey"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"fk violation on the content_id fkey",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"wrapped fk violation on the content_id fkey",
|
||||
fmt.Errorf("upserting stale media ID: %w",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"}),
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isMissingContentForeignKeyViolation(tc.err); got != tc.want {
|
||||
t.Fatalf("isMissingContentForeignKeyViolation(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPgConstraintViolation(t *testing.T) {
|
||||
const code = "23503"
|
||||
const constraint = "some_fkey"
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"generic", errors.New("boom"), false},
|
||||
{"wrong code", &pgconn.PgError{Code: "23505", ConstraintName: constraint}, false},
|
||||
{"wrong constraint", &pgconn.PgError{Code: code, ConstraintName: "other"}, false},
|
||||
{"match", &pgconn.PgError{Code: code, ConstraintName: constraint}, true},
|
||||
{"wrapped match", fmt.Errorf("x: %w", &pgconn.PgError{Code: code, ConstraintName: constraint}), true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isPgConstraintViolation(tc.err, code, constraint); got != tc.want {
|
||||
t.Fatalf("isPgConstraintViolation(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveStaleUpsertError(t *testing.T) {
|
||||
// The benign missing-parent race is swallowed (nil); every other error is
|
||||
// wrapped and propagated so a real failure still surfaces.
|
||||
if err := resolveStaleUpsertError(nil, "c", "tmdb"); err != nil {
|
||||
t.Fatalf("resolveStaleUpsertError(nil) = %v, want nil", err)
|
||||
}
|
||||
|
||||
fkErr := &pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"}
|
||||
if err := resolveStaleUpsertError(fkErr, "c", "tmdb"); err != nil {
|
||||
t.Fatalf("resolveStaleUpsertError(missing-content FK) = %v, want nil", err)
|
||||
}
|
||||
|
||||
other := &pgconn.PgError{Code: "23505", ConstraintName: "stale_media_ids_pkey"}
|
||||
if err := resolveStaleUpsertError(other, "c", "tmdb"); err == nil {
|
||||
t.Fatal("resolveStaleUpsertError(other pg error) = nil, want wrapped error")
|
||||
}
|
||||
|
||||
generic := errors.New("connection reset")
|
||||
if err := resolveStaleUpsertError(generic, "c", "tmdb"); err == nil ||
|
||||
!errors.Is(err, generic) {
|
||||
t.Fatalf("resolveStaleUpsertError(generic) = %v, want wrapped generic", err)
|
||||
}
|
||||
}
|
||||
@@ -212,6 +212,8 @@ const (
|
||||
PersonKindProducer PersonKind = 4
|
||||
PersonKindGuestStar PersonKind = 5
|
||||
PersonKindComposer PersonKind = 6
|
||||
PersonKindAuthor PersonKind = 7
|
||||
PersonKindNarrator PersonKind = 8
|
||||
)
|
||||
|
||||
// String returns the Jellyfin-compatible type string for this PersonKind.
|
||||
@@ -229,6 +231,10 @@ func (k PersonKind) String() string {
|
||||
return "GuestStar"
|
||||
case PersonKindComposer:
|
||||
return "Composer"
|
||||
case PersonKindAuthor:
|
||||
return "Author"
|
||||
case PersonKindNarrator:
|
||||
return "Narrator"
|
||||
default:
|
||||
return "Unknown"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
package models
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestPersonKindAudiobookRoles(t *testing.T) {
|
||||
cases := []struct {
|
||||
kind PersonKind
|
||||
want string
|
||||
}{
|
||||
{PersonKindAuthor, "Author"},
|
||||
{PersonKindNarrator, "Narrator"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := tc.kind.String(); got != tc.want {
|
||||
t.Errorf("%d.String() = %q, want %q", tc.kind, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -43,9 +43,6 @@ func (n *MarkerUpdateNotifier) MarkersUpdated(_ context.Context, file *models.Me
|
||||
credits := rangePayload(file.CreditsStart, file.CreditsEnd)
|
||||
recap := rangePayload(file.RecapStart, file.RecapEnd)
|
||||
preview := rangePayload(file.PreviewStart, file.PreviewEnd)
|
||||
if intro == nil && credits == nil && recap == nil && preview == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, session := range n.sessions.GetSessionsByMediaFileID(file.ID) {
|
||||
if session == nil || session.ID == "" || !session.HasRealtimeConnection {
|
||||
|
||||
@@ -82,3 +82,32 @@ func TestMarkerUpdateNotifierTargetsMatchingSessions(t *testing.T) {
|
||||
t.Fatalf("payload.Credits = %#v, want credits range", payload.Credits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerUpdateNotifierSendsAllClearedMarkers(t *testing.T) {
|
||||
sessions := NewSessionManager(0, 0)
|
||||
session, _ := sessions.StartSession(1, "profile-a", 100, PlayDirect, false)
|
||||
_ = sessions.SetRealtimeConnection(session.ID, true)
|
||||
|
||||
hub := NewRealtimeHub()
|
||||
conn := &dispatchTestConn{}
|
||||
reg := hub.Register(session.ID, conn)
|
||||
defer hub.Unregister(reg)
|
||||
|
||||
notifier := NewMarkerUpdateNotifier(sessions, hub)
|
||||
notifier.MarkersUpdated(context.Background(), &models.MediaFile{ID: 100})
|
||||
|
||||
if len(conn.messages) != 1 {
|
||||
t.Fatalf("messages = %d, want 1 all-cleared marker update", len(conn.messages))
|
||||
}
|
||||
event, ok := conn.messages[0].(EventEnvelope)
|
||||
if !ok {
|
||||
t.Fatalf("message type = %T, want EventEnvelope", conn.messages[0])
|
||||
}
|
||||
var payload MarkersUpdatedPayload
|
||||
if err := json.Unmarshal(event.Payload, &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal(payload): %v", err)
|
||||
}
|
||||
if payload.Intro != nil || payload.Credits != nil || payload.Recap != nil || payload.Preview != nil {
|
||||
t.Fatalf("payload markers = %#v, want all nil", payload)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,10 +93,11 @@ type SegmentProgress struct {
|
||||
// SegmentRecoveryDecision tells the segment handler whether to briefly wait
|
||||
// for ffmpeg or seek-restart immediately.
|
||||
type SegmentRecoveryDecision struct {
|
||||
Wait bool
|
||||
WaitTimeout time.Duration
|
||||
Reason string
|
||||
Progress SegmentProgress
|
||||
Wait bool
|
||||
WaitTimeout time.Duration
|
||||
RestartOnTimeout bool
|
||||
Reason string
|
||||
Progress SegmentProgress
|
||||
}
|
||||
|
||||
// defaultSegmentDuration is the segment length when not specified. Short
|
||||
@@ -109,6 +110,7 @@ const maxPersistedFFmpegChars = 2000
|
||||
|
||||
const (
|
||||
maxSequentialMissingSegments = 2
|
||||
activeSegmentWait = 12 * time.Second
|
||||
segmentWaitGrace = 1500 * time.Millisecond
|
||||
maxSegmentWait = 6 * time.Second
|
||||
minSegmentWait = 3 * time.Second
|
||||
@@ -1096,8 +1098,9 @@ func (s *TranscodeSession) SegmentProgress(time.Time) SegmentProgress {
|
||||
func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) SegmentRecoveryDecision {
|
||||
progress := s.SegmentProgress(now)
|
||||
decision := SegmentRecoveryDecision{
|
||||
WaitTimeout: segmentWaitTimeout(progress.SegmentDuration),
|
||||
Progress: progress,
|
||||
WaitTimeout: segmentWaitTimeout(progress.SegmentDuration),
|
||||
RestartOnTimeout: true,
|
||||
Progress: progress,
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -1112,6 +1115,8 @@ func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) Se
|
||||
case !progress.HasManifest:
|
||||
if segNum <= progress.StartSegmentNumber+1 {
|
||||
decision.Wait = true
|
||||
decision.WaitTimeout = activeSegmentWait
|
||||
decision.RestartOnTimeout = false
|
||||
decision.Reason = "startup_manifest_not_ready"
|
||||
} else {
|
||||
decision.Reason = "startup_request_beyond_window"
|
||||
@@ -1122,6 +1127,8 @@ func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) Se
|
||||
decision.Reason = "produced_output_stale"
|
||||
default:
|
||||
decision.Wait = true
|
||||
decision.WaitTimeout = activeSegmentWait
|
||||
decision.RestartOnTimeout = false
|
||||
decision.Reason = "near_produced_head"
|
||||
}
|
||||
|
||||
|
||||
@@ -486,12 +486,44 @@ func TestSegmentRecoveryDecisionWaitsForFreshNextSegment(t *testing.T) {
|
||||
if !decision.Wait {
|
||||
t.Fatalf("Wait = false, want true (reason=%s)", decision.Reason)
|
||||
}
|
||||
if decision.WaitTimeout != 3500*time.Millisecond {
|
||||
t.Fatalf("WaitTimeout = %s, want 3.5s", decision.WaitTimeout)
|
||||
if decision.WaitTimeout != 12*time.Second {
|
||||
t.Fatalf("WaitTimeout = %s, want 12s", decision.WaitTimeout)
|
||||
}
|
||||
if decision.Reason != "near_produced_head" {
|
||||
t.Fatalf("Reason = %q, want near_produced_head", decision.Reason)
|
||||
}
|
||||
if decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRecoveryDecisionUsesLongerWaitForStartupSegment(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
session := &TranscodeSession{
|
||||
outputDir: tempDir,
|
||||
running: true,
|
||||
opts: TranscodeOpts{
|
||||
TargetCodecVideo: "h264",
|
||||
SegmentDuration: 2,
|
||||
StartSegmentNumber: 0,
|
||||
},
|
||||
}
|
||||
|
||||
decision := session.SegmentRecoveryDecision(0, now)
|
||||
if !decision.Wait {
|
||||
t.Fatalf("Wait = false, want true (reason=%s)", decision.Reason)
|
||||
}
|
||||
if decision.WaitTimeout != 12*time.Second {
|
||||
t.Fatalf("WaitTimeout = %s, want 12s", decision.WaitTimeout)
|
||||
}
|
||||
if decision.Reason != "startup_manifest_not_ready" {
|
||||
t.Fatalf("Reason = %q, want startup_manifest_not_ready", decision.Reason)
|
||||
}
|
||||
if decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRecoveryDecisionRestartsWhenProducedOutputIsStale(t *testing.T) {
|
||||
@@ -516,6 +548,9 @@ func TestSegmentRecoveryDecisionRestartsWhenProducedOutputIsStale(t *testing.T)
|
||||
if decision.Wait {
|
||||
t.Fatal("Wait = true, want false")
|
||||
}
|
||||
if !decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = false, want true")
|
||||
}
|
||||
if decision.Reason != "produced_output_stale" {
|
||||
t.Fatalf("Reason = %q, want produced_output_stale", decision.Reason)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,11 @@ type MetadataProviderClient struct {
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type MarkerProviderClient struct {
|
||||
client pluginv1.MarkerProviderClient
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type MediaAnalyzerClient struct {
|
||||
client pluginv1.MediaAnalyzerClient
|
||||
timeout time.Duration
|
||||
@@ -103,6 +108,16 @@ func (c *Client) MetadataProvider(capabilityID string) (*MetadataProviderClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) MarkerProvider(capabilityID string) (*MarkerProviderClient, error) {
|
||||
if err := c.requireCapability("marker_provider.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MarkerProviderClient{
|
||||
client: c.rpc.MarkerProvider(),
|
||||
timeout: DefaultMarkerProviderTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) MediaAnalyzer(capabilityID string) (*MediaAnalyzerClient, error) {
|
||||
if err := c.requireCapability("media_analyzer.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
@@ -232,6 +247,24 @@ func (c *MetadataProviderClient) ResolveImageURLs(ctx context.Context, req *plug
|
||||
return c.client.ResolveImageURLs(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) FetchMarkers(ctx context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.FetchMarkers(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) SubmitMarker(ctx context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.SubmitMarker(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) GetMarkerProviderStats(ctx context.Context, req *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.GetMarkerProviderStats(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MediaAnalyzerClient) Analyze(ctx context.Context, req *pluginv1.AnalyzeMediaRequest) (*pluginv1.AnalyzeMediaResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -105,3 +105,42 @@ func TestClient_ScanSource_CapabilityGate(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestClient_MarkerProvider_CapabilityGate(t *testing.T) {
|
||||
t.Run("absent capability returns error", func(t *testing.T) {
|
||||
c := makeTestClient(t, nil)
|
||||
_, err := c.MarkerProvider("missing")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing marker_provider.v1 capability, got nil")
|
||||
}
|
||||
if !errors.Is(err, ErrCapabilityNotFound) {
|
||||
t.Errorf("expected ErrCapabilityNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("declared capability returns no error", func(t *testing.T) {
|
||||
c := makeTestClient(t, []*pluginv1.CapabilityDescriptor{
|
||||
{Type: "marker_provider.v1", Id: "markers"},
|
||||
})
|
||||
got, err := c.MarkerProvider("markers")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected non-nil MarkerProviderClient")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wrong id returns error", func(t *testing.T) {
|
||||
c := makeTestClient(t, []*pluginv1.CapabilityDescriptor{
|
||||
{Type: "marker_provider.v1", Id: "markers"},
|
||||
})
|
||||
_, err := c.MarkerProvider("other")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for mismatched marker_provider.v1 capability id, got nil")
|
||||
}
|
||||
if !errors.Is(err, ErrCapabilityNotFound) {
|
||||
t.Errorf("expected ErrCapabilityNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,11 +9,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultHealthCheckInterval = 30 * time.Second
|
||||
DefaultHealthFailureLimit = 3
|
||||
DefaultMetadataTimeout = 30 * time.Second
|
||||
DefaultAnalyzerTimeout = 5 * time.Minute
|
||||
DefaultControlTimeout = 10 * time.Second
|
||||
DefaultHealthCheckInterval = 30 * time.Second
|
||||
DefaultHealthFailureLimit = 3
|
||||
DefaultMetadataTimeout = 30 * time.Second
|
||||
DefaultMarkerProviderTimeout = 30 * time.Second
|
||||
DefaultAnalyzerTimeout = 5 * time.Minute
|
||||
DefaultControlTimeout = 10 * time.Second
|
||||
// DefaultScanSourceTimeout covers a single PollChanges call. Most scan
|
||||
// sources return quickly, but filesystem-backed sources may need a longer
|
||||
// bounded window for an initial baseline; cooperative plugins should still
|
||||
|
||||
+62
-15
@@ -9,6 +9,8 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -21,6 +23,7 @@ import (
|
||||
type pluginClient interface {
|
||||
Manifest() *pluginv1.PluginManifest
|
||||
MetadataProvider(capabilityID string) (*pluginhost.MetadataProviderClient, error)
|
||||
MarkerProvider(capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
MediaAnalyzer(capabilityID string) (*pluginhost.MediaAnalyzerClient, error)
|
||||
ScheduledTask(capabilityID string) (*pluginhost.ScheduledTaskClient, error)
|
||||
ScanSource(capabilityID string) (*pluginhost.ScanSourceClient, error)
|
||||
@@ -52,15 +55,17 @@ type serviceConfigStore interface {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
repositories *RepositoryStore
|
||||
installations serviceInstallationStore
|
||||
configs serviceConfigStore
|
||||
catalog *CatalogService
|
||||
installer *Installer
|
||||
archiveCache *ArchiveCache
|
||||
host Host
|
||||
testConfigSeq atomic.Int64
|
||||
dispatcher *EventDispatcher
|
||||
repositories *RepositoryStore
|
||||
installations serviceInstallationStore
|
||||
configs serviceConfigStore
|
||||
catalog *CatalogService
|
||||
installer *Installer
|
||||
archiveCache *ArchiveCache
|
||||
host Host
|
||||
testConfigSeq atomic.Int64
|
||||
dispatcher *EventDispatcher
|
||||
lifecycleMu sync.RWMutex
|
||||
lifecycleHooks []func(context.Context)
|
||||
}
|
||||
|
||||
// SetEventDispatcher wires the EventDispatcher into the Service. The
|
||||
@@ -69,12 +74,42 @@ type Service struct {
|
||||
// per-event store reads and needs no notification on install/enable/disable.
|
||||
func (s *Service) SetEventDispatcher(d *EventDispatcher) { s.dispatcher = d }
|
||||
|
||||
// OnLifecycleChange is a public hook invoked by API handlers and the installer
|
||||
// after every install / enable / disable / uninstall. It currently does
|
||||
// nothing — the dispatcher reads the live installation set on each event —
|
||||
// but the entry point is preserved so future cache-warming logic has a place
|
||||
// to attach without touching every call site again.
|
||||
func (s *Service) OnLifecycleChange(_ context.Context) {}
|
||||
// AddLifecycleHook registers a callback invoked after plugin install, enable,
|
||||
// disable, uninstall, or preload lifecycle changes.
|
||||
func (s *Service) AddLifecycleHook(hook func(context.Context)) {
|
||||
if s == nil || hook == nil {
|
||||
return
|
||||
}
|
||||
s.lifecycleMu.Lock()
|
||||
s.lifecycleHooks = append(s.lifecycleHooks, hook)
|
||||
s.lifecycleMu.Unlock()
|
||||
}
|
||||
|
||||
// OnLifecycleChange is invoked by API handlers and the installer after every
|
||||
// plugin lifecycle mutation. Hooks should be best-effort and log their own
|
||||
// errors so plugin admin operations are not failed by secondary cache refreshes.
|
||||
func (s *Service) OnLifecycleChange(ctx context.Context) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.lifecycleMu.RLock()
|
||||
hooks := append([]func(context.Context){}, s.lifecycleHooks...)
|
||||
s.lifecycleMu.RUnlock()
|
||||
for _, hook := range hooks {
|
||||
func(hook func(context.Context)) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
slog.Error(
|
||||
"plugin lifecycle hook panicked; continuing",
|
||||
"panic", recovered,
|
||||
"stack", string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}()
|
||||
hook(ctx)
|
||||
}(hook)
|
||||
}
|
||||
}
|
||||
|
||||
func NewService(
|
||||
repositories *RepositoryStore,
|
||||
@@ -411,6 +446,18 @@ func (s *Service) MetadataProviderClient(
|
||||
return client.MetadataProvider(capabilityID)
|
||||
}
|
||||
|
||||
func (s *Service) MarkerProviderClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
capabilityID string,
|
||||
) (*pluginhost.MarkerProviderClient, error) {
|
||||
client, err := s.ensureClient(ctx, installationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.MarkerProvider(capabilityID)
|
||||
}
|
||||
|
||||
func (s *Service) ScheduledTaskClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
|
||||
@@ -199,6 +199,10 @@ func (f *fakePluginClient) MetadataProvider(string) (*pluginhost.MetadataProvide
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginClient) MarkerProvider(string) (*pluginhost.MarkerProviderClient, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginClient) MediaAnalyzer(string) (*pluginhost.MediaAnalyzerClient, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestServiceLifecycleChangeContinuesAfterHookPanic(t *testing.T) {
|
||||
var service Service
|
||||
called := false
|
||||
service.AddLifecycleHook(func(context.Context) {
|
||||
panic("boom")
|
||||
})
|
||||
service.AddLifecycleHook(func(context.Context) {
|
||||
called = true
|
||||
})
|
||||
|
||||
service.OnLifecycleChange(context.Background())
|
||||
|
||||
if !called {
|
||||
t.Fatal("expected later lifecycle hook to run after earlier hook panicked")
|
||||
}
|
||||
}
|
||||
@@ -222,10 +222,11 @@ func (s *Service) GetDetail(ctx context.Context, viewer Viewer, mediaType MediaT
|
||||
return nil, err
|
||||
}
|
||||
|
||||
primaryAvailable, err := s.lookupAvailable(ctx, mediaType, []int{raw.ID})
|
||||
primaryPresence, err := s.lookupAvailable(ctx, mediaType, []int{raw.ID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
primaryMatch := primaryPresence[raw.ID]
|
||||
primaryRequests, err := s.store.ListActiveByTMDB(ctx, mediaType, []int{raw.ID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -258,8 +259,9 @@ func (s *Service) GetDetail(ctx context.Context, viewer Viewer, mediaType MediaT
|
||||
Networks: raw.Networks,
|
||||
Director: raw.Director,
|
||||
Creators: raw.Creators,
|
||||
Availability: availabilityValue(primaryAvailable[raw.ID]),
|
||||
Request: requestStateFor(viewer, policy, primaryAvailable[raw.ID], primaryRequests[raw.ID]),
|
||||
Availability: availabilityValue(primaryMatch.Available),
|
||||
LibraryContentID: primaryMatch.ContentID,
|
||||
Request: requestStateFor(viewer, policy, primaryMatch.Available, primaryRequests[raw.ID]),
|
||||
}
|
||||
if raw.TVDBID > 0 {
|
||||
tvdb := raw.TVDBID
|
||||
@@ -397,6 +399,9 @@ func (s *Service) ListMine(ctx context.Context, viewer Viewer, filter ListFilter
|
||||
if err := s.attachTargets(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reqs, nil
|
||||
}
|
||||
|
||||
@@ -411,6 +416,9 @@ func (s *Service) ListAdmin(ctx context.Context, viewer Viewer, filter ListFilte
|
||||
if err := s.attachTargets(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reqs, nil
|
||||
}
|
||||
|
||||
@@ -430,6 +438,49 @@ func (s *Service) attachTargets(ctx context.Context, reqs ...*Request) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) attachLibraryContent(ctx context.Context, reqs ...*Request) error {
|
||||
if s == nil || s.presence == nil || len(reqs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
type requestKey struct {
|
||||
mediaType MediaType
|
||||
tmdbID int
|
||||
}
|
||||
|
||||
candidatesByType := map[MediaType][]PresenceCandidate{}
|
||||
requestsByKey := map[requestKey][]*Request{}
|
||||
seen := map[requestKey]bool{}
|
||||
for _, req := range reqs {
|
||||
if req == nil || req.TMDBID <= 0 {
|
||||
continue
|
||||
}
|
||||
key := requestKey{mediaType: req.MediaType, tmdbID: req.TMDBID}
|
||||
requestsByKey[key] = append(requestsByKey[key], req)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
candidatesByType[req.MediaType] = append(candidatesByType[req.MediaType], requestPresenceCandidate(*req))
|
||||
}
|
||||
|
||||
for mediaType, candidates := range candidatesByType {
|
||||
matches, err := s.lookupPresence(ctx, mediaType, candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for tmdbID, match := range matches {
|
||||
if !match.Available || strings.TrimSpace(match.ContentID) == "" {
|
||||
continue
|
||||
}
|
||||
for _, req := range requestsByKey[requestKey{mediaType: mediaType, tmdbID: tmdbID}] {
|
||||
req.LibraryContentID = match.ContentID
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) GetRequest(ctx context.Context, viewer Viewer, id string) (*Request, error) {
|
||||
if err := s.ensureRequestsEnabled(ctx); err != nil {
|
||||
return nil, err
|
||||
@@ -444,6 +495,9 @@ func (s *Service) GetRequest(ctx context.Context, viewer Viewer, id string) (*Re
|
||||
if err := s.attachTargets(ctx, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
@@ -905,7 +959,7 @@ func (s *Service) enrichPage(ctx context.Context, viewer Viewer, raw *tmdb.Media
|
||||
idsByType[mediaType] = append(idsByType[mediaType], item.ID)
|
||||
}
|
||||
|
||||
available := map[MediaType]map[int]bool{}
|
||||
available := map[MediaType]map[int]PresenceMatch{}
|
||||
active := map[MediaType]map[int]*Request{}
|
||||
for mediaType, ids := range idsByType {
|
||||
presence, err := s.lookupAvailable(ctx, mediaType, ids)
|
||||
@@ -931,21 +985,22 @@ func (s *Service) enrichPage(ctx context.Context, viewer Viewer, raw *tmdb.Media
|
||||
if err != nil || item.ID <= 0 {
|
||||
continue
|
||||
}
|
||||
isAvailable := available[mediaType][item.ID]
|
||||
match := available[mediaType][item.ID]
|
||||
activeRequest := active[mediaType][item.ID]
|
||||
out.Results = append(out.Results, MediaResult{
|
||||
MediaType: mediaType,
|
||||
TMDBID: item.ID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterPath: item.PosterPath,
|
||||
BackdropPath: item.BackdropPath,
|
||||
ReleaseDate: item.ReleaseDate,
|
||||
Popularity: item.Popularity,
|
||||
VoteAverage: item.VoteAverage,
|
||||
Availability: availabilityValue(isAvailable),
|
||||
Request: requestStateFor(viewer, policy, isAvailable, activeRequest),
|
||||
MediaType: mediaType,
|
||||
TMDBID: item.ID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterPath: item.PosterPath,
|
||||
BackdropPath: item.BackdropPath,
|
||||
ReleaseDate: item.ReleaseDate,
|
||||
Popularity: item.Popularity,
|
||||
VoteAverage: item.VoteAverage,
|
||||
Availability: availabilityValue(match.Available),
|
||||
LibraryContentID: match.ContentID,
|
||||
Request: requestStateFor(viewer, policy, match.Available, activeRequest),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
@@ -958,14 +1013,6 @@ func (s *Service) lookupPresence(ctx context.Context, mediaType MediaType, candi
|
||||
return s.presence.Lookup(ctx, mediaType, candidates)
|
||||
}
|
||||
|
||||
func availabilityBoolMap(matches map[int]PresenceMatch) map[int]bool {
|
||||
out := map[int]bool{}
|
||||
for id, match := range matches {
|
||||
out[id] = match.Available
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func requestPresenceCandidate(req Request) PresenceCandidate {
|
||||
candidate := PresenceCandidate{
|
||||
TMDBID: req.TMDBID,
|
||||
@@ -1057,9 +1104,9 @@ func tmdbMediaType(mediaType MediaType) string {
|
||||
return "movie"
|
||||
}
|
||||
|
||||
func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids []int) (map[int]bool, error) {
|
||||
func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids []int) (map[int]PresenceMatch, error) {
|
||||
if s.presence == nil {
|
||||
return map[int]bool{}, nil
|
||||
return map[int]PresenceMatch{}, nil
|
||||
}
|
||||
candidates := make([]PresenceCandidate, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
@@ -1072,7 +1119,7 @@ func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return availabilityBoolMap(matches), nil
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
func (s *Service) enrichExternalIDs(ctx context.Context, input *CreateRequestInput) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package requests
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -353,6 +354,34 @@ func TestListMineAttachesTargets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMineAttachesLibraryContentID(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.mine = []*Request{{
|
||||
ID: "req-1",
|
||||
Provider: "tmdb",
|
||||
MediaType: MediaTypeMovie,
|
||||
TMDBID: 42,
|
||||
Title: "Test Movie",
|
||||
Status: StatusCompleted,
|
||||
Outcome: OutcomeActive,
|
||||
RequestedByUserID: 1,
|
||||
}}
|
||||
presence := &fakePresence{available: map[MediaType]map[int]bool{
|
||||
MediaTypeMovie: {42: true},
|
||||
}}
|
||||
|
||||
got, err := NewService(store, &fakeTMDBClient{}, presence).ListMine(context.Background(), testViewer(1), ListFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListMine returned error: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("ListMine returned %d requests, want 1", len(got))
|
||||
}
|
||||
if got[0].LibraryContentID != "movie-42" {
|
||||
t.Fatalf("library content id = %q, want movie-42", got[0].LibraryContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRequestBlocksWhenHydratedTVDBIDIsAvailable(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.settings.RequestsEnabled = true
|
||||
@@ -460,6 +489,9 @@ func TestSearchMarksSeriesAvailableByHydratedTVDBID(t *testing.T) {
|
||||
if got := page.Results[0].Availability; got != AvailabilityAvailable {
|
||||
t.Fatalf("availability = %q, want available", got)
|
||||
}
|
||||
if got := page.Results[0].LibraryContentID; got != "series-201992" {
|
||||
t.Fatalf("library content id = %q, want series-201992", got)
|
||||
}
|
||||
if page.Results[0].Request.Reason != "already_available" {
|
||||
t.Fatalf("request reason = %q, want already_available", page.Results[0].Request.Reason)
|
||||
}
|
||||
@@ -1594,18 +1626,30 @@ func (f *fakePresence) Lookup(_ context.Context, mediaType MediaType, candidates
|
||||
f.got = append(f.got, candidates...)
|
||||
for _, candidate := range candidates {
|
||||
if f.available != nil && f.available[mediaType][candidate.TMDBID] {
|
||||
out[candidate.TMDBID] = PresenceMatch{Available: true, MatchedProvider: "tmdb"}
|
||||
out[candidate.TMDBID] = PresenceMatch{
|
||||
Available: true,
|
||||
ContentID: fakePresenceContentID(mediaType, candidate.TMDBID),
|
||||
MatchedProvider: "tmdb",
|
||||
}
|
||||
continue
|
||||
}
|
||||
if candidate.TVDBID != nil && f.byTVDB != nil {
|
||||
if tmdbID, ok := f.byTVDB[mediaType][*candidate.TVDBID]; ok && tmdbID == candidate.TMDBID {
|
||||
out[candidate.TMDBID] = PresenceMatch{Available: true, MatchedProvider: "tvdb"}
|
||||
out[candidate.TMDBID] = PresenceMatch{
|
||||
Available: true,
|
||||
ContentID: fakePresenceContentID(mediaType, candidate.TMDBID),
|
||||
MatchedProvider: "tvdb",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fakePresenceContentID(mediaType MediaType, tmdbID int) string {
|
||||
return fmt.Sprintf("%s-%d", mediaType, tmdbID)
|
||||
}
|
||||
|
||||
func (f *fakePresence) LookupTMDB(_ context.Context, mediaType MediaType, ids []int) (map[int]bool, error) {
|
||||
out := map[int]bool{}
|
||||
for _, id := range ids {
|
||||
|
||||
+15
-12
@@ -144,6 +144,7 @@ type Request struct {
|
||||
Targets []Target `json:"targets,omitempty"`
|
||||
ExternalID string `json:"external_id,omitempty"`
|
||||
ExternalStatus string `json:"external_status,omitempty"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
@@ -169,18 +170,19 @@ type RequestState struct {
|
||||
}
|
||||
|
||||
type MediaResult struct {
|
||||
MediaType MediaType `json:"media_type"`
|
||||
TMDBID int `json:"tmdb_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterPath string `json:"poster_path,omitempty"`
|
||||
BackdropPath string `json:"backdrop_path,omitempty"`
|
||||
ReleaseDate string `json:"release_date,omitempty"`
|
||||
Popularity float64 `json:"popularity,omitempty"`
|
||||
VoteAverage float64 `json:"vote_average,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
Request RequestState `json:"request"`
|
||||
MediaType MediaType `json:"media_type"`
|
||||
TMDBID int `json:"tmdb_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterPath string `json:"poster_path,omitempty"`
|
||||
BackdropPath string `json:"backdrop_path,omitempty"`
|
||||
ReleaseDate string `json:"release_date,omitempty"`
|
||||
Popularity float64 `json:"popularity,omitempty"`
|
||||
VoteAverage float64 `json:"vote_average,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
Request RequestState `json:"request"`
|
||||
}
|
||||
|
||||
type MediaPage struct {
|
||||
@@ -228,6 +230,7 @@ type MediaDetail struct {
|
||||
Creators []string `json:"creators,omitempty"`
|
||||
Recommendations []MediaResult `json:"recommendations,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
Request RequestState `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// audioExtensions is the set of file extensions the audiobook and podcast
|
||||
// scanner branches recognize. Mirrors videoExtensions for video libraries.
|
||||
var audioExtensions = map[string]bool{
|
||||
".m4b": true,
|
||||
".m4a": true,
|
||||
".mp3": true,
|
||||
".flac": true,
|
||||
".opus": true,
|
||||
".ogg": true,
|
||||
}
|
||||
|
||||
// SupportsAudioFile reports whether the given path uses a recognized audio
|
||||
// file extension.
|
||||
func SupportsAudioFile(filePath string) bool {
|
||||
if filePath == "" {
|
||||
return false
|
||||
}
|
||||
return audioExtensions[strings.ToLower(filepath.Ext(filePath))]
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
package scanner
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSupportsAudioFile(t *testing.T) {
|
||||
cases := []struct {
|
||||
path string
|
||||
want bool
|
||||
}{
|
||||
{"book.m4b", true},
|
||||
{"chapter1.mp3", true},
|
||||
{"audio.M4A", true},
|
||||
{"sample.flac", true},
|
||||
{"podcast.opus", true},
|
||||
{"track.ogg", true},
|
||||
{"poster.jpg", false},
|
||||
{"movie.mkv", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := SupportsAudioFile(tc.path); got != tc.want {
|
||||
t.Errorf("SupportsAudioFile(%q) = %v, want %v", tc.path, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// parsedAudiobook is the structured output of parseAudiobookFolder.
|
||||
// The scanner write path (Task 8) converts this into media_items +
|
||||
// media_files + item_people rows.
|
||||
type parsedAudiobook struct {
|
||||
Title string
|
||||
Author string
|
||||
Narrator string
|
||||
Series string
|
||||
Year int
|
||||
Files []parsedAudiobookFile
|
||||
}
|
||||
|
||||
// parsedAudiobookFile is one audio file belonging to a parsed audiobook.
|
||||
// For single-file .m4b audiobooks there is exactly one entry; for
|
||||
// multi-file folders (Task 7) there is one per file.
|
||||
type parsedAudiobookFile struct {
|
||||
Path string
|
||||
Chapters []ChapterInfo
|
||||
}
|
||||
|
||||
// parseAudiobookFolder reads a single audiobook folder and returns its
|
||||
// structured representation. Recognized layouts:
|
||||
// - one audio file in the folder, optionally with embedded chapters
|
||||
// - multiple audio files in the folder; each becomes its own
|
||||
// parsedAudiobookFile with a single synthesized chapter (title =
|
||||
// filename stem); metadata comes from the first file's tags
|
||||
//
|
||||
// Returns an error wrapping os.ErrNotExist when the folder contains zero
|
||||
// audio files, so the caller can skip it.
|
||||
func parseAudiobookFolder(ctx context.Context, ffprobePath string, folderPath string) (*parsedAudiobook, error) {
|
||||
entries, err := os.ReadDir(folderPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read audiobook folder %s: %w", folderPath, err)
|
||||
}
|
||||
|
||||
var audioFiles []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if SupportsAudioFile(entry.Name()) {
|
||||
audioFiles = append(audioFiles, filepath.Join(folderPath, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(audioFiles) == 0 {
|
||||
return nil, fmt.Errorf("audiobook folder %s: %w", folderPath, os.ErrNotExist)
|
||||
}
|
||||
sort.Strings(audioFiles)
|
||||
|
||||
book := &parsedAudiobook{}
|
||||
|
||||
if len(audioFiles) == 1 {
|
||||
probed, err := ProbeFile(ctx, ffprobePath, audioFiles[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe audiobook file %s: %w", audioFiles[0], err)
|
||||
}
|
||||
book.populateFromTags(probed.FormatTags)
|
||||
book.Files = []parsedAudiobookFile{{
|
||||
Path: audioFiles[0],
|
||||
Chapters: probed.Chapters,
|
||||
}}
|
||||
return book, nil
|
||||
}
|
||||
|
||||
// Multi-file case: read header from the first file, synthesize one
|
||||
// chapter per file with title = filename stem. A 0..0 range means the
|
||||
// chapter duration is unknown; consumers should treat it as non-seekable.
|
||||
probedFirst, err := ProbeFile(ctx, ffprobePath, audioFiles[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe first audiobook file %s: %w", audioFiles[0], err)
|
||||
}
|
||||
book.populateFromTags(probedFirst.FormatTags)
|
||||
|
||||
book.Files = make([]parsedAudiobookFile, 0, len(audioFiles))
|
||||
for i, path := range audioFiles {
|
||||
stem := strings.TrimSuffix(filepath.Base(path), filepath.Ext(path))
|
||||
book.Files = append(book.Files, parsedAudiobookFile{
|
||||
Path: path,
|
||||
Chapters: []ChapterInfo{{
|
||||
Index: i,
|
||||
Title: stem,
|
||||
StartSeconds: 0,
|
||||
EndSeconds: 0,
|
||||
}},
|
||||
})
|
||||
}
|
||||
return book, nil
|
||||
}
|
||||
|
||||
// populateFromTags fills the audiobook's header fields (Title, Author,
|
||||
// Narrator, Series, Year) from the ffprobe format tags. Tags are
|
||||
// lower-cased by normalizeFormatTags upstream.
|
||||
func (b *parsedAudiobook) populateFromTags(tags map[string]string) {
|
||||
b.Title = firstNonEmpty(tags["title"], tags["album"])
|
||||
b.Author = firstNonEmpty(tags["artist"], tags["album_artist"], tags["composer"])
|
||||
b.Narrator = firstNonEmpty(tags["narrator"], tags["performer"])
|
||||
b.Series = firstNonEmpty(tags["album"], tags["series"], tags["mvnm"])
|
||||
if year := firstNonEmpty(tags["date"], tags["year"]); year != "" {
|
||||
if y := parseTagYear(year); y > 0 {
|
||||
b.Year = y
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseTagYear extracts a 4-digit year (e.g. 1900-9999) from a tag value
|
||||
// that may be a bare year ("2024"), an ISO date ("2024-05-23"), or a
|
||||
// padded form ("(2024)"). Returns 0 if no plausible year is found.
|
||||
func parseTagYear(s string) int {
|
||||
s = strings.TrimSpace(s)
|
||||
for i := 0; i+4 <= len(s); i++ {
|
||||
candidate := s[i : i+4]
|
||||
if isAllDigits(candidate) {
|
||||
year := 0
|
||||
for _, c := range candidate {
|
||||
year = year*10 + int(c-'0')
|
||||
}
|
||||
if year >= 1900 && year <= 9999 {
|
||||
return year
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func isAllDigits(s string) bool {
|
||||
if s == "" {
|
||||
return false
|
||||
}
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/titleutil"
|
||||
)
|
||||
|
||||
type filesystemRootContentFinder interface {
|
||||
FindContentIDByRootPath(ctx context.Context, folderID int, rootPath, preferredType string) (string, error)
|
||||
}
|
||||
|
||||
type filesystemMediaItemWriter interface {
|
||||
Upsert(ctx context.Context, item *models.MediaItem) error
|
||||
}
|
||||
|
||||
// ScanAudiobookFolder walks an audiobooks-typed media folder and writes
|
||||
// one media_items row per subdirectory it can parse as an audiobook,
|
||||
// plus the corresponding media_files rows and author/narrator links in
|
||||
// item_people.
|
||||
//
|
||||
// Each immediate subdirectory of one of folder.Paths is treated as a
|
||||
// single audiobook. Subdirectories that contain zero audio files are
|
||||
// silently skipped (parseAudiobookFolder returns os.ErrNotExist).
|
||||
//
|
||||
// This bypasses the per-file movie/TV pipeline because audiobooks are
|
||||
// inherently folder-scoped (one book = one item, possibly multi-file).
|
||||
func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaFolder) error {
|
||||
if s == nil || folder == nil {
|
||||
return fmt.Errorf("ScanAudiobookFolder: nil scanner or folder")
|
||||
}
|
||||
|
||||
var attempted int
|
||||
var succeeded int
|
||||
var failures []error
|
||||
for _, root := range folder.Paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
slog.Warn("audiobook scan: read root failed", "root", root, "error", err)
|
||||
attempted++
|
||||
failures = append(failures, fmt.Errorf("read root %s: %w", root, err))
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
subPath := filepath.Join(root, entry.Name())
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
attempted++
|
||||
if err := s.reconcileAudiobookFolder(ctx, folder, subPath); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
}
|
||||
slog.Warn("audiobook scan: folder failed",
|
||||
"folder_id", folder.ID,
|
||||
"path", subPath,
|
||||
"error", err,
|
||||
)
|
||||
failures = append(failures, fmt.Errorf("%s: %w", subPath, err))
|
||||
// Continue with siblings — one bad audiobook should not stop the scan.
|
||||
continue
|
||||
}
|
||||
succeeded++
|
||||
}
|
||||
}
|
||||
if attempted > 0 && succeeded == 0 && len(failures) > 0 {
|
||||
return fmt.Errorf("audiobook scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) reconcileAudiobookFolder(ctx context.Context, folder *models.MediaFolder, folderPath string) error {
|
||||
parsed, err := parseAudiobookFolder(ctx, s.ffprobePath, folderPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("parse audiobook folder %s: %w", folderPath, err)
|
||||
}
|
||||
|
||||
contentID, err := s.upsertAudiobookMediaItem(ctx, folder.ID, folderPath, parsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert audiobook item: %w", err)
|
||||
}
|
||||
if err := s.upsertAudiobookMediaFiles(ctx, folder, contentID, folderPath, parsed); err != nil {
|
||||
return fmt.Errorf("upsert audiobook files: %w", err)
|
||||
}
|
||||
if err := s.upsertAudiobookPeople(ctx, contentID, parsed); err != nil {
|
||||
return fmt.Errorf("upsert audiobook people: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("audiobook scan: indexed",
|
||||
"folder_id", folder.ID,
|
||||
"content_id", contentID,
|
||||
"title", parsed.Title,
|
||||
"author", parsed.Author,
|
||||
"files", len(parsed.Files),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertAudiobookMediaItem reuses an item already linked to the same filesystem
|
||||
// root, or creates a new row. It intentionally does not dedupe by title/year:
|
||||
// audiobooks often share titles, unknown years, or edition-specific credits.
|
||||
// Returns the content_id used.
|
||||
func (s *Scanner) upsertAudiobookMediaItem(ctx context.Context, folderID int, folderPath string, book *parsedAudiobook) (string, error) {
|
||||
if s.itemRepo == nil {
|
||||
return "", fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
if s.fileRepo == nil {
|
||||
return "", fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
return resolveAudiobookMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, folderPath, book)
|
||||
}
|
||||
|
||||
func resolveAudiobookMediaItem(
|
||||
ctx context.Context,
|
||||
rootFinder filesystemRootContentFinder,
|
||||
itemWriter filesystemMediaItemWriter,
|
||||
folderID int,
|
||||
folderPath string,
|
||||
book *parsedAudiobook,
|
||||
) (string, error) {
|
||||
if rootFinder == nil {
|
||||
return "", fmt.Errorf("root content finder not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return "", fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, folderPath, "audiobook")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find audiobook by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
id, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate content_id: %w", err)
|
||||
}
|
||||
item := &models.MediaItem{
|
||||
ContentID: id,
|
||||
Type: "audiobook",
|
||||
Title: book.Title,
|
||||
SortTitle: titleutil.DeriveDefaultSortTitle(book.Title),
|
||||
Year: book.Year,
|
||||
}
|
||||
if err := itemWriter.Upsert(ctx, item); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// upsertAudiobookMediaFiles writes one media_files row per audio file in the
|
||||
// parsed audiobook. The content_id ties each file back to the media_items row.
|
||||
// folderPath is used as the canonical_root_path / observed_root_path.
|
||||
func (s *Scanner) upsertAudiobookMediaFiles(
|
||||
ctx context.Context,
|
||||
folder *models.MediaFolder,
|
||||
contentID string,
|
||||
folderPath string,
|
||||
book *parsedAudiobook,
|
||||
) error {
|
||||
for _, af := range book.Files {
|
||||
chapters := make([]models.MediaChapter, len(af.Chapters))
|
||||
for i, ch := range af.Chapters {
|
||||
chapters[i] = models.MediaChapter{
|
||||
Index: ch.Index,
|
||||
Title: ch.Title,
|
||||
StartSeconds: ch.StartSeconds,
|
||||
EndSeconds: ch.EndSeconds,
|
||||
Source: ch.Source,
|
||||
}
|
||||
}
|
||||
|
||||
mf := models.MediaFile{
|
||||
ContentID: contentID,
|
||||
MediaFolderID: folder.ID,
|
||||
CanonicalRootPath: folderPath,
|
||||
ObservedRootPath: folderPath,
|
||||
ContentGroupKey: contentID,
|
||||
GroupKeyVersion: 1,
|
||||
BaseTitle: book.Title,
|
||||
BaseYear: book.Year,
|
||||
BaseType: "audiobook",
|
||||
IdentityConfidence: audiobookIdentityConfidence(book, af),
|
||||
FilePath: af.Path,
|
||||
Chapters: chapters,
|
||||
ProbeSource: "local",
|
||||
}
|
||||
|
||||
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
|
||||
return fmt.Errorf("upsert media file %s: %w", af.Path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertAudiobookPeople upserts author and narrator rows into item_people,
|
||||
// using the PersonRepository to find-or-create each person by name. When the
|
||||
// parsed credits differ, ReplacePeople makes scanner-derived credits authoritative.
|
||||
// Skips the DELETE+INSERT entirely when the existing credit set already
|
||||
// matches the desired set (case-insensitive on name, exact on kind).
|
||||
func (s *Scanner) upsertAudiobookPeople(ctx context.Context, contentID string, book *parsedAudiobook) error {
|
||||
if s.personRepo == nil {
|
||||
return fmt.Errorf("personRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
type credit struct {
|
||||
name string
|
||||
kind models.PersonKind
|
||||
}
|
||||
var credits []credit
|
||||
if book.Author != "" {
|
||||
credits = append(credits, credit{book.Author, models.PersonKindAuthor})
|
||||
}
|
||||
if book.Narrator != "" {
|
||||
credits = append(credits, credit{book.Narrator, models.PersonKindNarrator})
|
||||
}
|
||||
if len(credits) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
people := make([]models.ItemPerson, 0, len(credits))
|
||||
for i, c := range credits {
|
||||
personID, err := s.personRepo.FindOrCreate(ctx, models.Person{Name: c.name})
|
||||
if err != nil {
|
||||
return fmt.Errorf("find-or-create person %q: %w", c.name, err)
|
||||
}
|
||||
people = append(people, models.ItemPerson{
|
||||
Person: models.Person{ID: personID},
|
||||
Kind: c.kind,
|
||||
SortOrder: i,
|
||||
})
|
||||
}
|
||||
|
||||
// ReplacePeople deletes existing credits then inserts the new set.
|
||||
// For audiobook scanning this is acceptable — we re-derive all credits
|
||||
// from the file metadata on every scan, so the set is authoritative.
|
||||
return s.itemRepo.ReplacePeople(ctx, contentID, people)
|
||||
}
|
||||
|
||||
func audiobookIdentityConfidence(book *parsedAudiobook, file parsedAudiobookFile) string {
|
||||
if book == nil {
|
||||
return "low"
|
||||
}
|
||||
|
||||
score := 0
|
||||
if book.Title != "" {
|
||||
score++
|
||||
}
|
||||
if book.Author != "" || book.Narrator != "" {
|
||||
score++
|
||||
}
|
||||
if book.Year > 0 {
|
||||
score++
|
||||
}
|
||||
for _, ch := range file.Chapters {
|
||||
if ch.Title != "" && ch.EndSeconds > ch.StartSeconds {
|
||||
score++
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case score >= 4:
|
||||
return "high"
|
||||
case score > 0:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakeRootContentFinder struct {
|
||||
contentID string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeRootContentFinder) FindContentIDByRootPath(context.Context, int, string, string) (string, error) {
|
||||
f.calls++
|
||||
return f.contentID, f.err
|
||||
}
|
||||
|
||||
type fakeFilesystemItemWriter struct {
|
||||
upserts []*models.MediaItem
|
||||
}
|
||||
|
||||
func (f *fakeFilesystemItemWriter) Upsert(_ context.Context, item *models.MediaItem) error {
|
||||
cp := *item
|
||||
f.upserts = append(f.upserts, &cp)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestParseAudiobookFolderSingleM4B(t *testing.T) {
|
||||
ffprobePath := FFprobePathFromFFmpeg("ffmpeg")
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
ffprobePath = "ffprobe"
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
t.Skip("ffprobe not available")
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
got, err := parseAudiobookFolder(ctx, ffprobePath, "testdata/audiobook_fixtures/single_book")
|
||||
if err != nil {
|
||||
t.Fatalf("parseAudiobookFolder: %v", err)
|
||||
}
|
||||
if got.Title != "Test Audiobook" {
|
||||
t.Errorf("Title = %q, want %q", got.Title, "Test Audiobook")
|
||||
}
|
||||
if got.Author != "Test Author" {
|
||||
t.Errorf("Author = %q, want %q", got.Author, "Test Author")
|
||||
}
|
||||
if got.Series != "Test Series" {
|
||||
t.Errorf("Series = %q, want %q", got.Series, "Test Series")
|
||||
}
|
||||
if got.Year != 2024 {
|
||||
t.Errorf("Year = %d, want 2024", got.Year)
|
||||
}
|
||||
if len(got.Files) != 1 {
|
||||
t.Fatalf("got %d files, want 1", len(got.Files))
|
||||
}
|
||||
if len(got.Files[0].Chapters) != 2 {
|
||||
t.Errorf("file 0 chapters = %d, want 2", len(got.Files[0].Chapters))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAudiobookFolderMultiFile(t *testing.T) {
|
||||
ffprobePath := FFprobePathFromFFmpeg("ffmpeg")
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
ffprobePath = "ffprobe"
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
t.Skip("ffprobe not available")
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
got, err := parseAudiobookFolder(ctx, ffprobePath, "testdata/audiobook_fixtures/multi_file")
|
||||
if err != nil {
|
||||
t.Fatalf("parseAudiobookFolder: %v", err)
|
||||
}
|
||||
if got.Title != "Multi File Test" {
|
||||
t.Errorf("Title = %q, want %q", got.Title, "Multi File Test")
|
||||
}
|
||||
if got.Author != "Multi Author" {
|
||||
t.Errorf("Author = %q, want %q", got.Author, "Multi Author")
|
||||
}
|
||||
if got.Year != 2023 {
|
||||
t.Errorf("Year = %d, want 2023", got.Year)
|
||||
}
|
||||
if len(got.Files) != 3 {
|
||||
t.Fatalf("got %d files, want 3", len(got.Files))
|
||||
}
|
||||
for i, f := range got.Files {
|
||||
if len(f.Chapters) != 1 {
|
||||
t.Errorf("file %d: %d synthesized chapters, want 1", i, len(f.Chapters))
|
||||
continue
|
||||
}
|
||||
wantStem := fmt.Sprintf("part%d", i+1)
|
||||
if f.Chapters[0].Title != wantStem {
|
||||
t.Errorf("file %d: chapter title = %q, want %q", i, f.Chapters[0].Title, wantStem)
|
||||
}
|
||||
if f.Chapters[0].StartSeconds != 0 || f.Chapters[0].EndSeconds != 0 {
|
||||
t.Errorf("file %d: synthesized chapter range = %.3f..%.3f, want 0..0 unknown duration", i, f.Chapters[0].StartSeconds, f.Chapters[0].EndSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudiobookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
book := &parsedAudiobook{Title: "Tagged Book", Author: "Author", Narrator: "Narrator", Year: 2024}
|
||||
file := parsedAudiobookFile{Chapters: []ChapterInfo{{Title: "One", StartSeconds: 0, EndSeconds: 10}}}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
|
||||
book = &parsedAudiobook{Title: "Tagged Book"}
|
||||
file = parsedAudiobookFile{Chapters: []ChapterInfo{{Title: "Unknown duration"}}}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
|
||||
book = &parsedAudiobook{}
|
||||
file = parsedAudiobookFile{}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAudiobookFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
bookDir := filepath.Join(root, "bad-book")
|
||||
if err := os.Mkdir(bookDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir book dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(bookDir, "chapter.mp3"), []byte("not real audio"), 0o644); err != nil {
|
||||
t.Fatalf("write fake audio: %v", err)
|
||||
}
|
||||
|
||||
s := &Scanner{ffprobePath: "definitely-missing-ffprobe"}
|
||||
err := s.ScanAudiobookFolder(context.Background(), &models.MediaFolder{ID: 42, Paths: []string{root}})
|
||||
if err == nil {
|
||||
t.Fatal("ScanAudiobookFolder returned nil, want aggregate failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "folder_id=42") {
|
||||
t.Fatalf("error = %q, want folder id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAudiobookFolderReturnsCanceledContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanAudiobookFolder(ctx, &models.MediaFolder{ID: 42, Paths: []string{root}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ScanAudiobookFolder error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemReusesRootScopedContentID(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{contentID: "book-root-id"}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Author/Same Title",
|
||||
&parsedAudiobook{Title: "Same Title", Year: 0, Author: "Author A"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAudiobookMediaItem: %v", err)
|
||||
}
|
||||
if got != "book-root-id" {
|
||||
t.Fatalf("contentID = %q, want root-scoped id", got)
|
||||
}
|
||||
if finder.calls != 1 {
|
||||
t.Fatalf("root finder calls = %d, want 1", finder.calls)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Other Author/Same Title",
|
||||
&parsedAudiobook{Title: "Same Title", Year: 0, Author: "Author B"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAudiobookMediaItem: %v", err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatal("contentID is empty")
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].ContentID != got || writer.upserts[0].Type != "audiobook" {
|
||||
t.Fatalf("upserted item = %+v, contentID %q", writer.upserts[0], got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemPropagatesRootLookupError(t *testing.T) {
|
||||
wantErr := errors.New("root lookup failed")
|
||||
finder := &fakeRootContentFinder{err: wantErr}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
_, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Author/Book",
|
||||
&parsedAudiobook{Title: "Book"},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("upserts = %d, want 0", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
+586
-120
@@ -984,6 +984,7 @@ type segmentState struct {
|
||||
provider *string
|
||||
confidence *float64
|
||||
algorithm *string
|
||||
detectedAt *time.Time
|
||||
}
|
||||
|
||||
// applySegmentPatch merges the patched start/end into the segment state, then
|
||||
@@ -993,10 +994,14 @@ type segmentState struct {
|
||||
func applySegmentPatch(
|
||||
state *segmentState,
|
||||
legacySharedSource *string,
|
||||
update MarkerUpdate,
|
||||
source string,
|
||||
provider *string,
|
||||
confidence *float64,
|
||||
algorithm string,
|
||||
patchStart, patchEnd *float64,
|
||||
duration float64,
|
||||
segmentName string,
|
||||
mutationAt time.Time,
|
||||
) (bool, error) {
|
||||
if patchStart == nil && patchEnd == nil {
|
||||
return false, nil
|
||||
@@ -1024,23 +1029,62 @@ func applySegmentPatch(
|
||||
if effectiveSource == nil && state.start != nil && state.end != nil {
|
||||
effectiveSource = legacySharedSource
|
||||
}
|
||||
if !markers.CanWriteMarker(effectiveSource, state.confidence, update.MarkersSource, update.MarkersConfidence) {
|
||||
if !markers.CanWriteMarker(effectiveSource, state.confidence, source, confidence) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
src := update.MarkersSource
|
||||
algo := markerAlgorithm(update)
|
||||
src := source
|
||||
algo := algorithm
|
||||
nextState := segmentState{
|
||||
start: nextStart,
|
||||
end: nextEnd,
|
||||
source: &src,
|
||||
provider: provider,
|
||||
confidence: confidence,
|
||||
algorithm: &algo,
|
||||
detectedAt: &mutationAt,
|
||||
}
|
||||
if segmentEqual(*state, nextState) {
|
||||
return false, nil
|
||||
}
|
||||
state.start = nextStart
|
||||
state.end = nextEnd
|
||||
state.source = &src
|
||||
state.provider = update.MarkersProvider
|
||||
state.confidence = update.MarkersConfidence
|
||||
state.provider = provider
|
||||
state.confidence = confidence
|
||||
state.algorithm = &algo
|
||||
state.detectedAt = &mutationAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// segmentEqual reports whether two segment states are byte-equivalent. Used to
|
||||
// detect no-op writes so the transaction can short-circuit without a SQL UPDATE.
|
||||
// resolveSegmentProvenance returns the source/provider/confidence/algorithm to
|
||||
// write for a segment: the per-segment override when present, otherwise the
|
||||
// update's shared Markers* values. The algorithm always falls back to
|
||||
// external:<source> so writes carry an algorithm tag.
|
||||
func resolveSegmentProvenance(update MarkerUpdate, override *SegmentProvenance) (source string, provider *string, confidence *float64, algorithm string) {
|
||||
source = update.MarkersSource
|
||||
provider = update.MarkersProvider
|
||||
confidence = update.MarkersConfidence
|
||||
algorithm = update.MarkersAlgorithm
|
||||
if override != nil {
|
||||
if override.Source != "" {
|
||||
source = override.Source
|
||||
}
|
||||
provider = override.Provider
|
||||
confidence = override.Confidence
|
||||
if override.Algorithm != "" {
|
||||
algorithm = override.Algorithm
|
||||
}
|
||||
}
|
||||
if algorithm == "" {
|
||||
algorithm = "external:" + source
|
||||
}
|
||||
return source, provider, confidence, algorithm
|
||||
}
|
||||
|
||||
// segmentEqual reports whether two segment states are semantically equivalent.
|
||||
// detected_at is intentionally ignored so writing the same marker value does
|
||||
// not refresh provenance timestamps or create audit noise.
|
||||
func segmentEqual(a, b segmentState) bool {
|
||||
return ptrFloatEqual(a.start, b.start) &&
|
||||
ptrFloatEqual(a.end, b.end) &&
|
||||
@@ -1055,22 +1099,243 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
if update.MarkersSource == "" {
|
||||
return false, fmt.Errorf("marker source is required")
|
||||
}
|
||||
return r.UpsertAndClearMarkers(ctx, fileID, update, nil)
|
||||
}
|
||||
|
||||
// ClearMarkers nulls the given segment kinds (intro|credits|recap|preview) for
|
||||
// a file, including their provenance columns. Used by the admin manual-marker
|
||||
// API to remove a marker so detection/online fetch can repopulate it. Returns
|
||||
// whether a row was updated.
|
||||
func (r *FileRepository) ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error) {
|
||||
return r.upsertAndClearMarkers(ctx, fileID, nil, segments)
|
||||
}
|
||||
|
||||
// UpsertAndClearMarkers applies manual marker sets and clears in one row-locking
|
||||
// transaction so mixed PUT bodies cannot partially persist.
|
||||
func (r *FileRepository) UpsertAndClearMarkers(ctx context.Context, fileID int, update MarkerUpdate, clearSegments []string) (bool, error) {
|
||||
return r.upsertAndClearMarkers(ctx, fileID, &update, clearSegments)
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]MarkerEditAuditRow, error) {
|
||||
if len(fileIDs) == 0 {
|
||||
return []MarkerEditAuditRow{}, nil
|
||||
}
|
||||
return r.listMarkerEditAudit(ctx, "WHERE a.media_file_id = ANY ($1)", []any{fileIDs}, limit)
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListAllMarkerEditAudit(ctx context.Context, limit int) ([]MarkerEditAuditRow, error) {
|
||||
return r.listMarkerEditAudit(ctx, "", nil, limit)
|
||||
}
|
||||
|
||||
func (r *FileRepository) listMarkerEditAudit(ctx context.Context, whereClause string, args []any, limit int) ([]MarkerEditAuditRow, error) {
|
||||
limit = normalizeMarkerAuditLimit(limit)
|
||||
args = append(args, limit)
|
||||
limitPlaceholder := len(args)
|
||||
rows, err := r.pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT a.id,
|
||||
a.media_file_id,
|
||||
NULLIF(CASE WHEN mf.episode_id <> '' THEN mf.episode_id ELSE mf.content_id END, '') AS item_id,
|
||||
NULLIF(COALESCE(CASE WHEN mf.episode_id <> '' THEN 'episode' ELSE mi.type END, mf.base_type), '') AS item_type,
|
||||
NULLIF(COALESCE(e.title, mi.title, mf.base_title), '') AS media_title,
|
||||
NULLIF(mf.file_path, '') AS file_path,
|
||||
a.segment_kind,
|
||||
a.action,
|
||||
a.before_marker,
|
||||
a.after_marker,
|
||||
a.user_id,
|
||||
u.username,
|
||||
a.impersonator_user_id,
|
||||
iu.username,
|
||||
a.api_key_id,
|
||||
a.request_id,
|
||||
a.client_ip::text,
|
||||
a.user_agent,
|
||||
a.created_at
|
||||
FROM marker_edit_audit a
|
||||
LEFT JOIN media_files mf ON mf.id = a.media_file_id
|
||||
LEFT JOIN media_items mi ON mi.content_id = mf.content_id
|
||||
LEFT JOIN episodes e ON e.content_id = mf.episode_id
|
||||
LEFT JOIN users u ON u.id = a.user_id
|
||||
LEFT JOIN users iu ON iu.id = a.impersonator_user_id
|
||||
%s
|
||||
ORDER BY a.created_at DESC, a.id DESC
|
||||
LIMIT $%d`, whereClause, limitPlaceholder), args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list marker edit audit: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MarkerEditAuditRow, 0, limit)
|
||||
for rows.Next() {
|
||||
var row MarkerEditAuditRow
|
||||
var beforeJSON []byte
|
||||
var afterJSON []byte
|
||||
if err := rows.Scan(
|
||||
&row.ID,
|
||||
&row.MediaFileID,
|
||||
&row.ItemID,
|
||||
&row.ItemType,
|
||||
&row.MediaTitle,
|
||||
&row.FilePath,
|
||||
&row.SegmentKind,
|
||||
&row.Action,
|
||||
&beforeJSON,
|
||||
&afterJSON,
|
||||
&row.UserID,
|
||||
&row.Username,
|
||||
&row.ImpersonatorUserID,
|
||||
&row.ImpersonatorUsername,
|
||||
&row.APIKeyID,
|
||||
&row.RequestID,
|
||||
&row.ClientIP,
|
||||
&row.UserAgent,
|
||||
&row.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan marker edit audit row: %w", err)
|
||||
}
|
||||
before, err := unmarshalMarkerAuditSegment(beforeJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := unmarshalMarkerAuditSegment(afterJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.Before = before
|
||||
row.After = after
|
||||
out = append(out, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate marker edit audit rows: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func normalizeMarkerAuditLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return 25
|
||||
}
|
||||
if limit > 100 {
|
||||
return 100
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
type markerSegmentFlags struct {
|
||||
intro bool
|
||||
credits bool
|
||||
recap bool
|
||||
preview bool
|
||||
}
|
||||
|
||||
func (f markerSegmentFlags) any() bool {
|
||||
return f.intro || f.credits || f.recap || f.preview
|
||||
}
|
||||
|
||||
type markerMutationState struct {
|
||||
duration float64
|
||||
existingSource *string
|
||||
existingConfidence *float64
|
||||
intro segmentState
|
||||
credits segmentState
|
||||
recap segmentState
|
||||
preview segmentState
|
||||
}
|
||||
|
||||
func (r *FileRepository) upsertAndClearMarkers(ctx context.Context, fileID int, update *MarkerUpdate, clearSegments []string) (bool, error) {
|
||||
hasUpdate := update != nil && update.HasAnySegment()
|
||||
if hasUpdate && update.MarkersSource == "" {
|
||||
return false, fmt.Errorf("marker source is required")
|
||||
}
|
||||
clearFlags, err := markerClearFlags(clearSegments)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !hasUpdate && !clearFlags.any() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin marker upsert transaction: %w", err)
|
||||
return false, fmt.Errorf("begin marker mutation transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var (
|
||||
duration float64
|
||||
existingSource *string
|
||||
existingConfidence *float64
|
||||
intro segmentState
|
||||
credits segmentState
|
||||
recap segmentState
|
||||
preview segmentState
|
||||
)
|
||||
state, err := loadMarkerMutationState(ctx, tx, fileID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
before := state
|
||||
mutationAt := time.Now().UTC()
|
||||
changed := markerSegmentFlags{}
|
||||
|
||||
if hasUpdate {
|
||||
applied, err := applyMarkerUpdateToMutationState(update, &state, mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
changed.intro = changed.intro || applied.intro
|
||||
changed.credits = changed.credits || applied.credits
|
||||
changed.recap = changed.recap || applied.recap
|
||||
changed.preview = changed.preview || applied.preview
|
||||
}
|
||||
if clearFlags.intro {
|
||||
changed.intro = clearSegmentState(&state.intro) || changed.intro
|
||||
}
|
||||
if clearFlags.credits {
|
||||
changed.credits = clearSegmentState(&state.credits) || changed.credits
|
||||
}
|
||||
if clearFlags.recap {
|
||||
changed.recap = clearSegmentState(&state.recap) || changed.recap
|
||||
}
|
||||
if clearFlags.preview {
|
||||
changed.preview = clearSegmentState(&state.preview) || changed.preview
|
||||
}
|
||||
if !changed.any() {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker no-op transaction: %w", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
wrote, err := writeMarkerMutationState(ctx, tx, fileID, state)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if wrote {
|
||||
if audit, ok := MarkerAuditContextFromContext(ctx); ok {
|
||||
if err := insertMarkerEditAuditRows(ctx, tx, fileID, before, state, changed, audit, mutationAt); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker mutation transaction: %w", err)
|
||||
}
|
||||
return wrote, nil
|
||||
}
|
||||
|
||||
func markerClearFlags(segments []string) (markerSegmentFlags, error) {
|
||||
var flags markerSegmentFlags
|
||||
for _, seg := range segments {
|
||||
switch seg {
|
||||
case "intro":
|
||||
flags.intro = true
|
||||
case "credits":
|
||||
flags.credits = true
|
||||
case "recap":
|
||||
flags.recap = true
|
||||
case "preview":
|
||||
flags.preview = true
|
||||
default:
|
||||
return markerSegmentFlags{}, fmt.Errorf("invalid marker segment %q", seg)
|
||||
}
|
||||
}
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func loadMarkerMutationState(ctx context.Context, tx pgx.Tx, fileID int) (markerMutationState, error) {
|
||||
var state markerMutationState
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT COALESCE(duration, 0),
|
||||
markers_source,
|
||||
@@ -1081,120 +1346,132 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
intro_markers_provider,
|
||||
intro_markers_confidence,
|
||||
intro_markers_algorithm,
|
||||
intro_markers_detected_at,
|
||||
credits_start,
|
||||
credits_end,
|
||||
credits_markers_source,
|
||||
credits_markers_provider,
|
||||
credits_markers_confidence,
|
||||
credits_markers_algorithm,
|
||||
credits_markers_detected_at,
|
||||
recap_start,
|
||||
recap_end,
|
||||
recap_markers_source,
|
||||
recap_markers_provider,
|
||||
recap_markers_confidence,
|
||||
recap_markers_algorithm,
|
||||
recap_markers_detected_at,
|
||||
preview_start,
|
||||
preview_end,
|
||||
preview_markers_source,
|
||||
preview_markers_provider,
|
||||
preview_markers_confidence,
|
||||
preview_markers_algorithm
|
||||
preview_markers_algorithm,
|
||||
preview_markers_detected_at
|
||||
FROM media_files WHERE id = $1 FOR UPDATE`,
|
||||
fileID,
|
||||
).Scan(
|
||||
&duration,
|
||||
&existingSource,
|
||||
&existingConfidence,
|
||||
&intro.start, &intro.end, &intro.source, &intro.provider, &intro.confidence, &intro.algorithm,
|
||||
&credits.start, &credits.end, &credits.source, &credits.provider, &credits.confidence, &credits.algorithm,
|
||||
&recap.start, &recap.end, &recap.source, &recap.provider, &recap.confidence, &recap.algorithm,
|
||||
&preview.start, &preview.end, &preview.source, &preview.provider, &preview.confidence, &preview.algorithm,
|
||||
&state.duration,
|
||||
&state.existingSource,
|
||||
&state.existingConfidence,
|
||||
&state.intro.start, &state.intro.end, &state.intro.source, &state.intro.provider, &state.intro.confidence, &state.intro.algorithm, &state.intro.detectedAt,
|
||||
&state.credits.start, &state.credits.end, &state.credits.source, &state.credits.provider, &state.credits.confidence, &state.credits.algorithm, &state.credits.detectedAt,
|
||||
&state.recap.start, &state.recap.end, &state.recap.source, &state.recap.provider, &state.recap.confidence, &state.recap.algorithm, &state.recap.detectedAt,
|
||||
&state.preview.start, &state.preview.end, &state.preview.source, &state.preview.provider, &state.preview.confidence, &state.preview.algorithm, &state.preview.detectedAt,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, ErrFileNotFound
|
||||
return markerMutationState{}, ErrFileNotFound
|
||||
}
|
||||
return false, fmt.Errorf("load existing marker source: %w", err)
|
||||
return markerMutationState{}, fmt.Errorf("load existing marker source: %w", err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
originalIntro, originalCredits, originalRecap, originalPreview := intro, credits, recap, preview
|
||||
|
||||
introApplied, err := applySegmentPatch(&intro, existingSource, update, update.IntroStart, update.IntroEnd, duration, "intro")
|
||||
func applyMarkerUpdateToMutationState(update *MarkerUpdate, state *markerMutationState, mutationAt time.Time) (markerSegmentFlags, error) {
|
||||
var applied markerSegmentFlags
|
||||
introSrc, introProv, introConf, introAlgo := resolveSegmentProvenance(*update, update.IntroProvenance)
|
||||
var err error
|
||||
applied.intro, err = applySegmentPatch(&state.intro, state.existingSource, introSrc, introProv, introConf, introAlgo, update.IntroStart, update.IntroEnd, state.duration, "intro", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
creditsApplied, err := applySegmentPatch(&credits, existingSource, update, update.CreditsStart, update.CreditsEnd, duration, "credits")
|
||||
creditsSrc, creditsProv, creditsConf, creditsAlgo := resolveSegmentProvenance(*update, update.CreditsProvenance)
|
||||
applied.credits, err = applySegmentPatch(&state.credits, state.existingSource, creditsSrc, creditsProv, creditsConf, creditsAlgo, update.CreditsStart, update.CreditsEnd, state.duration, "credits", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
recapApplied, err := applySegmentPatch(&recap, existingSource, update, update.RecapStart, update.RecapEnd, duration, "recap")
|
||||
recapSrc, recapProv, recapConf, recapAlgo := resolveSegmentProvenance(*update, update.RecapProvenance)
|
||||
applied.recap, err = applySegmentPatch(&state.recap, state.existingSource, recapSrc, recapProv, recapConf, recapAlgo, update.RecapStart, update.RecapEnd, state.duration, "recap", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
previewApplied, err := applySegmentPatch(&preview, existingSource, update, update.PreviewStart, update.PreviewEnd, duration, "preview")
|
||||
previewSrc, previewProv, previewConf, previewAlgo := resolveSegmentProvenance(*update, update.PreviewProvenance)
|
||||
applied.preview, err = applySegmentPatch(&state.preview, state.existingSource, previewSrc, previewProv, previewConf, previewAlgo, update.PreviewStart, update.PreviewEnd, state.duration, "preview", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
anyApplied := introApplied || creditsApplied || recapApplied || previewApplied
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(existingSource, existingConfidence, update, anyApplied)
|
||||
|
||||
if segmentEqual(intro, originalIntro) &&
|
||||
segmentEqual(credits, originalCredits) &&
|
||||
segmentEqual(recap, originalRecap) &&
|
||||
segmentEqual(preview, originalPreview) &&
|
||||
ptrStringEqual(existingSource, nextSource) &&
|
||||
ptrFloatEqual(existingConfidence, nextConfidence) {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker no-op transaction: %w", err)
|
||||
}
|
||||
return false, nil
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
return applied, nil
|
||||
}
|
||||
|
||||
func writeMarkerMutationState(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
fileID int,
|
||||
state markerMutationState,
|
||||
) (bool, error) {
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
state.existingSource,
|
||||
state.existingConfidence,
|
||||
state.intro,
|
||||
state.credits,
|
||||
state.recap,
|
||||
state.preview,
|
||||
)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE media_files
|
||||
SET intro_start = $2,
|
||||
intro_end = $3,
|
||||
credits_start = $4,
|
||||
credits_end = $5,
|
||||
recap_start = $6,
|
||||
recap_end = $7,
|
||||
preview_start = $8,
|
||||
preview_end = $9,
|
||||
markers_source = $10,
|
||||
markers_confidence = $11,
|
||||
intro_markers_source = $12,
|
||||
intro_markers_provider = $13,
|
||||
intro_markers_confidence = $14,
|
||||
intro_markers_algorithm = $15,
|
||||
intro_markers_detected_at = CASE WHEN $16 THEN NOW() ELSE intro_markers_detected_at END,
|
||||
credits_markers_source = $17,
|
||||
credits_markers_provider = $18,
|
||||
credits_markers_confidence = $19,
|
||||
credits_markers_algorithm = $20,
|
||||
credits_markers_detected_at = CASE WHEN $21 THEN NOW() ELSE credits_markers_detected_at END,
|
||||
recap_markers_source = $22,
|
||||
recap_markers_provider = $23,
|
||||
recap_markers_confidence = $24,
|
||||
recap_markers_algorithm = $25,
|
||||
recap_markers_detected_at = CASE WHEN $26 THEN NOW() ELSE recap_markers_detected_at END,
|
||||
preview_markers_source = $27,
|
||||
preview_markers_provider = $28,
|
||||
preview_markers_confidence = $29,
|
||||
preview_markers_algorithm = $30,
|
||||
preview_markers_detected_at = CASE WHEN $31 THEN NOW() ELSE preview_markers_detected_at END,
|
||||
SET intro_start = $2::double precision,
|
||||
intro_end = $3::double precision,
|
||||
credits_start = $4::double precision,
|
||||
credits_end = $5::double precision,
|
||||
recap_start = $6::double precision,
|
||||
recap_end = $7::double precision,
|
||||
preview_start = $8::double precision,
|
||||
preview_end = $9::double precision,
|
||||
markers_source = $10::text,
|
||||
markers_confidence = $11::double precision,
|
||||
intro_markers_source = $12::text,
|
||||
intro_markers_provider = $13::text,
|
||||
intro_markers_confidence = $14::double precision,
|
||||
intro_markers_algorithm = $15::text,
|
||||
intro_markers_detected_at = $16::timestamptz,
|
||||
credits_markers_source = $17::text,
|
||||
credits_markers_provider = $18::text,
|
||||
credits_markers_confidence = $19::double precision,
|
||||
credits_markers_algorithm = $20::text,
|
||||
credits_markers_detected_at = $21::timestamptz,
|
||||
recap_markers_source = $22::text,
|
||||
recap_markers_provider = $23::text,
|
||||
recap_markers_confidence = $24::double precision,
|
||||
recap_markers_algorithm = $25::text,
|
||||
recap_markers_detected_at = $26::timestamptz,
|
||||
preview_markers_source = $27::text,
|
||||
preview_markers_provider = $28::text,
|
||||
preview_markers_confidence = $29::double precision,
|
||||
preview_markers_algorithm = $30::text,
|
||||
preview_markers_detected_at = $31::timestamptz,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`,
|
||||
fileID,
|
||||
intro.start, intro.end,
|
||||
credits.start, credits.end,
|
||||
recap.start, recap.end,
|
||||
preview.start, preview.end,
|
||||
state.intro.start, state.intro.end,
|
||||
state.credits.start, state.credits.end,
|
||||
state.recap.start, state.recap.end,
|
||||
state.preview.start, state.preview.end,
|
||||
nextSource, nextConfidence,
|
||||
intro.source, intro.provider, intro.confidence, intro.algorithm, introApplied,
|
||||
credits.source, credits.provider, credits.confidence, credits.algorithm, creditsApplied,
|
||||
recap.source, recap.provider, recap.confidence, recap.algorithm, recapApplied,
|
||||
preview.source, preview.provider, preview.confidence, preview.algorithm, previewApplied,
|
||||
state.intro.source, state.intro.provider, state.intro.confidence, state.intro.algorithm, state.intro.detectedAt,
|
||||
state.credits.source, state.credits.provider, state.credits.confidence, state.credits.algorithm, state.credits.detectedAt,
|
||||
state.recap.source, state.recap.provider, state.recap.confidence, state.recap.algorithm, state.recap.detectedAt,
|
||||
state.preview.source, state.preview.provider, state.preview.confidence, state.preview.algorithm, state.preview.detectedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("updating media markers: %w", err)
|
||||
@@ -1202,13 +1479,227 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
if tag.RowsAffected() == 0 {
|
||||
return false, ErrFileNotFound
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker upsert transaction: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func clearSegmentState(state *segmentState) bool {
|
||||
if segmentEqual(*state, segmentState{}) {
|
||||
return false
|
||||
}
|
||||
state.start = nil
|
||||
state.end = nil
|
||||
state.source = nil
|
||||
state.provider = nil
|
||||
state.confidence = nil
|
||||
state.algorithm = nil
|
||||
state.detectedAt = nil
|
||||
return true
|
||||
}
|
||||
|
||||
func insertMarkerEditAuditRows(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
fileID int,
|
||||
before markerMutationState,
|
||||
after markerMutationState,
|
||||
changed markerSegmentFlags,
|
||||
audit MarkerAuditContext,
|
||||
mutationAt time.Time,
|
||||
) error {
|
||||
type changedSegment struct {
|
||||
kind string
|
||||
before segmentState
|
||||
after segmentState
|
||||
}
|
||||
segments := make([]changedSegment, 0, 4)
|
||||
if changed.intro {
|
||||
segments = append(segments, changedSegment{"intro", before.intro, after.intro})
|
||||
}
|
||||
if changed.credits {
|
||||
segments = append(segments, changedSegment{"credits", before.credits, after.credits})
|
||||
}
|
||||
if changed.recap {
|
||||
segments = append(segments, changedSegment{"recap", before.recap, after.recap})
|
||||
}
|
||||
if changed.preview {
|
||||
segments = append(segments, changedSegment{"preview", before.preview, after.preview})
|
||||
}
|
||||
|
||||
for _, segment := range segments {
|
||||
beforeJSON, err := marshalMarkerAuditSegment(markerAuditSegmentForState(segment.before))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
afterJSON, err := marshalMarkerAuditSegment(markerAuditSegmentForState(segment.after))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
action := "set"
|
||||
if len(afterJSON) == 0 {
|
||||
action = "clear"
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO marker_edit_audit (
|
||||
media_file_id,
|
||||
segment_kind,
|
||||
action,
|
||||
before_marker,
|
||||
after_marker,
|
||||
user_id,
|
||||
impersonator_user_id,
|
||||
api_key_id,
|
||||
request_id,
|
||||
client_ip,
|
||||
user_agent,
|
||||
created_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5::jsonb, $6, $7, $8, $9, $10::inet, $11, $12)`,
|
||||
fileID,
|
||||
segment.kind,
|
||||
action,
|
||||
nullableJSON(beforeJSON),
|
||||
nullableJSON(afterJSON),
|
||||
audit.UserID,
|
||||
audit.ImpersonatorUserID,
|
||||
audit.APIKeyID,
|
||||
nullableString(audit.RequestID),
|
||||
nullableString(audit.ClientIP),
|
||||
nullableString(audit.UserAgent),
|
||||
mutationAt,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert marker edit audit row: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markerAuditSegmentForState(state segmentState) *MarkerAuditSegment {
|
||||
if state.start == nil || state.end == nil {
|
||||
return nil
|
||||
}
|
||||
return &MarkerAuditSegment{
|
||||
Start: cloneFloat(state.start),
|
||||
End: cloneFloat(state.end),
|
||||
Source: cloneString(state.source),
|
||||
Provider: cloneString(state.provider),
|
||||
Confidence: cloneFloat(state.confidence),
|
||||
Algorithm: cloneString(state.algorithm),
|
||||
DetectedAt: cloneTime(state.detectedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func marshalMarkerAuditSegment(segment *MarkerAuditSegment) ([]byte, error) {
|
||||
if segment == nil {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := json.Marshal(segment)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal marker audit segment: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func nullableJSON(data []byte) any {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func nullableString(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func recomputeSharedMarkerAttribution(
|
||||
legacySource *string,
|
||||
legacyConfidence *float64,
|
||||
states ...segmentState,
|
||||
) (*string, *float64) {
|
||||
var (
|
||||
bestSource *string
|
||||
bestConfidence *float64
|
||||
bestPriority int
|
||||
found bool
|
||||
)
|
||||
for _, state := range states {
|
||||
if state.start == nil || state.end == nil {
|
||||
continue
|
||||
}
|
||||
source := state.source
|
||||
confidence := state.confidence
|
||||
if source == nil {
|
||||
source = legacySource
|
||||
if confidence == nil {
|
||||
confidence = legacyConfidence
|
||||
}
|
||||
}
|
||||
if source == nil || *source == "" {
|
||||
continue
|
||||
}
|
||||
priority := models.MarkerSourcePriority(*source)
|
||||
if !found || priority > bestPriority || (priority == bestPriority && confidenceGreater(confidence, bestConfidence)) {
|
||||
sourceCopy := *source
|
||||
bestSource = &sourceCopy
|
||||
bestConfidence = cloneFloat(confidence)
|
||||
bestPriority = priority
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
return bestSource, bestConfidence
|
||||
}
|
||||
|
||||
func confidenceGreater(a, b *float64) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
return *a > *b
|
||||
}
|
||||
|
||||
func cloneFloat(v *float64) *float64 {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneString(v *string) *string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneTime(v *time.Time) *time.Time {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func unmarshalMarkerAuditSegment(data []byte) (*MarkerAuditSegment, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var segment MarkerAuditSegment
|
||||
if err := json.Unmarshal(data, &segment); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal marker audit segment: %w", err)
|
||||
}
|
||||
return &segment, nil
|
||||
}
|
||||
|
||||
func ptrFloatEqual(a, b *float64) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
@@ -1216,31 +1707,6 @@ func ptrFloatEqual(a, b *float64) bool {
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func nextSharedMarkerAttribution(
|
||||
existingSource *string,
|
||||
existingConfidence *float64,
|
||||
update MarkerUpdate,
|
||||
markerApplied bool,
|
||||
) (*string, *float64) {
|
||||
if !markerApplied {
|
||||
return existingSource, existingConfidence
|
||||
}
|
||||
if existingSource == nil || models.MarkerSourcePriority(update.MarkersSource) > models.MarkerSourcePriority(*existingSource) {
|
||||
return &update.MarkersSource, update.MarkersConfidence
|
||||
}
|
||||
if models.MarkerSourcePriority(update.MarkersSource) == models.MarkerSourcePriority(*existingSource) && update.MarkersConfidence != nil {
|
||||
return existingSource, update.MarkersConfidence
|
||||
}
|
||||
return existingSource, existingConfidence
|
||||
}
|
||||
|
||||
func markerAlgorithm(update MarkerUpdate) string {
|
||||
if update.MarkersAlgorithm != "" {
|
||||
return update.MarkersAlgorithm
|
||||
}
|
||||
return "external:" + update.MarkersSource
|
||||
}
|
||||
|
||||
func ptrStringEqual(a, b *string) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
|
||||
@@ -2,73 +2,115 @@ package scanner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestNextSharedMarkerAttributionPreservesHigherPriorityConfidence(t *testing.T) {
|
||||
existingSource := models.MarkerSourceManual
|
||||
existingConfidence := 1.0
|
||||
updateConfidence := 0.5
|
||||
func TestRecomputeSharedMarkerAttributionPreservesHigherPrioritySegment(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manualConfidence := 1.0
|
||||
scannerConfidence := 0.5
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &manual,
|
||||
confidence: &manualConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceManual {
|
||||
t.Fatalf("next source = %v, want manual", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != existingConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, existingConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != manualConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, manualConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionUpdatesSamePriorityConfidence(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.5
|
||||
updateConfidence := 0.8
|
||||
func TestRecomputeSharedMarkerAttributionUpdatesSamePriorityConfidence(t *testing.T) {
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
lowerConfidence := 0.5
|
||||
higherConfidence := 0.8
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &scannerSource,
|
||||
confidence: &lowerConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &scannerSource,
|
||||
confidence: &higherConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != updateConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, updateConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != higherConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, higherConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionPromotesHigherPrioritySource(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.5
|
||||
updateConfidence := 0.9
|
||||
func TestRecomputeSharedMarkerAttributionPromotesHigherPrioritySource(t *testing.T) {
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manual := models.MarkerSourceManual
|
||||
scannerConfidence := 0.5
|
||||
manualConfidence := 0.9
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceManual,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &manual,
|
||||
confidence: &manualConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceManual {
|
||||
t.Fatalf("next source = %v, want manual", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != updateConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, updateConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != manualConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, manualConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionDoesNotDowngradeConfidenceWhenMarkerRejected(t *testing.T) {
|
||||
func TestRecomputeSharedMarkerAttributionUsesLegacyAttributionForUnattributedSegment(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.9
|
||||
updateConfidence := 0.4
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, false)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
&existingSource,
|
||||
&existingConfidence,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner", nextSource)
|
||||
@@ -78,3 +120,122 @@ func TestNextSharedMarkerAttributionDoesNotDowngradeConfidenceWhenMarkerRejected
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplySegmentPatchSkipsSemanticNoop(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
confidence := 1.0
|
||||
algorithm := "manual:v1"
|
||||
detectedAt := time.Unix(100, 0).UTC()
|
||||
state := segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(60),
|
||||
source: &manual,
|
||||
confidence: &confidence,
|
||||
algorithm: &algorithm,
|
||||
detectedAt: &detectedAt,
|
||||
}
|
||||
|
||||
changed, err := applySegmentPatch(
|
||||
&state,
|
||||
nil,
|
||||
manual,
|
||||
nil,
|
||||
&confidence,
|
||||
algorithm,
|
||||
floatPtr(0),
|
||||
floatPtr(60),
|
||||
1800,
|
||||
"intro",
|
||||
time.Unix(200, 0).UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("applySegmentPatch returned error: %v", err)
|
||||
}
|
||||
if changed {
|
||||
t.Fatal("identical marker patch should be a semantic no-op")
|
||||
}
|
||||
if state.detectedAt == nil || !state.detectedAt.Equal(detectedAt) {
|
||||
t.Fatalf("detected_at changed on no-op: %v", state.detectedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearSegmentStateSkipsSemanticNoop(t *testing.T) {
|
||||
empty := segmentState{}
|
||||
if clearSegmentState(&empty) {
|
||||
t.Fatal("clearing an empty segment should be a no-op")
|
||||
}
|
||||
|
||||
manual := models.MarkerSourceManual
|
||||
state := segmentState{start: floatPtr(0), end: floatPtr(60), source: &manual}
|
||||
if !clearSegmentState(&state) {
|
||||
t.Fatal("clearing a populated segment should report a change")
|
||||
}
|
||||
if state.start != nil || state.end != nil || state.source != nil {
|
||||
t.Fatalf("segment was not cleared: %+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerAuditSegmentForStatePreservesBeforeAfterShape(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
algorithm := "manual:v1"
|
||||
confidence := 1.0
|
||||
detectedAt := time.Unix(300, 0).UTC()
|
||||
|
||||
segment := markerAuditSegmentForState(segmentState{
|
||||
start: floatPtr(5),
|
||||
end: floatPtr(65),
|
||||
source: &manual,
|
||||
confidence: &confidence,
|
||||
algorithm: &algorithm,
|
||||
detectedAt: &detectedAt,
|
||||
})
|
||||
if segment == nil {
|
||||
t.Fatal("expected audit segment")
|
||||
}
|
||||
if segment.Start == nil || *segment.Start != 5 || segment.End == nil || *segment.End != 65 {
|
||||
t.Fatalf("audit segment bounds = %v..%v, want 5..65", segment.Start, segment.End)
|
||||
}
|
||||
if segment.Source == nil || *segment.Source != manual || segment.Algorithm == nil || *segment.Algorithm != algorithm {
|
||||
t.Fatalf("audit provenance = source %v algorithm %v", segment.Source, segment.Algorithm)
|
||||
}
|
||||
if segment.DetectedAt == nil || !segment.DetectedAt.Equal(detectedAt) {
|
||||
t.Fatalf("detected_at = %v, want %v", segment.DetectedAt, detectedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeSharedMarkerAttributionDropsClearedManualSegment(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manualConfidence := 1.0
|
||||
scannerConfidence := 0.8
|
||||
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
&manual,
|
||||
&manualConfidence,
|
||||
segmentState{},
|
||||
segmentState{
|
||||
start: floatPtr(120),
|
||||
end: floatPtr(180),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner after manual segment clear", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != scannerConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, scannerConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeSharedMarkerAttributionClearsWhenNoSegmentsRemain(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
manualConfidence := 1.0
|
||||
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(&manual, &manualConfidence, segmentState{})
|
||||
if nextSource != nil || nextConfidence != nil {
|
||||
t.Fatalf("shared attribution = %v/%v, want nil/nil with no remaining segments", nextSource, nextConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func floatPtr(v float64) *float64 { return &v }
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MarkerAuditContext struct {
|
||||
UserID *int
|
||||
ImpersonatorUserID *int
|
||||
APIKeyID *int64
|
||||
RequestID string
|
||||
ClientIP string
|
||||
UserAgent string
|
||||
}
|
||||
|
||||
type markerAuditContextKey struct{}
|
||||
|
||||
func WithMarkerAuditContext(ctx context.Context, audit MarkerAuditContext) context.Context {
|
||||
return context.WithValue(ctx, markerAuditContextKey{}, audit)
|
||||
}
|
||||
|
||||
func MarkerAuditContextFromContext(ctx context.Context) (MarkerAuditContext, bool) {
|
||||
audit, ok := ctx.Value(markerAuditContextKey{}).(MarkerAuditContext)
|
||||
return audit, ok
|
||||
}
|
||||
|
||||
type MarkerAuditSegment struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Source *string `json:"source"`
|
||||
Provider *string `json:"provider"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
Algorithm *string `json:"algorithm"`
|
||||
DetectedAt *time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
type MarkerEditAuditRow struct {
|
||||
ID int64
|
||||
MediaFileID int
|
||||
ItemID *string
|
||||
ItemType *string
|
||||
MediaTitle *string
|
||||
FilePath *string
|
||||
SegmentKind string
|
||||
Action string
|
||||
Before *MarkerAuditSegment
|
||||
After *MarkerAuditSegment
|
||||
UserID *int
|
||||
Username *string
|
||||
ImpersonatorUserID *int
|
||||
ImpersonatorUsername *string
|
||||
APIKeyID *int64
|
||||
RequestID *string
|
||||
ClientIP *string
|
||||
UserAgent *string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// parsedPodcastShow is what parsePodcastShow returns. The scanner write
|
||||
// path converts this into media_items (type='podcast') + episodes +
|
||||
// media_files rows.
|
||||
type parsedPodcastShow struct {
|
||||
Title string
|
||||
Author string
|
||||
Year int
|
||||
Episodes []parsedPodcastEpisode
|
||||
}
|
||||
|
||||
type parsedPodcastEpisode struct {
|
||||
Path string
|
||||
Title string
|
||||
Track int
|
||||
}
|
||||
|
||||
// parsePodcastShow walks a single subdirectory of a podcast library and
|
||||
// returns the show's metadata + episode list. Each audio file inside
|
||||
// becomes one episode; tags are read from each file individually so
|
||||
// per-episode titles surface correctly.
|
||||
//
|
||||
// Returns an os.ErrNotExist-wrapped error if the folder contains zero
|
||||
// audio files.
|
||||
func parsePodcastShow(ctx context.Context, ffprobePath string, folderPath string) (*parsedPodcastShow, error) {
|
||||
entries, err := os.ReadDir(folderPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read podcast folder %s: %w", folderPath, err)
|
||||
}
|
||||
var audioFiles []string
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
if SupportsAudioFile(entry.Name()) {
|
||||
audioFiles = append(audioFiles, filepath.Join(folderPath, entry.Name()))
|
||||
}
|
||||
}
|
||||
if len(audioFiles) == 0 {
|
||||
return nil, fmt.Errorf("podcast show %s: %w", folderPath, os.ErrNotExist)
|
||||
}
|
||||
sort.Strings(audioFiles)
|
||||
|
||||
show := &parsedPodcastShow{}
|
||||
for idx, path := range audioFiles {
|
||||
probed, err := ProbeFile(ctx, ffprobePath, path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe podcast file %s: %w", path, err)
|
||||
}
|
||||
tags := probed.FormatTags
|
||||
if show.Title == "" {
|
||||
show.Title = firstNonEmpty(tags["album"], tags["show"], filepath.Base(folderPath))
|
||||
show.Author = firstNonEmpty(tags["artist"], tags["album_artist"])
|
||||
if year := firstNonEmpty(tags["date"], tags["year"]); year != "" {
|
||||
if y := parseTagYear(year); y > 0 {
|
||||
show.Year = y
|
||||
}
|
||||
}
|
||||
}
|
||||
stem := filepath.Base(path)
|
||||
stem = stem[:len(stem)-len(filepath.Ext(stem))]
|
||||
track := idx + 1
|
||||
if t := tags["track"]; t != "" {
|
||||
if parsed := parseTrackNumber(t); parsed > 0 {
|
||||
track = parsed
|
||||
}
|
||||
}
|
||||
show.Episodes = append(show.Episodes, parsedPodcastEpisode{
|
||||
Path: path,
|
||||
Title: firstNonEmpty(tags["title"], stem),
|
||||
Track: track,
|
||||
})
|
||||
}
|
||||
return show, nil
|
||||
}
|
||||
|
||||
// parseTrackNumber accepts ID3-style track values which may be "5" or
|
||||
// "5/12". Returns 0 if no leading integer is present.
|
||||
func parseTrackNumber(s string) int {
|
||||
n := 0
|
||||
for _, c := range s {
|
||||
if c < '0' || c > '9' {
|
||||
break
|
||||
}
|
||||
n = n*10 + int(c-'0')
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,243 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/titleutil"
|
||||
)
|
||||
|
||||
// ScanPodcastFolder walks a podcasts-typed media folder and writes one
|
||||
// media_items row (type='podcast') per immediate subdirectory it can parse
|
||||
// as a podcast show, plus one episodes row and one media_files row per
|
||||
// audio file inside.
|
||||
//
|
||||
// RSS-subscribed feeds (podcast_feeds table) are handled by sub-plan 5;
|
||||
// this method covers filesystem-only ingestion.
|
||||
func (s *Scanner) ScanPodcastFolder(ctx context.Context, folder *models.MediaFolder) error {
|
||||
if s == nil || folder == nil {
|
||||
return fmt.Errorf("ScanPodcastFolder: nil scanner or folder")
|
||||
}
|
||||
|
||||
var attempted int
|
||||
var succeeded int
|
||||
var failures []error
|
||||
for _, root := range folder.Paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
slog.Warn("podcast scan: read root failed", "root", root, "error", err)
|
||||
attempted++
|
||||
failures = append(failures, fmt.Errorf("read root %s: %w", root, err))
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
subPath := filepath.Join(root, entry.Name())
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
attempted++
|
||||
if err := s.reconcilePodcastShow(ctx, folder, subPath); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
}
|
||||
slog.Warn("podcast scan: show failed",
|
||||
"folder_id", folder.ID,
|
||||
"path", subPath,
|
||||
"error", err,
|
||||
)
|
||||
failures = append(failures, fmt.Errorf("%s: %w", subPath, err))
|
||||
// Continue with siblings — one bad show should not stop the scan.
|
||||
continue
|
||||
}
|
||||
succeeded++
|
||||
}
|
||||
}
|
||||
if attempted > 0 && succeeded == 0 && len(failures) > 0 {
|
||||
return fmt.Errorf("podcast scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Scanner) reconcilePodcastShow(ctx context.Context, folder *models.MediaFolder, folderPath string) error {
|
||||
parsed, err := parsePodcastShow(ctx, s.ffprobePath, folderPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("parse podcast show %s: %w", folderPath, err)
|
||||
}
|
||||
|
||||
showContentID, err := s.upsertPodcastMediaItem(ctx, folder.ID, folderPath, parsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert podcast item: %w", err)
|
||||
}
|
||||
if err := s.upsertPodcastEpisodesAndFiles(ctx, folder, showContentID, folderPath, parsed); err != nil {
|
||||
return fmt.Errorf("upsert podcast episodes+files: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("podcast scan: indexed",
|
||||
"folder_id", folder.ID,
|
||||
"content_id", showContentID,
|
||||
"title", parsed.Title,
|
||||
"author", parsed.Author,
|
||||
"episodes", len(parsed.Episodes),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertPodcastMediaItem reuses an item already linked to the same filesystem
|
||||
// root, or creates a new row. It intentionally avoids title/year-only dedupe
|
||||
// because filesystem podcasts can have duplicate titles and missing years.
|
||||
// Returns the content_id used.
|
||||
func (s *Scanner) upsertPodcastMediaItem(ctx context.Context, folderID int, folderPath string, show *parsedPodcastShow) (string, error) {
|
||||
if s.itemRepo == nil {
|
||||
return "", fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
if s.fileRepo == nil {
|
||||
return "", fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
return resolvePodcastMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, folderPath, show)
|
||||
}
|
||||
|
||||
func resolvePodcastMediaItem(
|
||||
ctx context.Context,
|
||||
rootFinder filesystemRootContentFinder,
|
||||
itemWriter filesystemMediaItemWriter,
|
||||
folderID int,
|
||||
folderPath string,
|
||||
show *parsedPodcastShow,
|
||||
) (string, error) {
|
||||
if rootFinder == nil {
|
||||
return "", fmt.Errorf("root content finder not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return "", fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, folderPath, "podcast")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find podcast by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
id, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate content_id: %w", err)
|
||||
}
|
||||
item := &models.MediaItem{
|
||||
ContentID: id,
|
||||
Type: "podcast",
|
||||
Title: show.Title,
|
||||
SortTitle: titleutil.DeriveDefaultSortTitle(show.Title),
|
||||
Year: show.Year,
|
||||
}
|
||||
if err := itemWriter.Upsert(ctx, item); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// upsertPodcastEpisodesAndFiles writes one episodes row and one media_files
|
||||
// row per episode in the parsed show. The episode's content_id is linked back
|
||||
// through media_files.episode_id so playback and library queries can resolve
|
||||
// it.
|
||||
//
|
||||
// Podcast episodes use season_number=0 (no season concept) and
|
||||
// episode_number=track.
|
||||
func (s *Scanner) upsertPodcastEpisodesAndFiles(
|
||||
ctx context.Context,
|
||||
folder *models.MediaFolder,
|
||||
showContentID string,
|
||||
folderPath string,
|
||||
show *parsedPodcastShow,
|
||||
) error {
|
||||
if s.episodeRepo == nil {
|
||||
return fmt.Errorf("episodeRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
for _, ep := range show.Episodes {
|
||||
episodeID, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate episode content_id: %w", err)
|
||||
}
|
||||
|
||||
episode := &models.Episode{
|
||||
ContentID: episodeID,
|
||||
SeriesID: showContentID,
|
||||
SeasonNumber: 0,
|
||||
EpisodeNumber: ep.Track,
|
||||
Title: ep.Title,
|
||||
}
|
||||
// Upsert writes back the stored content_id (preserving existing on conflict).
|
||||
if err := s.episodeRepo.Upsert(ctx, episode); err != nil {
|
||||
return fmt.Errorf("upsert episode %q: %w", ep.Title, err)
|
||||
}
|
||||
|
||||
mf := models.MediaFile{
|
||||
ContentID: showContentID,
|
||||
EpisodeID: episode.ContentID,
|
||||
SeasonNumber: 0,
|
||||
EpisodeNumber: ep.Track,
|
||||
MediaFolderID: folder.ID,
|
||||
CanonicalRootPath: folderPath,
|
||||
ObservedRootPath: folderPath,
|
||||
ContentGroupKey: showContentID,
|
||||
GroupKeyVersion: 1,
|
||||
BaseTitle: show.Title,
|
||||
BaseYear: show.Year,
|
||||
BaseType: "podcast",
|
||||
IdentityConfidence: podcastIdentityConfidence(show, ep),
|
||||
FilePath: ep.Path,
|
||||
ProbeSource: "local",
|
||||
}
|
||||
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
|
||||
return fmt.Errorf("upsert media file %s: %w", ep.Path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func podcastIdentityConfidence(show *parsedPodcastShow, episode parsedPodcastEpisode) string {
|
||||
if show == nil {
|
||||
return "low"
|
||||
}
|
||||
|
||||
score := 0
|
||||
if show.Title != "" {
|
||||
score++
|
||||
}
|
||||
if show.Author != "" {
|
||||
score++
|
||||
}
|
||||
if show.Year > 0 {
|
||||
score++
|
||||
}
|
||||
if episode.Title != "" {
|
||||
score++
|
||||
}
|
||||
if episode.Track > 0 {
|
||||
score++
|
||||
}
|
||||
|
||||
switch {
|
||||
case score >= 5:
|
||||
return "high"
|
||||
case score > 0:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestParsePodcastShow(t *testing.T) {
|
||||
ffprobePath := FFprobePathFromFFmpeg("ffmpeg")
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
ffprobePath = "ffprobe"
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
t.Skip("ffprobe not available")
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
got, err := parsePodcastShow(ctx, ffprobePath, "testdata/podcast_fixtures/show_a")
|
||||
if err != nil {
|
||||
t.Fatalf("parsePodcastShow: %v", err)
|
||||
}
|
||||
if got.Title != "Show A" {
|
||||
t.Errorf("Title = %q, want %q", got.Title, "Show A")
|
||||
}
|
||||
if got.Author != "Show A Host" {
|
||||
t.Errorf("Author = %q, want %q", got.Author, "Show A Host")
|
||||
}
|
||||
if got.Year != 2024 {
|
||||
t.Errorf("Year = %d, want 2024", got.Year)
|
||||
}
|
||||
if len(got.Episodes) != 3 {
|
||||
t.Fatalf("got %d episodes, want 3", len(got.Episodes))
|
||||
}
|
||||
for i, ep := range got.Episodes {
|
||||
wantTrack := i + 1
|
||||
if ep.Track != wantTrack {
|
||||
t.Errorf("episode %d Track = %d, want %d", i, ep.Track, wantTrack)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodcastIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
show := &parsedPodcastShow{Title: "Tagged Show", Author: "Host", Year: 2024}
|
||||
episode := parsedPodcastEpisode{Title: "Episode", Track: 3}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
|
||||
show = &parsedPodcastShow{Title: "Tagged Show"}
|
||||
episode = parsedPodcastEpisode{Title: "Episode"}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
|
||||
show = &parsedPodcastShow{}
|
||||
episode = parsedPodcastEpisode{}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanPodcastFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
showDir := filepath.Join(root, "bad-show")
|
||||
if err := os.Mkdir(showDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir show dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(showDir, "episode.mp3"), []byte("not real audio"), 0o644); err != nil {
|
||||
t.Fatalf("write fake audio: %v", err)
|
||||
}
|
||||
|
||||
s := &Scanner{ffprobePath: "definitely-missing-ffprobe"}
|
||||
err := s.ScanPodcastFolder(context.Background(), &models.MediaFolder{ID: 43, Paths: []string{root}})
|
||||
if err == nil {
|
||||
t.Fatal("ScanPodcastFolder returned nil, want aggregate failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "folder_id=43") {
|
||||
t.Fatalf("error = %q, want folder id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanPodcastFolderReturnsCanceledContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanPodcastFolder(ctx, &models.MediaFolder{ID: 43, Paths: []string{root}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ScanPodcastFolder error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemReusesRootScopedContentID(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{contentID: "podcast-root-id"}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Same Show",
|
||||
&parsedPodcastShow{Title: "Same Show", Year: 0, Author: "Host A"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePodcastMediaItem: %v", err)
|
||||
}
|
||||
if got != "podcast-root-id" {
|
||||
t.Fatalf("contentID = %q, want root-scoped id", got)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Another Same Show",
|
||||
&parsedPodcastShow{Title: "Same Show", Year: 0, Author: "Host B"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePodcastMediaItem: %v", err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatal("contentID is empty")
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].ContentID != got || writer.upserts[0].Type != "podcast" {
|
||||
t.Fatalf("upserted item = %+v, contentID %q", writer.upserts[0], got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemPropagatesRootLookupError(t *testing.T) {
|
||||
wantErr := errors.New("root lookup failed")
|
||||
finder := &fakeRootContentFinder{err: wantErr}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
_, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Show",
|
||||
&parsedPodcastShow{Title: "Show"},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("upserts = %d, want 0", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
@@ -233,6 +233,7 @@ func convertProbeData(raw *ffprobeOutput) *ProbeData {
|
||||
}
|
||||
|
||||
pd.Chapters = normalizeChapters(raw.Chapters, pd.Duration)
|
||||
pd.FormatTags = normalizeFormatTags(raw.Format.Tags)
|
||||
|
||||
return pd
|
||||
}
|
||||
@@ -452,6 +453,20 @@ func isHDR(colorTransfer string) bool {
|
||||
return strings.Contains(ct, "smpte2084") || strings.Contains(ct, "arib-std-b67")
|
||||
}
|
||||
|
||||
// normalizeFormatTags lowercases tag keys so callers can look up
|
||||
// "title", "artist", "album" without worrying about ffprobe's mixed-case
|
||||
// output. Trims whitespace from values.
|
||||
func normalizeFormatTags(raw map[string]string) map[string]string {
|
||||
if len(raw) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]string, len(raw))
|
||||
for k, v := range raw {
|
||||
out[strings.ToLower(strings.TrimSpace(k))] = strings.TrimSpace(v)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// detectContainer maps ffprobe format names to common container names.
|
||||
func detectContainer(formatName string) string {
|
||||
// ffprobe format_name can contain multiple names separated by commas
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestProbeFileAudiobookFixture(t *testing.T) {
|
||||
if _, err := exec.LookPath("ffmpeg"); err != nil {
|
||||
t.Skip("ffmpeg not available; skipping fixture-based test")
|
||||
}
|
||||
|
||||
ffprobePath := FFprobePathFromFFmpeg("ffmpeg")
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
// Fall back to plain "ffprobe" on PATH
|
||||
ffprobePath = "ffprobe"
|
||||
if _, err := exec.LookPath(ffprobePath); err != nil {
|
||||
t.Skip("ffprobe not available; skipping fixture-based test")
|
||||
}
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
got, err := ProbeFile(ctx, ffprobePath, "testdata/audiobook_fixtures/single_book/book.m4b")
|
||||
if err != nil {
|
||||
t.Fatalf("ProbeFile: %v", err)
|
||||
}
|
||||
|
||||
// Chapters
|
||||
if len(got.Chapters) != 2 {
|
||||
t.Errorf("Chapters = %d, want 2", len(got.Chapters))
|
||||
} else {
|
||||
if got.Chapters[0].Title != "Intro" {
|
||||
t.Errorf("Chapters[0].Title = %q, want %q", got.Chapters[0].Title, "Intro")
|
||||
}
|
||||
if got.Chapters[1].Title != "Outro" {
|
||||
t.Errorf("Chapters[1].Title = %q, want %q", got.Chapters[1].Title, "Outro")
|
||||
}
|
||||
}
|
||||
|
||||
// Format tags
|
||||
if got.FormatTags == nil {
|
||||
t.Fatal("FormatTags is nil")
|
||||
}
|
||||
if title := got.FormatTags["title"]; title != "Test Audiobook" {
|
||||
t.Errorf("FormatTags[\"title\"] = %q, want %q", title, "Test Audiobook")
|
||||
}
|
||||
if artist := got.FormatTags["artist"]; artist != "Test Author" {
|
||||
t.Errorf("FormatTags[\"artist\"] = %q, want %q", artist, "Test Author")
|
||||
}
|
||||
if album := got.FormatTags["album"]; album != "Test Series" {
|
||||
t.Errorf("FormatTags[\"album\"] = %q, want %q", album, "Test Series")
|
||||
}
|
||||
}
|
||||
+95
-15
@@ -108,6 +108,9 @@ type Scanner struct {
|
||||
folderRepo *catalog.FolderRepository
|
||||
libraryRepo *catalog.LibraryItemRepository
|
||||
episodeLibraryRepo *catalog.EpisodeLibraryRepository
|
||||
itemRepo *catalog.ItemRepository
|
||||
personRepo *catalog.PersonRepository
|
||||
episodeRepo *catalog.EpisodeRepository
|
||||
ffprobePath string
|
||||
s3Client *s3client.Client // public assets bucket (may be nil)
|
||||
workers int
|
||||
@@ -157,6 +160,9 @@ func NewScanner(fileRepo *FileRepository, ffprobePath string, s3Client *s3client
|
||||
folderRepo: catalog.NewFolderRepository(fileRepo.Pool()),
|
||||
libraryRepo: catalog.NewLibraryItemRepository(fileRepo.Pool()),
|
||||
episodeLibraryRepo: catalog.NewEpisodeLibraryRepository(fileRepo.Pool()),
|
||||
itemRepo: catalog.NewItemRepository(fileRepo.Pool()),
|
||||
personRepo: catalog.NewPersonRepository(fileRepo.Pool()),
|
||||
episodeRepo: catalog.NewEpisodeRepository(fileRepo.Pool()),
|
||||
ffprobePath: ffprobePath,
|
||||
s3Client: s3Client,
|
||||
workers: workers,
|
||||
@@ -192,9 +198,28 @@ func (s *Scanner) SetMetadataQueueProducer(producer MetadataQueueProducer) {
|
||||
// ScanFolder walks a media folder's directory tree, discovers media files,
|
||||
// probes them for technical data, and upserts them into the database.
|
||||
// Files previously in the DB that no longer exist on disk are marked as missing.
|
||||
//
|
||||
// Audiobook libraries are handled by ScanAudiobookFolder and podcast
|
||||
// libraries by ScanPodcastFolder; both bypass the per-file movie/TV
|
||||
// pipeline entirely.
|
||||
func (s *Scanner) ScanFolder(ctx context.Context, folder *models.MediaFolder) (*ScanResult, error) {
|
||||
watchCtx, stopWatch := s.watchFolderContext(ctx, folder.ID)
|
||||
defer stopWatch()
|
||||
|
||||
if isAudiobookLibraryType(folder.Type) {
|
||||
if err := s.ScanAudiobookFolder(watchCtx, folder); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanResult{}, nil
|
||||
}
|
||||
|
||||
if isPodcastLibraryType(folder.Type) {
|
||||
if err := s.ScanPodcastFolder(watchCtx, folder); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &ScanResult{}, nil
|
||||
}
|
||||
|
||||
return s.scanPaths(watchCtx, folder, folder.Paths, folder.Paths, true)
|
||||
}
|
||||
|
||||
@@ -215,6 +240,61 @@ func isMovieLibraryType(libraryType string) bool {
|
||||
return false
|
||||
}
|
||||
}
|
||||
func isAudiobookLibraryType(libraryType string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(libraryType)) {
|
||||
case "audiobook", "audiobooks":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
func isPodcastLibraryType(libraryType string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(libraryType)) {
|
||||
case "podcast", "podcasts":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// walkMode tells walkLogicalTree which file extensions to surface and
|
||||
// which library-specific filename heuristics (sample/extra skipping)
|
||||
// to apply.
|
||||
type walkMode int
|
||||
|
||||
const (
|
||||
walkModeVideo walkMode = iota // bare video walk: video extensions, no movie skipping
|
||||
walkModeMovie // movie library: video extensions + sample/extra skipping
|
||||
walkModeAudiobook // audiobook library: audio extensions, no skipping
|
||||
walkModePodcast // podcast library: audio extensions, no skipping
|
||||
)
|
||||
|
||||
// walkModeFor derives a walkMode from a media_folders.type string.
|
||||
// Unknown types default to walkModeVideo so existing call sites that
|
||||
// pass arbitrary types preserve their prior behavior.
|
||||
func walkModeFor(folderType string) walkMode {
|
||||
switch {
|
||||
case isMovieLibraryType(folderType):
|
||||
return walkModeMovie
|
||||
case isAudiobookLibraryType(folderType):
|
||||
return walkModeAudiobook
|
||||
case isPodcastLibraryType(folderType):
|
||||
return walkModePodcast
|
||||
default:
|
||||
return walkModeVideo
|
||||
}
|
||||
}
|
||||
|
||||
// acceptsExt reports whether the given lowercased extension belongs to
|
||||
// the file types this walk mode is looking for.
|
||||
func (m walkMode) acceptsExt(ext string) bool {
|
||||
switch m {
|
||||
case walkModeAudiobook, walkModePodcast:
|
||||
return audioExtensions[ext]
|
||||
default:
|
||||
return videoExtensions[ext]
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalWalkPath(path string) (string, error) {
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
@@ -236,7 +316,7 @@ func walkLogicalTree(
|
||||
ctx context.Context,
|
||||
logicalPath string,
|
||||
physicalPath string,
|
||||
movieLibrary bool,
|
||||
mode walkMode,
|
||||
visitedPhysicalDirs map[string]struct{},
|
||||
filePaths *[]string,
|
||||
) error {
|
||||
@@ -264,22 +344,22 @@ func walkLogicalTree(
|
||||
return nil
|
||||
}
|
||||
if targetInfo.IsDir() {
|
||||
return walkLogicalTree(ctx, logicalPath, resolved, movieLibrary, visitedPhysicalDirs, filePaths)
|
||||
return walkLogicalTree(ctx, logicalPath, resolved, mode, visitedPhysicalDirs, filePaths)
|
||||
}
|
||||
if movieLibrary && shouldSkipMovieSupplementalFile(logicalPath) {
|
||||
if mode == walkModeMovie && shouldSkipMovieSupplementalFile(logicalPath) {
|
||||
return nil
|
||||
}
|
||||
if videoExtensions[strings.ToLower(filepath.Ext(logicalPath))] {
|
||||
if mode.acceptsExt(strings.ToLower(filepath.Ext(logicalPath))) {
|
||||
*filePaths = append(*filePaths, logicalPath)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if !info.IsDir() {
|
||||
if movieLibrary && shouldSkipMovieSupplementalFile(logicalPath) {
|
||||
if mode == walkModeMovie && shouldSkipMovieSupplementalFile(logicalPath) {
|
||||
return nil
|
||||
}
|
||||
if videoExtensions[strings.ToLower(filepath.Ext(logicalPath))] {
|
||||
if mode.acceptsExt(strings.ToLower(filepath.Ext(logicalPath))) {
|
||||
*filePaths = append(*filePaths, logicalPath)
|
||||
}
|
||||
return nil
|
||||
@@ -298,7 +378,7 @@ func walkLogicalTree(
|
||||
if isIgnoredDirectoryPath(logicalPath) {
|
||||
return nil
|
||||
}
|
||||
if movieLibrary && shouldSkipMovieSupplementalDir(logicalPath) {
|
||||
if mode == walkModeMovie && shouldSkipMovieSupplementalDir(logicalPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -329,31 +409,31 @@ func walkLogicalTree(
|
||||
continue
|
||||
}
|
||||
if targetInfo.IsDir() {
|
||||
if err := walkLogicalTree(ctx, logicalChild, resolved, movieLibrary, visitedPhysicalDirs, filePaths); err != nil {
|
||||
if err := walkLogicalTree(ctx, logicalChild, resolved, mode, visitedPhysicalDirs, filePaths); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if movieLibrary && shouldSkipMovieSupplementalFile(logicalChild) {
|
||||
if mode == walkModeMovie && shouldSkipMovieSupplementalFile(logicalChild) {
|
||||
continue
|
||||
}
|
||||
if videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))] {
|
||||
if mode.acceptsExt(strings.ToLower(filepath.Ext(entry.Name()))) {
|
||||
*filePaths = append(*filePaths, logicalChild)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
if err := walkLogicalTree(ctx, logicalChild, physicalChild, movieLibrary, visitedPhysicalDirs, filePaths); err != nil {
|
||||
if err := walkLogicalTree(ctx, logicalChild, physicalChild, mode, visitedPhysicalDirs, filePaths); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if movieLibrary && shouldSkipMovieSupplementalFile(logicalChild) {
|
||||
if mode == walkModeMovie && shouldSkipMovieSupplementalFile(logicalChild) {
|
||||
continue
|
||||
}
|
||||
if videoExtensions[strings.ToLower(filepath.Ext(entry.Name()))] {
|
||||
if mode.acceptsExt(strings.ToLower(filepath.Ext(entry.Name()))) {
|
||||
*filePaths = append(*filePaths, logicalChild)
|
||||
}
|
||||
}
|
||||
@@ -364,7 +444,7 @@ func walkLogicalTree(
|
||||
func collectLogicalFilePaths(ctx context.Context, walkRoots []string, libraryType string) ([]string, error) {
|
||||
filePaths := make([]string, 0)
|
||||
visitedPhysicalDirs := make(map[string]struct{})
|
||||
movieLibrary := isMovieLibraryType(libraryType)
|
||||
mode := walkModeFor(libraryType)
|
||||
|
||||
for _, rootPath := range walkRoots {
|
||||
if ctx != nil {
|
||||
@@ -376,7 +456,7 @@ func collectLogicalFilePaths(ctx context.Context, walkRoots []string, libraryTyp
|
||||
if cleanRoot == "" || cleanRoot == "." {
|
||||
continue
|
||||
}
|
||||
if err := walkLogicalTree(ctx, cleanRoot, cleanRoot, movieLibrary, visitedPhysicalDirs, &filePaths); err != nil {
|
||||
if err := walkLogicalTree(ctx, cleanRoot, cleanRoot, mode, visitedPhysicalDirs, &filePaths); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,3 +160,43 @@ func testStringSliceContains(values []string, target string) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestIsAudiobookLibraryType(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"audiobooks", true},
|
||||
{"audiobook", true},
|
||||
{"Audiobook", true},
|
||||
{" AUDIOBOOKS ", true},
|
||||
{"movies", false},
|
||||
{"series", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isAudiobookLibraryType(tc.in); got != tc.want {
|
||||
t.Errorf("isAudiobookLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPodcastLibraryType(t *testing.T) {
|
||||
cases := []struct {
|
||||
in string
|
||||
want bool
|
||||
}{
|
||||
{"podcasts", true},
|
||||
{"podcast", true},
|
||||
{"Podcast", true},
|
||||
{" PODCASTS ", true},
|
||||
{"series", false},
|
||||
{"audiobooks", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := isPodcastLibraryType(tc.in); got != tc.want {
|
||||
t.Errorf("isPodcastLibraryType(%q) = %v, want %v", tc.in, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user