Compare commits
62
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
7fb9bef7ed | ||
|
|
843bfdf3eb | ||
|
|
dd04f5b354 | ||
|
|
86a77beae2 | ||
|
|
0327336ed9 | ||
|
|
5f317da348 | ||
|
|
3460678b6b | ||
|
|
bde7fc80b3 | ||
|
|
ce6e85ddb9 | ||
|
|
165b0205ae | ||
|
|
bbc421a809 | ||
|
|
399807a6fe | ||
|
|
b16e014dc6 | ||
|
|
5e43be2278 | ||
|
|
edc01ef733 | ||
|
|
69c5e6d1a1 | ||
|
|
0cd37aa618 | ||
|
|
4d894379df | ||
|
|
44fc0090c7 | ||
|
|
38c7fb44e1 | ||
|
|
c0222a55e1 | ||
|
|
ea6636f9d9 | ||
|
|
8c172ec9dd | ||
|
|
514b72cddb | ||
|
|
6983925b2e | ||
|
|
d897124e94 | ||
|
|
543534a058 | ||
|
|
234491f1cc | ||
|
|
cb0cb04534 | ||
|
|
d4dc695e1d | ||
|
|
f4b9d4cf38 | ||
|
|
fd41f56381 | ||
|
|
aef670c3bc | ||
|
|
3a43537e67 | ||
|
|
bc2b815391 | ||
|
|
7af0cf241e | ||
|
|
209fb50f16 | ||
|
|
a5a0fad86e | ||
|
|
9c6964e1a1 | ||
|
|
901285b225 | ||
|
|
620cd2746b | ||
|
|
355da1a4fd | ||
|
|
8deb18273d | ||
|
|
a8227723bf | ||
|
|
28ef3ac3c8 | ||
|
|
6d77e5a135 | ||
|
|
09204687fd | ||
|
|
1fb09efc9b | ||
|
|
064963f45f | ||
|
|
214171ac90 | ||
|
|
31b2716089 | ||
|
|
6564ee235a | ||
|
|
c16d876007 | ||
|
|
266b4453ca | ||
|
|
3ba4373e72 | ||
|
|
0163df3683 | ||
|
|
356c014f41 | ||
|
|
17c4f19db0 | ||
|
|
807384f4af | ||
|
|
c5daad0dcb | ||
|
|
1490b6318d | ||
|
|
1e36d42bcc |
@@ -1,7 +1,7 @@
|
||||
# Repository Guidelines
|
||||
|
||||
## Project Structure & Module Organization
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/` as paired numbered `.up.sql` and `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
`cmd/silo` contains the main server entrypoint. Backend code lives in `internal/`, organized by domain (`api`, `catalog`, `metadata`, `playback`, `scanner`, `jellycompat`, etc.); keep new code in the package that owns the behavior instead of creating catch-all helpers. Database changes belong in `migrations/sql/` as Goose SQL migrations. Legacy converted migrations intentionally keep their original numeric versions so existing `schema_versions` rows can bootstrap cleanly into Goose. New migrations must use timestamped filenames created with `make migrate-create NAME=add_thing`; do not run `goose fix` or create paired `.up.sql` / `.down.sql` files. The React frontend lives in `web/src/`, with feature code split across `components/`, `pages/`, `hooks/`, `player/`, and `lib/`. Reference material belongs in `docs/architecture/` or `docs/superpowers/{specs,plans}/`; ad hoc SQL helpers live in `scripts/`.
|
||||
|
||||
When creating or editing `docs/superpowers/specs/` or `docs/superpowers/plans/`, never include local absolute filesystem paths or transient worktree IDs. Use repository-relative paths and wording like "Commands assume the repository root is the cwd."
|
||||
|
||||
@@ -53,6 +53,7 @@ Use the checked-in `Makefile` for the common paths:
|
||||
- `make dev-frontend`: start the Vite dev server with HMR
|
||||
- `make dev-proxy` / `make dev-transcode`: run standalone worker modes
|
||||
- `make lint`: run `golangci-lint` and frontend ESLint
|
||||
- `make migrate-status` / `make migrate-up`: inspect or apply Goose migrations through Silo's legacy-safe bootstrapping runner
|
||||
|
||||
Run before opening a merge request:
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks
|
||||
.PHONY: frontend build dev-frontend dev-backend dev-proxy dev-transcode lint clean jellyfin-web-bundle migrate-continuum-check verify-local-paths install-hooks migrate-create migrate-validate migrate-status migrate-up
|
||||
|
||||
GIT_COMMON_DIR := $(strip $(shell git rev-parse --git-common-dir 2>/dev/null))
|
||||
MAIN_CHECKOUT_ROOT := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/..))
|
||||
SHARED_MAKEFILE_LOCAL := $(if $(GIT_COMMON_DIR),$(abspath $(GIT_COMMON_DIR)/../Makefile.local))
|
||||
DEFAULT_PLUGIN_SDK_DIR := $(abspath ../silo-plugin-sdk)
|
||||
SHARED_PLUGIN_SDK_DIR := $(if $(MAIN_CHECKOUT_ROOT),$(abspath $(MAIN_CHECKOUT_ROOT)/../silo-plugin-sdk))
|
||||
GOOSE := go run github.com/pressly/goose/v3/cmd/goose@v3.27.1
|
||||
GOOSE_DIR := migrations/sql
|
||||
ENV_FILE ?= .env
|
||||
|
||||
ifneq ($(wildcard $(DEFAULT_PLUGIN_SDK_DIR)),)
|
||||
DEV_PLUGIN_SDK_DIR ?= $(DEFAULT_PLUGIN_SDK_DIR)
|
||||
@@ -54,6 +57,23 @@ lint:
|
||||
verify-local-paths:
|
||||
scripts/check-local-path-leaks.sh
|
||||
|
||||
# Create a timestamped Goose SQL migration. Usage: make migrate-create NAME=add_thing
|
||||
migrate-create:
|
||||
@if [ -z "$(NAME)" ]; then echo "usage: make migrate-create NAME=add_thing"; exit 1; fi
|
||||
$(GOOSE) -dir $(GOOSE_DIR) create "$(NAME)" sql
|
||||
|
||||
# Validate Goose migration annotations and SQL parsing without touching a database.
|
||||
migrate-validate:
|
||||
$(GOOSE) -dir $(GOOSE_DIR) validate
|
||||
|
||||
# Show Goose migration status through Silo's bootstrapping runner.
|
||||
migrate-status:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-status
|
||||
|
||||
# Apply pending Goose migrations through Silo's bootstrapping runner.
|
||||
migrate-up:
|
||||
go run ./cmd/silo/ --env "$(ENV_FILE)" --migrate-only
|
||||
|
||||
# Install repo-local git hooks for this checkout/worktree.
|
||||
install-hooks:
|
||||
@existing="$$(git config --local core.hooksPath 2>/dev/null || true)"; \
|
||||
|
||||
@@ -178,8 +178,36 @@ Technical notes:
|
||||
| `make dev-backend` | Run Go backend (integrated mode) |
|
||||
| `make dev-proxy` | Run a standalone proxy node |
|
||||
| `make dev-transcode` | Run a standalone transcode node |
|
||||
| `make migrate-create NAME=add_thing` | Create a timestamped Goose SQL migration |
|
||||
| `make migrate-validate` | Validate Goose migration files without touching a database |
|
||||
| `make migrate-status` | Show Goose migration status using Silo's bootstrapping runner |
|
||||
| `make migrate-up` | Apply pending Goose migrations using Silo's bootstrapping runner |
|
||||
| `make clean` | Remove build artifacts |
|
||||
|
||||
### Database Migrations
|
||||
|
||||
PostgreSQL schema migrations are managed by Goose. Migration SQL files live in
|
||||
`migrations/sql/` and use Goose annotations. Converted legacy migrations keep
|
||||
their original numeric versions so existing `schema_versions` rows can bootstrap
|
||||
cleanly into Goose without replaying old SQL. New migrations should be created
|
||||
with timestamped filenames:
|
||||
|
||||
```sh
|
||||
make migrate-create NAME=add_thing
|
||||
make migrate-validate
|
||||
```
|
||||
|
||||
Do not run `goose fix`; timestamped migrations are the repository policy because
|
||||
they avoid version collisions across parallel PRs. The existing `001`-style
|
||||
files are historical compatibility records, not the naming pattern for new work.
|
||||
Runtime migrations are applied by the integrated/API server only. Proxy and
|
||||
transcode modes never mutate schema.
|
||||
For existing installs, use `make migrate-status` and `make migrate-up` rather
|
||||
than invoking the Goose CLI directly; those targets copy legacy
|
||||
`schema_versions` rows into `public.goose_db_version` under the migration lock
|
||||
before reading or applying migrations. Set `ENV_FILE=path/to/.env` when the
|
||||
database URL should be read from a non-default env file.
|
||||
|
||||
### Running Tests
|
||||
|
||||
```sh
|
||||
@@ -285,5 +313,5 @@ internal/
|
||||
scanner/ Media file discovery and FFProbe
|
||||
worker/ Background jobs (scan, match, reconcile)
|
||||
web/ React + TypeScript frontend (Vite, Tailwind, shadcn/ui)
|
||||
migrations/ PostgreSQL schema migrations
|
||||
migrations/sql/ Goose-managed PostgreSQL schema migrations
|
||||
```
|
||||
|
||||
+278
-30
@@ -5,7 +5,6 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
@@ -16,6 +15,7 @@ import (
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
@@ -25,11 +25,11 @@ import (
|
||||
chimiddleware "github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/google/uuid"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
sdkcapability "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginsdk/capability"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/activitylog"
|
||||
@@ -56,7 +56,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/logfilter"
|
||||
"github.com/Silo-Server/silo-server/internal/logstream"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/markers/introdb"
|
||||
"github.com/Silo-Server/silo-server/internal/mdblist"
|
||||
"github.com/Silo-Server/silo-server/internal/metadata"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -255,6 +254,8 @@ func maybeApplyPostgresTuning(ctx context.Context, pool *pgxpool.Pool, appMaxCon
|
||||
|
||||
func main() {
|
||||
envFile := flag.String("env", ".env", "path to .env bootstrap file")
|
||||
migrateOnly := flag.Bool("migrate-only", false, "apply database migrations and exit")
|
||||
migrateStatus := flag.Bool("migrate-status", false, "show database migration status and exit")
|
||||
flag.Parse()
|
||||
|
||||
ctx := context.Background()
|
||||
@@ -274,12 +275,48 @@ func main() {
|
||||
defer pool.Close()
|
||||
slog.Info("connected to PostgreSQL")
|
||||
|
||||
if *migrateStatus {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
statuses, statusErr := database.MigrationStatuses(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if statusErr != nil {
|
||||
log.Fatalf("failed to read migration status: %v", statusErr)
|
||||
}
|
||||
|
||||
fmt.Printf("%-8s %8s %-25s %s\n", "STATE", "VERSION", "APPLIED_AT", "MIGRATION")
|
||||
for _, status := range statuses {
|
||||
appliedAt := "-"
|
||||
if !status.AppliedAt.IsZero() {
|
||||
appliedAt = status.AppliedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
source := status.Source
|
||||
if source != "" {
|
||||
source = filepath.Base(source)
|
||||
} else {
|
||||
source = "-"
|
||||
}
|
||||
fmt.Printf("%-8s %8d %-25s %s\n", status.State, status.Version, appliedAt, source)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if *migrateOnly {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql")
|
||||
migCancel()
|
||||
if migErr != nil {
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
slog.Info("database migrations applied")
|
||||
return
|
||||
}
|
||||
|
||||
// Run migrations only for integrated/api modes. Proxy and transcode nodes
|
||||
// should never alter the schema — they may scale independently and would
|
||||
// race or apply migrations before the primary node is deliberately upgraded.
|
||||
if bc.Mode == "integrated" || bc.Mode == "api" || bc.Mode == "" {
|
||||
migCtx, migCancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "."); migErr != nil {
|
||||
if migErr := database.RunMigrations(migCtx, pool, migrations.FS, "sql"); migErr != nil {
|
||||
migCancel()
|
||||
log.Fatalf("failed to run migrations: %v", migErr)
|
||||
}
|
||||
@@ -508,36 +545,34 @@ func main() {
|
||||
}
|
||||
if deps.DB != nil {
|
||||
markerRegistry := markers.NewRegistry(slog.Default())
|
||||
introdbAPIKey, err := settingsRepo.Get(appCtx, "introdb.api_key")
|
||||
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
|
||||
slog.Warn("load introdb.api_key from settings failed; provider will run with no key",
|
||||
markerProviderConfig := markers.NewProviderConfigStore(deps.DB)
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("load marker provider config failed; falling back to registration-order fetch",
|
||||
"error", err)
|
||||
introdbAPIKey = ""
|
||||
}
|
||||
introdbClient := introdb.NewClient(introdbAPIKey)
|
||||
if err := markerRegistry.Register(introdb.NewProvider(introdbClient)); err != nil {
|
||||
log.Fatalf("register introdb marker provider: %v", err)
|
||||
}
|
||||
deps.OnServerSettingUpdated = func(_ context.Context, key, value string) {
|
||||
if key == "introdb.api_key" {
|
||||
introdbClient.SetAPIKey(value)
|
||||
} else {
|
||||
markerRegistry.UseConfigStore(markerProviderConfig)
|
||||
if deps.EventBus != nil {
|
||||
if err := deps.EventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventMarkerProviderConfigChanged {
|
||||
return
|
||||
}
|
||||
if err := markerProviderConfig.Reload(appCtx); err != nil {
|
||||
slog.Warn("reload marker provider config failed", "provider", event.Payload, "error", err)
|
||||
}
|
||||
}); err != nil {
|
||||
slog.Warn("subscribe marker provider config reload failed", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if eventBus != nil {
|
||||
_ = eventBus.Subscribe(appCtx, cache.ChannelAdmin, func(event cache.Event) {
|
||||
if event.Type != cache.EventSettingsChanged || event.Payload != "introdb.api_key" {
|
||||
return
|
||||
}
|
||||
value, loadErr := settingsRepo.Get(context.Background(), "introdb.api_key")
|
||||
if loadErr != nil {
|
||||
slog.Warn("introdb api key reload failed", "error", loadErr)
|
||||
return
|
||||
}
|
||||
introdbClient.SetAPIKey(value)
|
||||
})
|
||||
}
|
||||
deps.MarkerProviderConfig = markerProviderConfig
|
||||
deps.MarkerRegistry = markerRegistry
|
||||
deps.MarkerResolver = markers.NewDBExternalIDResolver(deps.DB)
|
||||
markerResolver := markers.NewDBExternalIDResolver(deps.DB)
|
||||
deps.MarkerResolver = markerResolver
|
||||
markerContributionStore := markers.NewContributionStore(deps.DB)
|
||||
deps.MarkerContributionStore = markerContributionStore
|
||||
deps.MarkerContributionService = markers.NewContributionService(
|
||||
markerRegistry, markerResolver, markerProviderConfig, markerContributionStore, slog.Default(),
|
||||
)
|
||||
}
|
||||
var watchProviderService *watchsync.Service
|
||||
if deps.DB != nil {
|
||||
@@ -749,6 +784,22 @@ func main() {
|
||||
installer,
|
||||
plugins.NewHostAdapter(pluginHost),
|
||||
)
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
markerPluginResolver := markers.NewPluginResolverAdapter(pluginService)
|
||||
pluginService.AddLifecycleHook(func(ctx context.Context) {
|
||||
if err := reloadMarkerPluginProviders(
|
||||
ctx,
|
||||
deps.MarkerRegistry,
|
||||
deps.MarkerProviderConfig,
|
||||
installationStore,
|
||||
runtimeConfigStore,
|
||||
settingsRepo,
|
||||
markerPluginResolver,
|
||||
); err != nil {
|
||||
slog.Warn("reload marker plugin providers failed", "error", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if err := pluginService.PreloadEnabled(appCtx); err != nil {
|
||||
log.Fatalf("preload enabled plugins: %v", err)
|
||||
}
|
||||
@@ -1387,6 +1438,11 @@ func main() {
|
||||
if deps.IntroAnalyzer != nil {
|
||||
taskMgr.Register(tasks.NewDetectIntroMarkersTask(deps.IntroAnalyzer, settingsRepo))
|
||||
}
|
||||
if deps.MarkerContributionService != nil && deps.MarkerProviderConfig != nil && deps.MarkerContributionStore != nil && deps.FileRepo != nil {
|
||||
taskMgr.Register(tasks.NewContributeMarkersTask(
|
||||
deps.MarkerContributionService, deps.MarkerProviderConfig, deps.MarkerContributionStore, deps.FileRepo,
|
||||
))
|
||||
}
|
||||
if chapterBackfiller, ok := deps.ChapterThumbnailQueuer.(*chapterthumbs.Service); ok {
|
||||
taskMgr.Register(tasks.NewChapterThumbnailBackfillTask(chapterBackfiller, 25))
|
||||
}
|
||||
@@ -2129,6 +2185,198 @@ func configureS3Clients(cfg *config.Config, deps *api.Dependencies) {
|
||||
}
|
||||
}
|
||||
|
||||
type markerPluginCapabilityStore interface {
|
||||
ListEnabled(ctx context.Context) ([]*plugins.Installation, error)
|
||||
ListCapabilities(ctx context.Context, installationID int) ([]*plugins.Capability, error)
|
||||
}
|
||||
|
||||
type markerPluginRuntimeConfigStore interface {
|
||||
ListGlobalConfigs(ctx context.Context, installationID int) ([]*plugins.RuntimeConfig, error)
|
||||
PutGlobalConfig(ctx context.Context, installationID int, key string, value map[string]any) error
|
||||
}
|
||||
|
||||
type markerLegacySettingsStore interface {
|
||||
Get(ctx context.Context, key string) (string, error)
|
||||
}
|
||||
|
||||
func reloadMarkerPluginProviders(
|
||||
ctx context.Context,
|
||||
registry *markers.Registry,
|
||||
configStore *markers.ProviderConfigStore,
|
||||
store markerPluginCapabilityStore,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
resolver *markers.PluginResolverAdapter,
|
||||
) error {
|
||||
if registry == nil {
|
||||
return nil
|
||||
}
|
||||
var providers []markers.Provider
|
||||
if store == nil || resolver == nil {
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
installations, err := store.ListEnabled(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list enabled plugin installations: %w", err)
|
||||
}
|
||||
sort.Slice(installations, func(i, j int) bool {
|
||||
if installations[i] == nil {
|
||||
return false
|
||||
}
|
||||
if installations[j] == nil {
|
||||
return true
|
||||
}
|
||||
return installations[i].ID < installations[j].ID
|
||||
})
|
||||
|
||||
nextPriority := 1000
|
||||
for _, installation := range installations {
|
||||
if installation == nil {
|
||||
continue
|
||||
}
|
||||
capabilities, err := store.ListCapabilities(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err)
|
||||
}
|
||||
sort.Slice(capabilities, func(i, j int) bool {
|
||||
if capabilities[i] == nil {
|
||||
return false
|
||||
}
|
||||
if capabilities[j] == nil {
|
||||
return true
|
||||
}
|
||||
return capabilities[i].ID < capabilities[j].ID
|
||||
})
|
||||
for _, capability := range capabilities {
|
||||
if capability == nil || capability.Type != sdkcapability.MarkerProvider {
|
||||
continue
|
||||
}
|
||||
descriptor, err := plugins.DecodeCapability(capability)
|
||||
if err != nil {
|
||||
return fmt.Errorf("decode marker provider capability %d/%s: %w", installation.ID, capability.ID, err)
|
||||
}
|
||||
metadataMap := markerCapabilityMetadata(descriptor)
|
||||
provider, err := markers.NewPluginProvider(markers.PluginProviderOptions{
|
||||
InstallationID: installation.ID,
|
||||
CapabilityID: capability.ID,
|
||||
DisplayName: firstNonEmptyMarkerText(descriptor.GetDisplayName(), capability.ID),
|
||||
PluginID: installation.PluginID,
|
||||
RequiredExternalIDs: markers.PluginRequiredExternalIDsFromMetadata(metadataMap),
|
||||
}, resolver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
providers = append(providers, provider)
|
||||
|
||||
priority := nextPriority
|
||||
nextPriority++
|
||||
if configuredPriority, ok := markers.PluginDefaultFetchPriorityFromMetadata(metadataMap); ok {
|
||||
priority = configuredPriority
|
||||
}
|
||||
if configStore != nil {
|
||||
defaultConfig := markers.ProviderConfig{
|
||||
Provider: provider.ID(),
|
||||
FetchEnabled: true,
|
||||
FetchPriority: priority,
|
||||
ContributeEnabled: false,
|
||||
ContributeAutoLocal: false,
|
||||
ContributeMinConfidence: 0.95,
|
||||
}
|
||||
if legacy, ok := legacyIntroDBProviderConfig(configStore, installation, capability, provider.ID()); ok {
|
||||
defaultConfig = legacy
|
||||
}
|
||||
if err := configStore.Ensure(ctx, defaultConfig); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := copyLegacyIntroDBPluginConfig(ctx, runtimeConfigs, legacySettings, installation, capability); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return registry.SetProviders(providers)
|
||||
}
|
||||
|
||||
func legacyIntroDBProviderConfig(
|
||||
configStore *markers.ProviderConfigStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
providerID string,
|
||||
) (markers.ProviderConfig, bool) {
|
||||
if configStore == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
if _, exists := configStore.Get(providerID); exists {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy, ok := configStore.Get("introdb")
|
||||
if !ok {
|
||||
return markers.ProviderConfig{}, false
|
||||
}
|
||||
legacy.Provider = providerID
|
||||
return legacy, true
|
||||
}
|
||||
|
||||
func copyLegacyIntroDBPluginConfig(
|
||||
ctx context.Context,
|
||||
runtimeConfigs markerPluginRuntimeConfigStore,
|
||||
legacySettings markerLegacySettingsStore,
|
||||
installation *plugins.Installation,
|
||||
capability *plugins.Capability,
|
||||
) error {
|
||||
if runtimeConfigs == nil ||
|
||||
legacySettings == nil ||
|
||||
installation == nil ||
|
||||
capability == nil ||
|
||||
installation.PluginID != "silo.theintrodb" ||
|
||||
capability.ID != "introdb" {
|
||||
return nil
|
||||
}
|
||||
configs, err := runtimeConfigs.ListGlobalConfigs(ctx, installation.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("list TheIntroDB plugin config: %w", err)
|
||||
}
|
||||
for _, config := range configs {
|
||||
if config != nil && config.Key == "account" {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
apiKey, err := legacySettings.Get(ctx, "introdb.api_key")
|
||||
if err != nil {
|
||||
return fmt.Errorf("load legacy introdb.api_key: %w", err)
|
||||
}
|
||||
if strings.TrimSpace(apiKey) == "" {
|
||||
return nil
|
||||
}
|
||||
if err := runtimeConfigs.PutGlobalConfig(ctx, installation.ID, "account", map[string]any{
|
||||
"api_key": strings.TrimSpace(apiKey),
|
||||
}); err != nil {
|
||||
return fmt.Errorf("copy legacy introdb.api_key to plugin config: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markerCapabilityMetadata(descriptor *pluginv1.CapabilityDescriptor) map[string]any {
|
||||
if descriptor == nil || descriptor.GetMetadata() == nil {
|
||||
return nil
|
||||
}
|
||||
return descriptor.GetMetadata().AsMap()
|
||||
}
|
||||
|
||||
func firstNonEmptyMarkerText(values ...string) string {
|
||||
for _, value := range values {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// mapFolderTypeToMediaType maps silo's MediaFolder.Type values
|
||||
// ("movies", "series", "mixed") to the SDK's MediaType values
|
||||
// ("movie", "tv", "mixed"). Unknown values map to "mixed".
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/plugins"
|
||||
)
|
||||
|
||||
type fakeMarkerRuntimeConfigs struct {
|
||||
configs map[int][]*plugins.RuntimeConfig
|
||||
puts []plugins.RuntimeConfig
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) ListGlobalConfigs(_ context.Context, installationID int) ([]*plugins.RuntimeConfig, error) {
|
||||
return append([]*plugins.RuntimeConfig(nil), f.configs[installationID]...), nil
|
||||
}
|
||||
|
||||
func (f *fakeMarkerRuntimeConfigs) PutGlobalConfig(_ context.Context, installationID int, key string, value map[string]any) error {
|
||||
f.puts = append(f.puts, plugins.RuntimeConfig{InstallationID: installationID, Key: key, Value: value})
|
||||
f.configs[installationID] = append(f.configs[installationID], &plugins.RuntimeConfig{
|
||||
InstallationID: installationID,
|
||||
Key: key,
|
||||
Value: value,
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
type fakeMarkerLegacySettings map[string]string
|
||||
|
||||
func (f fakeMarkerLegacySettings) Get(_ context.Context, key string) (string, error) {
|
||||
return f[key], nil
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigCopiesAPIKeyOnce(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
installation := &plugins.Installation{ID: 42, PluginID: "silo.theintrodb"}
|
||||
capability := &plugins.Capability{ID: "introdb"}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": " legacy-key "},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
got := runtimeConfigs.puts[0]
|
||||
if got.InstallationID != 42 || got.Key != "account" || got.Value["api_key"] != "legacy-key" {
|
||||
t.Fatalf("put = %+v, want account api_key copy", got)
|
||||
}
|
||||
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "new-key"},
|
||||
installation,
|
||||
capability,
|
||||
); err != nil {
|
||||
t.Fatalf("second copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 1 {
|
||||
t.Fatalf("second copy overwrote config; puts = %d, want 1", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyLegacyIntroDBPluginConfigIgnoresOtherPlugins(t *testing.T) {
|
||||
runtimeConfigs := &fakeMarkerRuntimeConfigs{configs: map[int][]*plugins.RuntimeConfig{}}
|
||||
if err := copyLegacyIntroDBPluginConfig(
|
||||
context.Background(),
|
||||
runtimeConfigs,
|
||||
fakeMarkerLegacySettings{"introdb.api_key": "legacy-key"},
|
||||
&plugins.Installation{ID: 7, PluginID: "silo.other"},
|
||||
&plugins.Capability{ID: "introdb"},
|
||||
); err != nil {
|
||||
t.Fatalf("copyLegacyIntroDBPluginConfig: %v", err)
|
||||
}
|
||||
if len(runtimeConfigs.puts) != 0 {
|
||||
t.Fatalf("puts = %d, want 0 for non-TheIntroDB plugin", len(runtimeConfigs.puts))
|
||||
}
|
||||
}
|
||||
@@ -17,14 +17,14 @@
|
||||
| Path | Created/Modified | Purpose |
|
||||
|---|---|---|
|
||||
| `docs/superpowers/plans/artifacts/2026-05-24-audiobooks-discovery-findings.md` | Create | Findings doc produced by Task 1. Locks the remaining schema/code decisions. |
|
||||
| `migrations/139_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/139_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/140_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/140_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/141_media_libraries_kind.up.sql` | Create (conditional — see Task 4 step 1) | Add `kind` column to `media_libraries`. |
|
||||
| `migrations/141_media_libraries_kind.down.sql` | Create (same condition) | Drop the column. |
|
||||
| `migrations/142_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/142_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `migrations/147_abs_sessions.up.sql` | Create | New table parallel to `jellycompat_sessions`. |
|
||||
| `migrations/147_abs_sessions.down.sql` | Create | Drop the table. |
|
||||
| `migrations/157_podcast_feeds.up.sql` | Create | New table for RSS-subscribed podcast metadata. |
|
||||
| `migrations/157_podcast_feeds.down.sql` | Create | Drop the table. |
|
||||
| `migrations/159_media_folders_kind_noop.up.sql` | Create (conditional — see Task 4 step 1) | Document `media_folders.type='audiobooks'`; no column change needed. |
|
||||
| `migrations/159_media_folders_kind_noop.down.sql` | Create (same condition) | No-op rollback. |
|
||||
| `migrations/160_audiobooks_feature_flag.up.sql` | Create | Insert `audiobooks.enabled='false'` into `server_settings`. |
|
||||
| `migrations/160_audiobooks_feature_flag.down.sql` | Create | Delete the setting row. |
|
||||
| `internal/audiobooks/doc.go` | Create | Package comment + intent. |
|
||||
| `internal/audiobooks/service.go` | Create | Empty `Service` struct, `New` constructor, `Enabled()` reader. |
|
||||
| `internal/audiobooks/service_test.go` | Create | One unit test proving the package compiles and `Enabled()` returns the setting value. |
|
||||
@@ -240,15 +240,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 2: Migration 139 — `abs_sessions`
|
||||
## Task 2: Migration 147 — `abs_sessions`
|
||||
|
||||
Parallel of `jellycompat_sessions` for Audiobookshelf clients. Lets ABS
|
||||
mobile/desktop apps reconnect without re-authenticating against silo's
|
||||
main `auth_sessions`.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/139_abs_sessions.up.sql`
|
||||
- Create: `migrations/139_abs_sessions.down.sql`
|
||||
- Create: `migrations/147_abs_sessions.up.sql`
|
||||
- Create: `migrations/147_abs_sessions.down.sql`
|
||||
|
||||
> **Note:** If Task 1, Step 1.2 found the next number is not 139, rename
|
||||
> both filenames to match. All other tasks downstream of 139 shift by the
|
||||
@@ -266,7 +266,7 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/139_abs_sessions.up.sql` with:
|
||||
- [ ] **Create** `migrations/147_abs_sessions.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Audiobookshelf-compatible client sessions. Parallel to
|
||||
@@ -277,7 +277,7 @@ main `auth_sessions`.
|
||||
CREATE TABLE IF NOT EXISTS public.abs_sessions (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
user_id INTEGER NOT NULL REFERENCES public.users(id) ON DELETE CASCADE,
|
||||
token TEXT NOT NULL,
|
||||
token_hash TEXT NOT NULL,
|
||||
device_id TEXT NOT NULL,
|
||||
device_name TEXT,
|
||||
client_name TEXT,
|
||||
@@ -287,8 +287,8 @@ main `auth_sessions`.
|
||||
revoked_at TIMESTAMP WITH TIME ZONE
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token
|
||||
ON public.abs_sessions (token);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_abs_sessions_token_hash
|
||||
ON public.abs_sessions (token_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_abs_sessions_user_device
|
||||
ON public.abs_sessions (user_id, device_id);
|
||||
@@ -299,12 +299,12 @@ main `auth_sessions`.
|
||||
|
||||
### Step 2.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/139_abs_sessions.down.sql` with:
|
||||
- [ ] **Create** `migrations/147_abs_sessions.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_last_seen;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_user_device;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token;
|
||||
DROP INDEX IF EXISTS public.idx_abs_sessions_token_hash;
|
||||
DROP TABLE IF EXISTS public.abs_sessions;
|
||||
```
|
||||
|
||||
@@ -328,27 +328,27 @@ main `auth_sessions`.
|
||||
- [ ] **Run** (each command should now succeed):
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.abs_sessions"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 139;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 147;"
|
||||
```
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=139`.
|
||||
Expected: `\d` prints the full column list; the `SELECT` returns one row with `version=147`.
|
||||
|
||||
- [ ] **Run** a sample insert/delete to prove constraints work:
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "
|
||||
INSERT INTO abs_sessions (user_id, token, device_id, device_name, client_name, client_version)
|
||||
VALUES (1, 'test-token-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token = 'test-token-001';
|
||||
INSERT INTO abs_sessions (user_id, token_hash, device_id, device_name, client_name, client_version)
|
||||
VALUES ((SELECT id FROM users ORDER BY id LIMIT 1), 'test-token-hash-001', 'test-device-001', 'Test Device', 'AbsTestClient', '1.0.0');
|
||||
DELETE FROM abs_sessions WHERE token_hash = 'test-token-hash-001';
|
||||
"
|
||||
```
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (User id 1 must exist — silo's first user.)
|
||||
Expected: `INSERT 0 1` then `DELETE 1`. (At least one user must exist.)
|
||||
|
||||
### Step 2.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/139_abs_sessions.up.sql migrations/139_abs_sessions.down.sql
|
||||
git add migrations/147_abs_sessions.up.sql migrations/147_abs_sessions.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 139 add abs_sessions table
|
||||
feat(audiobooks): migration 147 add abs_sessions table
|
||||
|
||||
Parallel of jellycompat_sessions for Audiobookshelf-compatible clients.
|
||||
Lets ABS mobile/desktop apps maintain a device-bound session that
|
||||
@@ -361,15 +361,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Migration 140 — `podcast_feeds`
|
||||
## Task 3: Migration 157 — `podcast_feeds`
|
||||
|
||||
One row per subscribed podcast. The podcast itself lives in `media_items`
|
||||
(`type='podcast'`); this side table carries the RSS-specific metadata the
|
||||
feed refresher needs.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/140_podcast_feeds.up.sql`
|
||||
- Create: `migrations/140_podcast_feeds.down.sql`
|
||||
- Create: `migrations/157_podcast_feeds.up.sql`
|
||||
- Create: `migrations/157_podcast_feeds.down.sql`
|
||||
|
||||
### Step 3.1: Write the failing assertion
|
||||
|
||||
@@ -381,7 +381,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.up.sql` with:
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- RSS-feed metadata for subscribed podcasts. One row per podcast
|
||||
@@ -416,7 +416,7 @@ feed refresher needs.
|
||||
|
||||
### Step 3.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/140_podcast_feeds.down.sql` with:
|
||||
- [ ] **Create** `migrations/157_podcast_feeds.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_podcast_feeds_due_for_refresh;
|
||||
@@ -438,17 +438,17 @@ feed refresher needs.
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "\d public.podcast_feeds"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 140;"
|
||||
sudo docker exec silo-prod-postgres-1 psql -U silo -d silo -c "SELECT version FROM schema_versions WHERE version = 157;"
|
||||
```
|
||||
Expected: column list printed; one row with `version=140`.
|
||||
Expected: column list printed; one row with `version=157`.
|
||||
|
||||
### Step 3.6: Commit
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/140_podcast_feeds.up.sql migrations/140_podcast_feeds.down.sql
|
||||
git add migrations/157_podcast_feeds.up.sql migrations/157_podcast_feeds.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 140 add podcast_feeds table
|
||||
feat(audiobooks): migration 157 add podcast_feeds table
|
||||
|
||||
Side table on media_items for RSS-subscribed podcasts. Holds feed URL,
|
||||
ETag/Last-Modified for conditional fetches, last-refresh timestamp, and
|
||||
@@ -462,7 +462,7 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Migration 141 — `media_libraries.kind` (conditional)
|
||||
## Task 4: Migration 159 — `media_libraries.kind` audiobook value (no-op)
|
||||
|
||||
Adds a discriminator column so the scanner can branch on
|
||||
`audiobooks`/`podcasts` libraries without filename heuristics.
|
||||
@@ -474,8 +474,8 @@ Adds a discriminator column so the scanner can branch on
|
||||
> Otherwise, proceed with the steps below.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/141_media_libraries_kind.up.sql`
|
||||
- Create: `migrations/141_media_libraries_kind.down.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.up.sql`
|
||||
- Create: `migrations/159_media_folders_kind_noop.down.sql`
|
||||
|
||||
### Step 4.1: Write the failing assertion
|
||||
|
||||
@@ -487,7 +487,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.up.sql` with:
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Per-library content discriminator. Lets the scanner choose the right
|
||||
@@ -509,7 +509,7 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
### Step 4.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/141_media_libraries_kind.down.sql` with:
|
||||
- [ ] **Create** `migrations/159_media_folders_kind_noop.down.sql` with:
|
||||
|
||||
```sql
|
||||
DROP INDEX IF EXISTS public.idx_media_libraries_kind;
|
||||
@@ -551,9 +551,9 @@ Adds a discriminator column so the scanner can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/141_media_libraries_kind.up.sql migrations/141_media_libraries_kind.down.sql
|
||||
git add migrations/159_media_folders_kind_noop.up.sql migrations/159_media_folders_kind_noop.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 141 add media_libraries.kind
|
||||
feat(audiobooks): migration 159 document audiobooks media folder type
|
||||
|
||||
Per-library discriminator that lets the scanner pick the right parser
|
||||
('movies', 'tv', 'audiobooks', 'podcasts') instead of inferring from
|
||||
@@ -567,15 +567,15 @@ EOF
|
||||
|
||||
---
|
||||
|
||||
## Task 5: Migration 142 — `audiobooks.enabled` feature flag
|
||||
## Task 5: Migration 160 — `audiobooks.enabled` feature flag
|
||||
|
||||
Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
"audiobooks compiled in but turned off" vs "fully live". Default is
|
||||
`false` so this sub-plan's landing is a strict no-op for users.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/142_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/142_audiobooks_feature_flag.down.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.up.sql`
|
||||
- Create: `migrations/160_audiobooks_feature_flag.down.sql`
|
||||
|
||||
### Step 5.1: Write the failing assertion
|
||||
|
||||
@@ -589,11 +589,11 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.2: Create the up migration
|
||||
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.up.sql` with:
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.up.sql` with:
|
||||
|
||||
```sql
|
||||
-- Master kill-switch for the absorbed audiobooks feature. Defaults to
|
||||
-- 'false' so landing migrations 139-142 is a no-op for users; operators
|
||||
-- 'false' so landing migrations 147/157/159/160 is a no-op for users; operators
|
||||
-- flip this to 'true' at cutover.
|
||||
|
||||
INSERT INTO server_settings (key, value) VALUES ('audiobooks.enabled', 'false')
|
||||
@@ -602,7 +602,7 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
### Step 5.3: Create the down migration
|
||||
|
||||
- [ ] **Create** `migrations/142_audiobooks_feature_flag.down.sql` with:
|
||||
- [ ] **Create** `migrations/160_audiobooks_feature_flag.down.sql` with:
|
||||
|
||||
```sql
|
||||
DELETE FROM server_settings WHERE key = 'audiobooks.enabled';
|
||||
@@ -625,9 +625,9 @@ Adds a `server_settings` row so subsequent sub-plans can branch on
|
||||
|
||||
- [ ] **Run:**
|
||||
```bash
|
||||
git add migrations/142_audiobooks_feature_flag.up.sql migrations/142_audiobooks_feature_flag.down.sql
|
||||
git add migrations/160_audiobooks_feature_flag.up.sql migrations/160_audiobooks_feature_flag.down.sql
|
||||
git commit -m "$(cat <<'EOF'
|
||||
feat(audiobooks): migration 142 add audiobooks.enabled flag
|
||||
feat(audiobooks): migration 160 add audiobooks.enabled flag
|
||||
|
||||
Server-settings row that gates the absorbed audiobooks feature.
|
||||
Defaults to 'false' so sub-plan 1 lands as a strict no-op; subsequent
|
||||
@@ -646,7 +646,7 @@ EOF
|
||||
Creates an empty but compiling Go package, wires its constructor into
|
||||
`cmd/silo/main.go` so the package is referenced from the binary, and
|
||||
ships a unit test that proves the `Enabled()` reader returns the flag
|
||||
value seeded by migration 142.
|
||||
value seeded by migration 160.
|
||||
|
||||
This task proves the package compiles inside the rest of silo without
|
||||
changing any runtime behavior. No routes are mounted. No scheduled
|
||||
@@ -721,7 +721,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.2: Run the test to verify it fails
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
|
||||
Expected: build failure with messages like `undefined: New` and
|
||||
`package audiobooks not found`.
|
||||
@@ -777,9 +777,9 @@ tasks are registered. No DB writes happen.
|
||||
return &Service{settings: settings}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by migration
|
||||
// 142 and toggled by operators) is currently true. Any value other than
|
||||
// the literal string "true" reads as false; this matches how silo
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
@@ -795,7 +795,7 @@ tasks are registered. No DB writes happen.
|
||||
|
||||
### Step 6.5: Run the test to verify it passes
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/...`
|
||||
Expected: `ok github.com/Silo-Server/silo-server/internal/audiobooks ...`
|
||||
|
||||
### Step 6.6: Wire the constructor into `cmd/silo/main.go`
|
||||
@@ -831,13 +831,13 @@ are constructed and pass a settings store to.
|
||||
|
||||
### Step 6.7: Verify everything still compiles and tests pass
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go build ./...`
|
||||
- [ ] **Run:** `go build ./...`
|
||||
Expected: exit 0, no output.
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && go test ./internal/audiobooks/... ./cmd/...`
|
||||
- [ ] **Run:** `go test ./internal/audiobooks/... ./cmd/...`
|
||||
Expected: all passes.
|
||||
|
||||
- [ ] **Run:** `cd /opt/silo-server && make lint`
|
||||
- [ ] **Run:** `make lint`
|
||||
Expected: no new lint findings introduced by this task.
|
||||
|
||||
### Step 6.8: Smoke-test the running binary
|
||||
@@ -883,9 +883,9 @@ EOF
|
||||
| Spec section | Task(s) that cover it |
|
||||
|---|---|
|
||||
| Data model: reused tables | Task 1 (verification only) |
|
||||
| Data model: `abs_sessions` (mig 134→139) | Task 2 |
|
||||
| Data model: `podcast_feeds` (mig 135→140) | Task 3 |
|
||||
| Data model: `media_libraries.kind` (mig 136→141) | Task 4 |
|
||||
| Data model: `abs_sessions` (migration 147) | Task 2 |
|
||||
| Data model: `podcast_feeds` (migration 157) | Task 3 |
|
||||
| Data model: `media_folders.type` audiobook value (migration 159 no-op) | Task 4 |
|
||||
| Data model: data migration = none | n/a — confirmed in spec |
|
||||
| Feature flag (`audiobooks.enabled`) | Task 5 |
|
||||
| Architecture: `internal/audiobooks/` package | Task 6 |
|
||||
@@ -909,9 +909,8 @@ fires).
|
||||
test matches the production interface defined in `service.go`. Method
|
||||
name `GetString`, signature `(ctx context.Context, key string) (string, error)` — consistent across both files.
|
||||
|
||||
**Note on migration numbering:** Tasks 2–5 use numbers `139..142` based
|
||||
on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. Task 1 Step 1.2 re-verifies this; if
|
||||
the executing agent finds the highest existing number has moved, it
|
||||
must shift filenames in Tasks 2–5 by the same delta and update the
|
||||
matching task headings and commit-message references.
|
||||
**Note on migration numbering:** Tasks 2–5 originally used numbers `139..142`
|
||||
based on a snapshot showing `138_search_number_word_normalization.up.sql` as
|
||||
the highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment in this sub-plan — file tags only.
|
||||
**Goal:** Make silo's existing scanner recognize audiobook + podcast libraries (`media_folders.type='audiobooks'` or `'podcasts'`), walk them to find audio files, and write `media_items.type='audiobook'` / `media_items.type='podcast'` rows with chapter metadata and author/narrator role links. No external metadata enrichment runs in this sub-plan; the scanner extracts local tags and identity hints, and the audiobook metadata plugin/provider enriches canonical metadata later.
|
||||
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. No changes to the matching/enrichment metadata queues — those are deferred.
|
||||
**Architecture:** New constants (`PersonKindAuthor=7`, `PersonKindNarrator=8`, `audioExtensions`). New helpers (`isAudiobookLibraryType`, `isPodcastLibraryType`, `SupportsAudioFile`). New parser file `internal/scanner/audiobook.go` for the folder-to-media-items mapping. The chapter extraction reuses the existing `probe.go` ffprobe helper. Author/narrator names are upserted into the existing `people` and `item_people` tables via the same code paths used for movie actors. Matching/enrichment queue integration for the audiobook metadata plugin/provider is deferred.
|
||||
|
||||
**Tech Stack:** Go 1.26 + pgx; ffprobe via `os/exec` (already vendored as `internal/scanner/probe.go`); ID3/MP4 tag parsing — verify what silo already vendors before introducing a new library (acceptable: `github.com/dhowden/tag` if not already present, OR shell out to ffprobe which exposes tag dictionaries).
|
||||
|
||||
@@ -408,8 +408,8 @@ ffmpeg -y -f lavfi -i "anullsrc=r=22050:cl=mono" -t 10 -metadata title="Test Aud
|
||||
-metadata artist="Test Author" -metadata album="Test Series" \
|
||||
-metadata:s:a:0 language=eng \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a
|
||||
# Add chapters via a temporary metadata file
|
||||
cat > /tmp/chapters.ffmeta <<'EOF'
|
||||
# Add chapters via a repo-local metadata fixture
|
||||
cat > internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta <<'EOF'
|
||||
;FFMETADATA1
|
||||
[CHAPTER]
|
||||
TIMEBASE=1/1000
|
||||
@@ -423,9 +423,9 @@ END=10000
|
||||
title=Outro
|
||||
EOF
|
||||
ffmpeg -y -i internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a \
|
||||
-i /tmp/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
-i internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta -map_metadata 1 -codec copy \
|
||||
internal/scanner/testdata/audiobook_fixtures/single_book/book.m4b
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a /tmp/chapters.ffmeta
|
||||
rm internal/scanner/testdata/audiobook_fixtures/single_book/raw.m4a internal/scanner/testdata/audiobook_fixtures/single_book/chapters.ffmeta
|
||||
```
|
||||
|
||||
Add a test that calls the chapter-extraction function on `book.m4b` and asserts:
|
||||
@@ -1008,7 +1008,6 @@ Verify the full sub-plan landed cleanly.
|
||||
### Step 10.1: Full tests + build
|
||||
|
||||
```bash
|
||||
cd /opt/silo-server
|
||||
go build ./...
|
||||
go test ./internal/scanner/... ./internal/models/... ./internal/audiobooks/...
|
||||
go vet ./...
|
||||
@@ -1019,7 +1018,7 @@ All green.
|
||||
### Step 10.2: Rebuild + recreate silo container
|
||||
|
||||
```bash
|
||||
sudo docker build -t silo:latest /opt/silo-server
|
||||
sudo docker build -t silo:latest .
|
||||
sudo docker compose -p silo-prod up -d --force-recreate silo
|
||||
until [ "$(sudo docker inspect -f '{{.State.Health.Status}}' silo-prod-silo-1 2>/dev/null)" = "healthy" ]; do sleep 2; done; echo healthy
|
||||
```
|
||||
@@ -1065,7 +1064,7 @@ Otherwise: no commit needed.
|
||||
| Multi-file audiobooks | T7 |
|
||||
| Filesystem podcasts | T9 |
|
||||
| RSS podcasts | deferred to sub-plan 5 |
|
||||
| External enrichment (OpenLibrary/Google Books) | dropped per spec |
|
||||
| External enrichment through audiobook metadata plugin/provider | deferred per spec |
|
||||
|
||||
**Placeholder scan:** Task 8 has language like "grep for upsertMediaItem or similar in scanner.go to find it" — this is intentional. The scanner's write helpers are too deeply embedded for me to enumerate without reading more code; the implementer subagent will discover them. If a subagent reports BLOCKED on Task 8 because the write helpers aren't shaped how this plan assumes, the controller dispatches a discovery sub-task.
|
||||
|
||||
|
||||
@@ -2,12 +2,12 @@
|
||||
|
||||
**Goal:** Port the plugin's `podcastfeed` package as a first-party scheduled task in silo, so RSS-subscribed podcasts get their episode lists kept up to date.
|
||||
|
||||
**Source plugin:** `/opt/silo_plugins/silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
**Source plugin:** `../silo-plugin-audiobooks/internal/podcastfeed/refresher.go` (353 LOC) + `refresher_test.go` (295 LOC)
|
||||
|
||||
## Tasks
|
||||
|
||||
### Task 1: Port the refresher package
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `/opt/silo-server/internal/audiobooks/podcastfeed/`
|
||||
- Copy `refresher.go` + `refresher_test.go` from plugin to `internal/audiobooks/podcastfeed/`
|
||||
- Update imports (drop plugin SDK; use silo's `pgxpool` + `log/slog`)
|
||||
- Rewrite SQL queries against silo's `podcast_feeds` table (created in sub-plan 1 migration 140), `media_items` (type='podcast'), and `episodes`
|
||||
- Drop the plugin's `presentation_libraries` reference; subscriptions are tracked by `podcast_feeds.media_item_id`
|
||||
|
||||
@@ -979,7 +979,7 @@ Expected: `healthy`.
|
||||
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT version, filename FROM schema_versions WHERE version=174;"
|
||||
"SELECT version_id FROM public.goose_db_version WHERE version_id=174 AND is_applied;"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name='autoscan_sources' AND column_name='label';"
|
||||
```
|
||||
|
||||
@@ -0,0 +1,617 @@
|
||||
# TheIntroDB read-path correctness (Phase 1 / Layer A) — implementation plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
|
||||
|
||||
**Goal:** Fix the three read-path correctness gaps in the TheIntroDB provider so it matches the
|
||||
real v3 API contract: (1) honor **TVDB** ids in lookups (today they are silently dropped),
|
||||
(2) capture and use the **real per-segment `confidence`** instead of a hardcoded `0.9`, and
|
||||
(3) when the API returns multiple candidate segments, **pick the most-submitted / highest-confidence**
|
||||
one instead of the first usable. This is Phase 1 of
|
||||
[the marker sources & contribution design](../specs/2026-06-06-marker-sources-and-contribution-design.md);
|
||||
it stands alone, ships value immediately (anime / TheTVDB-first libraries start getting markers),
|
||||
and unblocks the "query-all / best-wins" merge in later phases.
|
||||
|
||||
**Architecture:** All changes are contained within `internal/markers/introdb` plus one added
|
||||
field on `markers.Marker`. The resolver already populates the TVDB id into
|
||||
`Request.ExternalIDs` — only the provider and HTTP client drop it. The HTTP `Client` gains a
|
||||
`tvdbID` argument on `FetchEpisode`/`FetchMovie` (query preference `tmdb → tvdb → imdb`, cache
|
||||
keys extended); the provider reads `ExternalIDKeyTVDB`; `segmentTimestamps` gains `confidence`
|
||||
and `submission_count`; `pickMarker` uses real confidence (with a named default) and ranks
|
||||
candidates. The write path (`BuildUpdatePayload` / `UpsertMarkers`) is **not** touched here.
|
||||
|
||||
**Out of scope (deferred to Phase 2):** threading *per-segment* confidence through
|
||||
`MarkerUpdatePayload` and `UpsertMarkers`. That refactor only becomes observable once
|
||||
`Registry.FetchMerged` can combine segments from different providers; with a single provider
|
||||
today the existing "max confidence across segments" collapse is equivalent, and changing it now
|
||||
would break `TestBuildUpdatePayloadAggregatesConfidence` for no functional gain. It lands with
|
||||
the merge dispatch in Phase 2.
|
||||
|
||||
**Tech Stack:** Go (the `internal/markers/introdb` package is pure Go — no libvips/CGO),
|
||||
PostgreSQL is not involved (no migration), no frontend changes. Tests run in a throwaway Go
|
||||
container (the host has no Go toolchain); a named volume caches module downloads.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
`internal/markers` and `internal/markers/introdb` are pure Go, so no libvips is needed:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
To run a single test, append `-run TestName` and narrow the package path, e.g.
|
||||
`go test ./internal/markers/introdb/ -run TestFetchEpisodeSendsTVDBWhenNoTMDB -v`.
|
||||
|
||||
---
|
||||
|
||||
## File structure
|
||||
|
||||
- `internal/markers/introdb/types.go` — **modify**. Add `Confidence` + `SubmissionCount` to `segmentTimestamps`; add the `defaultConfidence` const.
|
||||
- `internal/markers/types.go` — **modify**. Add `SubmissionCount` to `markers.Marker`.
|
||||
- `internal/markers/introdb/client.go` — **modify**. `tvdbID` arg on `FetchEpisode`/`FetchMovie`; query preference `tmdb → tvdb → imdb`; cache keys include tvdb.
|
||||
- `internal/markers/introdb/provider.go` — **modify**. Read `ExternalIDKeyTVDB` and pass it through; rewrite `pickMarker` for real confidence + best-candidate.
|
||||
- `internal/markers/introdb/client_test.go` — **create**. httptest-based client tests (TVDB query, TMDB preference, caching).
|
||||
- `internal/markers/introdb/provider_test.go` — **create**. End-to-end provider tests (TVDB-only resolves, real/default confidence, best-candidate).
|
||||
|
||||
No migration, no wiring change (`cmd/silo/main.go` constructs the provider but never calls the
|
||||
client directly), no frontend change.
|
||||
|
||||
---
|
||||
|
||||
## Task 1: Type plumbing — capture confidence + submission_count
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/types.go`
|
||||
- Modify: `internal/markers/types.go`
|
||||
|
||||
This task is pure struct plumbing (no behavior yet); later tasks' tests exercise it. Verify by
|
||||
compilation.
|
||||
|
||||
- [ ] **Step 1: Add the response fields + default constant in `introdb/types.go`**
|
||||
|
||||
Replace the `Algorithm` const block to add `defaultConfidence` after it:
|
||||
|
||||
```go
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// defaultConfidence is applied when TheIntroDB omits a per-segment confidence
|
||||
// in the /media response. Real per-segment confidence is preferred when present.
|
||||
const defaultConfidence = 0.9
|
||||
```
|
||||
|
||||
Replace the `segmentTimestamps` struct with the version that decodes the two extra fields the
|
||||
v3 `/media` response carries:
|
||||
|
||||
```go
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end). Confidence and SubmissionCount are optional per-segment
|
||||
// quality signals used to pick among multiple candidates.
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
Confidence *float64 `json:"confidence,omitempty"`
|
||||
SubmissionCount *int `json:"submission_count,omitempty"`
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Add `SubmissionCount` to `markers.Marker`**
|
||||
|
||||
In `internal/markers/types.go`, replace the `Marker` struct:
|
||||
|
||||
```go
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Verify the packages still compile**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./internal/markers/...
|
||||
```
|
||||
|
||||
Expected: no output, exit 0. (Existing `write_test.go` / `types_test.go` use field-keyed
|
||||
literals, so the added field does not break them.)
|
||||
|
||||
- [ ] **Step 4: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/types.go internal/markers/types.go
|
||||
git commit -m "feat(markers): decode introdb per-segment confidence + submission_count"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 2: TVDB lookups (gap #1)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/client.go`
|
||||
- Modify: `internal/markers/introdb/provider.go`
|
||||
- Create: `internal/markers/introdb/client_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing client tests**
|
||||
|
||||
Create `internal/markers/introdb/client_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFetchEpisodeSendsTVDBWhenNoTMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "55555", "tt1234567", 2, 3, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "55555" {
|
||||
t.Errorf("tvdb_id = %q, want 55555", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "" {
|
||||
t.Errorf("tmdb_id = %q, want empty", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("imdb_id should be omitted when tvdb present, got %q", gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodePrefersTMDBOverTVDBAndIMDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchEpisode(context.Background(), "111", "222", "tt333", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tmdb_id") != "111" {
|
||||
t.Errorf("tmdb_id = %q, want 111", gotQuery.Get("tmdb_id"))
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "" || gotQuery.Get("imdb_id") != "" {
|
||||
t.Errorf("only tmdb_id expected, got tvdb=%q imdb=%q", gotQuery.Get("tvdb_id"), gotQuery.Get("imdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMovieSendsTVDB(t *testing.T) {
|
||||
var gotQuery url.Values
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQuery = r.URL.Query()
|
||||
_, _ = w.Write([]byte(`{"type":"movie"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
if _, err := c.FetchMovie(context.Background(), "", "888", "", 0); err != nil {
|
||||
t.Fatalf("FetchMovie: %v", err)
|
||||
}
|
||||
if gotQuery.Get("tvdb_id") != "888" {
|
||||
t.Errorf("tvdb_id = %q, want 888", gotQuery.Get("tvdb_id"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchEpisodeCachesByID(t *testing.T) {
|
||||
var hits int32
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
atomic.AddInt32(&hits, 1)
|
||||
_, _ = w.Write([]byte(`{"type":"episode"}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
for i := 0; i < 3; i++ {
|
||||
if _, err := c.FetchEpisode(context.Background(), "", "999", "", 1, 1, 0); err != nil {
|
||||
t.Fatalf("FetchEpisode: %v", err)
|
||||
}
|
||||
}
|
||||
if hits != 1 {
|
||||
t.Fatalf("server hits = %d, want 1 (cached after first)", hits)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: FAIL — compile error (`FetchEpisode`/`FetchMovie` still take the old 6/4-arg
|
||||
signatures without `tvdbID`).
|
||||
|
||||
- [ ] **Step 3: Add `tvdbID` to `FetchEpisode` in `client.go`**
|
||||
|
||||
Replace the whole `FetchEpisode` method:
|
||||
|
||||
```go
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, tvdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Add `tvdbID` to `FetchMovie` in `client.go`**
|
||||
|
||||
Replace the whole `FetchMovie` method:
|
||||
|
||||
```go
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID, tvdbID, or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, tvdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id, tvdb_id, or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
case tvdbID != "":
|
||||
q.Set("tvdb_id", tvdbID)
|
||||
default:
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, tvdbID, imdbID, durationMS))
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 5: Extend the cache-key helpers in `client.go`**
|
||||
|
||||
Replace both `cacheKeyEpisode` and `cacheKeyMovie`:
|
||||
|
||||
```go
|
||||
func cacheKeyEpisode(tmdbID, tvdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:%s:s%de%d:d%d", tvdbID, season, episode, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, tvdbID, imdbID string, durationMS int64) string {
|
||||
switch {
|
||||
case tmdbID != "":
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
case tvdbID != "":
|
||||
return fmt.Sprintf("tvdb:movie:%s:d%d", tvdbID, durationMS)
|
||||
default:
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 6: Read TVDB in the provider and pass it through (`provider.go`)**
|
||||
|
||||
Replace the id-extraction block in `FetchMarkers` (the `tmdbID`/`imdbID` lines and the empty
|
||||
guard):
|
||||
|
||||
```go
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
tvdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTVDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && tvdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
```
|
||||
|
||||
Then update the two client calls in the `switch req.Kind` block to pass `tvdbID`:
|
||||
|
||||
```go
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, tvdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
```
|
||||
```go
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, tvdbID, imdbID, durationMS)
|
||||
```
|
||||
|
||||
- [ ] **Step 7: Run the client tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run 'TestFetch' -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestFetch*` tests).
|
||||
|
||||
- [ ] **Step 8: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/client.go internal/markers/introdb/provider.go internal/markers/introdb/client_test.go
|
||||
git commit -m "fix(markers): honor TVDB ids in TheIntroDB lookups"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 3: Real confidence + best-candidate selection (gaps #2, #3)
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/introdb/provider.go` (`pickMarker`)
|
||||
- Create: `internal/markers/introdb/provider_test.go`
|
||||
|
||||
- [ ] **Step 1: Write the failing provider tests**
|
||||
|
||||
Create `internal/markers/introdb/provider_test.go`:
|
||||
|
||||
```go
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
func newProvider(t *testing.T, body string) *Provider {
|
||||
t.Helper()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = w.Write([]byte(body))
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c := NewClient("")
|
||||
c.SetBaseURL(srv.URL)
|
||||
return NewProvider(c)
|
||||
}
|
||||
|
||||
func episodeReq(ids map[string]string) markers.Request {
|
||||
return markers.Request{
|
||||
Kind: markers.ItemKindEpisode,
|
||||
ExternalIDs: ids,
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Duration: 30 * time.Minute,
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderResolvesTVDBOnly(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, err := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTVDB: "777"}))
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if len(res.Markers) != 1 || res.Markers[0].Kind != markers.MarkerKindIntro {
|
||||
t.Fatalf("expected one intro marker, got %+v", res.Markers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderUsesRealConfidence(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000,"confidence":0.42}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if res.Markers[0].Confidence != 0.42 {
|
||||
t.Errorf("confidence = %v, want 0.42 (real value, not hardcoded)", res.Markers[0].Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderDefaultsConfidenceWhenAbsent(t *testing.T) {
|
||||
p := newProvider(t, `{"type":"episode","intro":[{"end_ms":60000}]}`)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if res.Markers[0].Confidence != defaultConfidence {
|
||||
t.Errorf("confidence = %v, want default %v", res.Markers[0].Confidence, defaultConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderPicksMostSubmittedCandidate(t *testing.T) {
|
||||
body := `{"type":"episode","intro":[
|
||||
{"end_ms":50000,"confidence":0.6,"submission_count":2},
|
||||
{"end_ms":61000,"confidence":0.5,"submission_count":9}
|
||||
]}`
|
||||
p := newProvider(t, body)
|
||||
res, _ := p.FetchMarkers(context.Background(), episodeReq(map[string]string{markers.ExternalIDKeyTMDB: "1"}))
|
||||
if len(res.Markers) != 1 {
|
||||
t.Fatalf("want 1 marker, got %d", len(res.Markers))
|
||||
}
|
||||
if got := res.Markers[0].End; got != 61*time.Second {
|
||||
t.Errorf("picked end = %v, want 61s (the submission_count=9 candidate)", got)
|
||||
}
|
||||
if res.Markers[0].SubmissionCount != 9 {
|
||||
t.Errorf("submission_count = %d, want 9", res.Markers[0].SubmissionCount)
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Run the tests to verify they fail**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: FAIL — `TestProviderUsesRealConfidence` sees `0.9` (hardcoded), `TestProviderPicksMostSubmittedCandidate`
|
||||
gets the first usable (`end=50s`, `SubmissionCount=0`). (`TestProviderResolvesTVDBOnly` already
|
||||
passes from Task 2 — it guards against regression.)
|
||||
|
||||
- [ ] **Step 3: Rewrite `pickMarker` in `provider.go`**
|
||||
|
||||
Replace the entire `pickMarker` function:
|
||||
|
||||
```go
|
||||
// pickMarker selects the best usable segment from a TheIntroDB response array.
|
||||
// `requireEnd` is true for segments where the end timestamp is the load-bearing
|
||||
// field (intro, recap) — they're allowed to start at 0 if `start_ms` is omitted.
|
||||
// For trailing segments (credits, preview) the start is required but the end
|
||||
// defaults to the file duration. When several candidates are usable (e.g. no
|
||||
// duration match narrowed the set), the most-submitted one wins, with higher
|
||||
// confidence breaking ties; with a single candidate this is the previous
|
||||
// first-usable behavior. Real per-segment confidence is used when present,
|
||||
// falling back to defaultConfidence only when the API omits it.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
best := markers.Marker{}
|
||||
bestSubs := -1
|
||||
found := false
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
confidence := defaultConfidence
|
||||
if s.Confidence != nil {
|
||||
confidence = *s.Confidence
|
||||
}
|
||||
subs := 0
|
||||
if s.SubmissionCount != nil {
|
||||
subs = *s.SubmissionCount
|
||||
}
|
||||
if !found || subs > bestSubs || (subs == bestSubs && confidence > best.Confidence) {
|
||||
best = markers.Marker{Kind: kind, Start: start, End: end, Confidence: confidence, SubmissionCount: subs}
|
||||
bestSubs = subs
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Run the tests to verify they pass**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/introdb/ -run TestProvider -v
|
||||
```
|
||||
|
||||
Expected: PASS (all four `TestProvider*` tests).
|
||||
|
||||
- [ ] **Step 5: Commit**
|
||||
|
||||
```bash
|
||||
git add internal/markers/introdb/provider.go internal/markers/introdb/provider_test.go
|
||||
git commit -m "feat(markers): use real introdb confidence and pick best candidate"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Task 4: Full-package regression + integration build
|
||||
|
||||
**Files:** none (verification only)
|
||||
|
||||
- [ ] **Step 1: Run the entire markers test suite**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
Expected: PASS — the new `introdb` tests plus the existing `write_test.go` /
|
||||
`types_test.go` (unchanged behavior: `BuildUpdatePayload` still aggregates max confidence,
|
||||
`FetchFirstHit` still uses registration order).
|
||||
|
||||
- [ ] **Step 2: Build the binary to confirm nothing downstream broke**
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go build ./...
|
||||
```
|
||||
|
||||
Expected: exit 0. (The only callers of the changed client methods are in `provider.go`, updated
|
||||
in Task 2; `cmd/silo/main.go` constructs the provider but never calls the client directly.)
|
||||
|
||||
- [ ] **Step 3: Confirm the working tree is clean**
|
||||
|
||||
```bash
|
||||
git status
|
||||
```
|
||||
|
||||
Expected: clean (all changes committed in Tasks 1–3; this task adds no files).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Gap coverage:** TVDB lookups (Task 2, `TestFetch*` + `TestProviderResolvesTVDBOnly`), real
|
||||
confidence (Task 3, `TestProviderUsesRealConfidence` + `…DefaultsConfidenceWhenAbsent`),
|
||||
best-candidate (Task 3, `TestProviderPicksMostSubmittedCandidate`). Maps to design gaps #1–#3.
|
||||
- **Containment:** the client signature change touches exactly one production caller
|
||||
(`provider.go`); `markers.Marker` is constructed only in `provider.go`. Verified by grep before
|
||||
planning, re-verified by the Task 4 `go build ./...`.
|
||||
- **No behavior regressions:** the write path is untouched, so `BuildUpdatePayload`'s
|
||||
max-confidence aggregation and its test remain valid; per-segment confidence is deferred to
|
||||
Phase 2 where the merge actually consumes it.
|
||||
- **No placeholders:** every code step shows the complete replacement; every run step shows the
|
||||
command and the expected result. No migration, no frontend, no wiring changes.
|
||||
- **Default confidence:** `defaultConfidence = 0.9` preserves today's effective value for
|
||||
responses that omit confidence, so existing stored markers keep equivalent provenance.
|
||||
@@ -0,0 +1,482 @@
|
||||
# Marker sources & TheIntroDB contribution — implementation plan (all phases)
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. Implement phases in order; each phase is independently shippable.
|
||||
|
||||
**Goal:** Implement the design in
|
||||
[2026-06-06-marker-sources-and-contribution-design.md](../specs/2026-06-06-marker-sources-and-contribution-design.md):
|
||||
fix the TheIntroDB read-path correctness gaps, formalize a multi-source provider model
|
||||
(query-all / best-wins), and add the ability to contribute Silo's own markers back to
|
||||
TheIntroDB — configured **per provider, off by default**. Server-side API + data model only;
|
||||
web UI is a separate follow-up.
|
||||
|
||||
**Architecture:** Five sequential phases. Phase 1 fixes the existing provider (TVDB, real
|
||||
confidence, best-candidate). Phase 2 adds a per-provider config table, per-segment marker
|
||||
provenance, and `Registry.FetchMerged`. Phase 3 adds the submission client, the contribution
|
||||
audit table, and the `ContributionService` engine. Phase 4 exposes the full admin API (manual
|
||||
markers, provider config, contribute, history). Phase 5 adds the daily auto-contribution task.
|
||||
|
||||
**Tech Stack:** Go (chi handlers, pgx repositories), PostgreSQL (paired numbered migrations),
|
||||
no frontend in this plan. Most touched packages (`internal/markers`, `internal/markers/introdb`,
|
||||
`internal/taskmanager/tasks`) are pure Go; `internal/api/handlers` needs libvips (CGO image
|
||||
deps). Tests run in a throwaway Go container (host has no Go toolchain).
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
---
|
||||
|
||||
## Phase overview & dependencies
|
||||
|
||||
| Phase | Delivers | Depends on | New migrations |
|
||||
|------|----------|-----------|----------------|
|
||||
| 1 | TheIntroDB read-path correctness (TVDB, real confidence, best-candidate) | — | none |
|
||||
| 2 | `marker_provider_config` + per-segment provenance + `FetchMerged` | 1 (real confidence) | `marker_provider_config` |
|
||||
| 3 | Submission client + `marker_contributions` + `ContributionService` | 2 (`Submitter`, provider config) | `marker_contributions` |
|
||||
| 4 | Admin API: manual markers, provider config, contribute, history | 3 (service) | none |
|
||||
| 5 | Daily auto-contribution task | 3 (service), 4 (optional) | none |
|
||||
|
||||
Phase 1's line-level steps are also captured standalone in
|
||||
[2026-06-06-marker-introdb-readpath-correctness.md](2026-06-06-marker-introdb-readpath-correctness.md);
|
||||
they are summarized here so this document is self-contained.
|
||||
|
||||
---
|
||||
|
||||
## Running tests (host has no Go toolchain)
|
||||
|
||||
Pure-Go packages (`internal/markers`, `internal/markers/introdb`, `internal/taskmanager/...`):
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
go test ./internal/markers/... -v
|
||||
```
|
||||
|
||||
`internal/api/handlers` (Phase 4) needs libvips:
|
||||
|
||||
```bash
|
||||
docker run --rm -v "$PWD":/app -w /app -v silo-gomod:/go/pkg/mod \
|
||||
-e GOFLAGS=-mod=mod golang:1.26 \
|
||||
sh -c 'apt-get update >/dev/null && apt-get install -y --no-install-recommends libvips-dev >/dev/null && go test ./internal/api/handlers/ -run TestAdminMarkers -v'
|
||||
```
|
||||
|
||||
Whole-project compile: `go build ./...` (same container).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — TheIntroDB read-path correctness
|
||||
|
||||
**Goal:** Honor TVDB ids, capture/use real per-segment confidence + submission_count, and pick
|
||||
the best candidate among multiple. Contained to `internal/markers/introdb` plus one field on
|
||||
`markers.Marker`. No migration, no write-path change.
|
||||
|
||||
**Files:** `internal/markers/introdb/{types.go,client.go,provider.go}` (modify),
|
||||
`internal/markers/types.go` (modify, add `Marker.SubmissionCount`),
|
||||
`internal/markers/introdb/{client_test.go,provider_test.go}` (create).
|
||||
|
||||
### Task 1.1 — Type plumbing
|
||||
- [ ] Add `Confidence *float64` and `SubmissionCount *int` to `segmentTimestamps`
|
||||
(`introdb/types.go`); add `const defaultConfidence = 0.9`.
|
||||
- [ ] Add `SubmissionCount int` to `markers.Marker` (`markers/types.go`).
|
||||
- [ ] `go build ./internal/markers/...` → exit 0. Commit.
|
||||
|
||||
### Task 1.2 — TVDB lookups (gap #1)
|
||||
- [ ] Create `introdb/client_test.go` (httptest): asserts `tvdb_id` sent when only tvdb given,
|
||||
tmdb preferred over tvdb/imdb, movie tvdb, and cache collapses repeat calls. Run → FAIL (compile).
|
||||
- [ ] `client.go`: `FetchEpisode`/`FetchMovie` gain a `tvdbID` arg; query preference
|
||||
`tmdb → tvdb → imdb` via a `switch`; the empty-id guard includes tvdb; `cacheKeyEpisode`/
|
||||
`cacheKeyMovie` include a `tvdb:` branch.
|
||||
- [ ] `provider.go` `FetchMarkers`: read `req.ExternalIDs[markers.ExternalIDKeyTVDB]`, include it
|
||||
in the empty guard, pass it to both client calls.
|
||||
- [ ] Run client tests → PASS. Commit.
|
||||
|
||||
### Task 1.3 — Real confidence + best-candidate (gaps #2, #3)
|
||||
- [ ] Create `introdb/provider_test.go`: TVDB-only resolves; real confidence used; default when
|
||||
absent; most-submitted candidate wins (ties broken by confidence). Run → FAIL.
|
||||
- [ ] Rewrite `pickMarker` to use real confidence (fallback `defaultConfidence`), set
|
||||
`SubmissionCount`, and select the candidate with the highest `(submission_count, confidence)`.
|
||||
- [ ] Run → PASS. Commit.
|
||||
|
||||
### Task 1.4 — Regression
|
||||
- [ ] `go test ./internal/markers/... && go build ./...` → all green (existing `write_test.go`/
|
||||
`types_test.go` unaffected; write path untouched). Commit if needed.
|
||||
|
||||
(Full code for every step is in the standalone Phase 1 plan linked above.)
|
||||
|
||||
---
|
||||
|
||||
# Phase 2 — Per-provider config, per-segment provenance, FetchMerged
|
||||
|
||||
**Goal:** Add the `marker_provider_config` table that drives which providers are queried (and,
|
||||
later, submitted to); make the marker write path carry **per-segment** provider/confidence/
|
||||
algorithm so a merged multi-provider result writes correct provenance; add
|
||||
`Registry.FetchMerged` (query all enabled providers, keep the best candidate per segment) and
|
||||
switch the lazy-playback path to it. With only TheIntroDB enabled, behavior is unchanged.
|
||||
|
||||
**Files:**
|
||||
- Create: `migrations/<next>_marker_provider_config.{up,down}.sql`
|
||||
- Create: `internal/markers/provider_config.go`, `internal/markers/provider_config_test.go`
|
||||
- Modify: `internal/markers/types.go` (`Marker.ProviderID`/`Algorithm`; `Registry.FetchMerged`)
|
||||
- Modify: `internal/markers/write.go` (per-segment provenance in `MarkerUpdatePayload`/`BuildUpdatePayload`)
|
||||
- Modify: `internal/scanner/file_repo.go` (`MarkerUpdate` + `applySegmentPatch` per-segment provenance)
|
||||
- Modify: the `markers.MarkerUpdatePayload` → `scanner.MarkerUpdate` conversion (in the lazy-markers handler/adapter)
|
||||
- Modify: `internal/api/handlers/playback_lazy_markers.go` (call `FetchMerged`)
|
||||
- Modify: `cmd/silo/main.go` (construct the config store; pass to the registry)
|
||||
|
||||
### Task 2.1 — Migration: `marker_provider_config`
|
||||
- [ ] Write `migrations/<next>_marker_provider_config.up.sql` (use the next free number; latest is 180):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY,
|
||||
fetch_enabled boolean NOT NULL DEFAULT true,
|
||||
fetch_priority integer NOT NULL DEFAULT 100,
|
||||
contribute_enabled boolean NOT NULL DEFAULT false,
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false,
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: `DROP TABLE public.marker_provider_config;`
|
||||
- [ ] Commit.
|
||||
|
||||
### Task 2.2 — Provider config store
|
||||
- [ ] Create `internal/markers/provider_config.go`:
|
||||
|
||||
```go
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
// ProviderConfigStore is a cached read/write façade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; Update writes through and refreshes.
|
||||
type ProviderConfigStore struct { /* pool, mu, cache map[string]ProviderConfig */ }
|
||||
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error // load all rows into cache
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig // snapshot copy
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool)
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, p ProviderConfig) error // upsert + refresh
|
||||
// EnabledForFetch returns fetch_enabled providers sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig
|
||||
```
|
||||
|
||||
- [ ] Hot-reload: subscribe the store's `Reload` to the existing `cache.EventSettingsChanged`
|
||||
Redis channel (mirror the `introdb.api_key` reload wiring in `cmd/silo/main.go`), or add a
|
||||
dedicated `marker_provider_config_changed` event published by `Update`. Either is acceptable;
|
||||
document which.
|
||||
- [ ] `provider_config_test.go`: with a real/pgxmock pool or an injected fake, assert
|
||||
`EnabledForFetch` filters disabled rows and sorts by priority. Commit.
|
||||
|
||||
### Task 2.3 — Per-segment marker provenance (write path)
|
||||
|
||||
This makes a merged result (intro from provider A, credits from provider B) persist the correct
|
||||
per-segment `*_markers_provider` / `*_markers_confidence` / `*_markers_algorithm`. The DB columns
|
||||
already exist; today `applySegmentPatch` applies one shared triple.
|
||||
|
||||
- [ ] Add to `markers.Marker`: `ProviderID string`, `Algorithm string` (so each segment carries
|
||||
its origin). `provider.go` (Phase 1) sets these from `ProviderID`/`Algorithm` per marker.
|
||||
- [ ] Rework `markers.MarkerUpdatePayload` (`write.go`) from one shared `Confidence`/`Provider`/
|
||||
`Algorithm` to a per-segment struct, e.g.:
|
||||
|
||||
```go
|
||||
type SegmentPayload struct { Start, End *float64; Provider *string; Confidence *float64; Algorithm string }
|
||||
type MarkerUpdatePayload struct {
|
||||
Intro, Credits, Recap, Preview SegmentPayload
|
||||
Source string // shared source class (e.g. "online"); per-kind provider differs
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Update `BuildUpdatePayload` to fill each `SegmentPayload` from the matching `Marker`
|
||||
(provider/confidence/algorithm per segment). Update `write_test.go`
|
||||
(`TestBuildUpdatePayloadAggregatesConfidence` → assert per-segment confidence: intro 0.7,
|
||||
credits 0.9, recap 0.5).
|
||||
- [ ] Read `applySegmentPatch` and the `MarkerUpdatePayload → scanner.MarkerUpdate` conversion
|
||||
first, then extend `scanner.MarkerUpdate` + `applySegmentPatch` so each segment's
|
||||
provider/confidence/algorithm come from that segment (not a shared field). `nextSharedMarkerAttribution`
|
||||
keeps populating the legacy shared `markers_source`/`markers_confidence` columns from the
|
||||
highest-confidence applied segment.
|
||||
- [ ] `go test ./internal/markers/... ./internal/scanner/...` → green. Commit.
|
||||
|
||||
### Task 2.4 — `Registry.FetchMerged`
|
||||
- [ ] Give `Registry` access to the enabled set (constructor takes a `ProviderConfigStore`, or
|
||||
`FetchMerged` takes an `[]ProviderConfig` snapshot). Add:
|
||||
|
||||
```go
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the candidate with the highest (SubmissionCount, Confidence),
|
||||
// breaking ties by the provider's fetch_priority. Per-segment ProviderID/
|
||||
// Algorithm are preserved so the write path records correct provenance.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error)
|
||||
```
|
||||
|
||||
Algorithm: fan out via goroutines (bounded) to enabled providers; collect `(ProviderConfig,
|
||||
Result, error)`; for each `MarkerKind`, choose the best `Marker` across results; assemble a
|
||||
`Result` whose `Markers` carry their winning provider's `ProviderID`/`Algorithm`. Log and skip
|
||||
provider errors (like `FetchFirstHit`). Return `ok=false` if no segment found.
|
||||
|
||||
- [ ] `types_test.go`: add a merge test — two fake providers, one wins intro, the other wins
|
||||
credits by submission_count; assert the merged result has both with correct per-segment
|
||||
ProviderID. Keep the existing `FetchFirstHit` tests. Commit.
|
||||
|
||||
### Task 2.5 — Switch lazy path + wire store
|
||||
- [ ] In `internal/api/handlers/playback_lazy_markers.go`, replace the `FetchFirstHit` call in
|
||||
`fetchOnlineMarkersForPlayback` with `FetchMerged`.
|
||||
- [ ] In `cmd/silo/main.go`, construct `NewProviderConfigStore`, `Reload` it at startup, wire its
|
||||
hot-reload, and pass it to the registry. (TheIntroDB stays registered as today.)
|
||||
- [ ] `go build ./... && go test ./internal/markers/...` → green. With only `introdb` enabled,
|
||||
`FetchMerged` returns the same result `FetchFirstHit` did — verify a provider test asserting
|
||||
single-provider parity. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 3 — Submission client, contribution tracking, service engine
|
||||
|
||||
**Goal:** Build the machinery to submit a marker to a provider, idempotently and audited —
|
||||
without yet wiring a trigger. Adds the `Submitter` capability, the introdb submission/stats
|
||||
client methods, the `marker_contributions` table, and the `ContributionService`.
|
||||
|
||||
**Files:**
|
||||
- Modify: `internal/markers/types.go` (`Submitter`, `SubmissionRequest`/`SubmissionResult`/`UserStats`)
|
||||
- Modify: `internal/markers/introdb/{types.go,client.go,provider.go}` (submit + stats; implement `Submitter`)
|
||||
- Create: `migrations/<next>_marker_contributions.{up,down}.sql`
|
||||
- Create: `internal/markers/contribution_repo.go`, `internal/markers/contribute.go` (+ tests)
|
||||
- Modify: `cmd/silo/main.go` (construct the service)
|
||||
|
||||
### Task 3.1 — `Submitter` capability + DTOs
|
||||
- [ ] Add to `internal/markers/types.go`:
|
||||
|
||||
```go
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string // tmdb/imdb (tvdb not accepted by /submit; tmdb required)
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start, End *time.Duration // nil start ok for intro/recap; nil end ok for credits/preview
|
||||
Duration time.Duration
|
||||
}
|
||||
type SubmissionResult struct { ID string; Status string; Weight float64 } // status: pending|accepted|rejected
|
||||
type UserStats struct { Total, Accepted, Pending, Rejected int; AcceptanceRate float64; CurrentStreak, BestStreak int }
|
||||
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
|
||||
- [ ] Commit (compiles; no implementer yet).
|
||||
|
||||
### Task 3.2 — introdb submission client
|
||||
- [ ] `introdb/types.go`: add the request/response shapes for `POST /v3/submit`
|
||||
(`tmdb_id`,`imdb_id`,`type`,`segment`,`season`,`episode`,`video_duration_ms`,`start_ms`,`end_ms`;
|
||||
response `{submissions:[{id,status,weight,...}]}`) and `GET /v3/user/stats`. Remove the
|
||||
"Submissions are intentionally not supported" package note.
|
||||
- [ ] `introdb/client.go`: add `SubmitSegment(ctx, ...) (*submitResponse, error)` (POST,
|
||||
`Authorization: Bearer` **required** — return an error if `apiKey == ""`; honor
|
||||
`X-UsageLimit-Reset` on 429) and `FetchUserStats(ctx) (*userStats, error)` (GET).
|
||||
- [ ] `introdb/provider.go`: implement `markers.Submitter` — map `SubmissionRequest` to the
|
||||
client call (apply the v3 null rules: intro/recap drop a zero start to `null`; credits/preview
|
||||
send `null` end when absent), translate `SubmissionResult`/`UserStats`.
|
||||
- [ ] `client_test.go`/`provider_test.go`: httptest asserts the submit body shape, bearer header,
|
||||
and that an empty key errors before any HTTP call; stats parse. Commit.
|
||||
|
||||
### Task 3.3 — `marker_contributions` table + repo
|
||||
- [ ] Migration `<next>_marker_contributions.up.sql` (per design §C4):
|
||||
|
||||
```sql
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL,
|
||||
segment_kind text NOT NULL,
|
||||
source text NOT NULL,
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL,
|
||||
submission_id uuid,
|
||||
status text NOT NULL,
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
- [ ] `.down.sql`: drop the table.
|
||||
- [ ] `internal/markers/contribution_repo.go`: `ContributionStore` with `AlreadySubmitted(ctx,
|
||||
fileID, provider, kind, contentHash) (bool, error)`, `Record(ctx, ContributionRow) error`
|
||||
(upsert on the unique key), `ListByFile(ctx, fileID) ([]ContributionRow, error)`. Add a
|
||||
`ContentHash(kind string, startMs, endMs, durationMs *int64) string` helper (stable SHA-256).
|
||||
- [ ] Repo test (value-hash idempotency, list). Commit.
|
||||
|
||||
### Task 3.4 — `ContributionService`
|
||||
- [ ] `internal/markers/contribute.go`:
|
||||
|
||||
```go
|
||||
type ContributionService struct { /* registry, resolver, cfg *ProviderConfigStore, store *ContributionStore, log */ }
|
||||
|
||||
// ContributeFile submits the file's eligible markers to enabled submitter providers.
|
||||
// opts may scope to one provider and/or specific segment kinds. Auto callers pass
|
||||
// requireAutoLocal=true so only providers with contribute_auto_local participate.
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error)
|
||||
```
|
||||
|
||||
Per provider (from `cfg.List()` filtered to `Submitter` + `contribute_enabled`, and
|
||||
`contribute_auto_local` when `opts.Auto`):
|
||||
1. **Eligibility (C2):** for each requested segment present on the file, require
|
||||
`source ∈ {scanner, manual}` (skip `online` — circular); resolve external IDs via the existing
|
||||
resolver (skip if none / tmdb required by `/submit`); for auto require `source=scanner`,
|
||||
`kind=intro`, `confidence ≥ contribute_min_confidence`.
|
||||
2. **Idempotency:** compute `ContentHash`; skip if `AlreadySubmitted`.
|
||||
3. **Submit:** `provider.(Submitter).SubmitMarker(...)`.
|
||||
4. **Persist:** `store.Record(...)` with returned status/submission_id (or `status="error"`).
|
||||
|
||||
- [ ] `contribute_test.go` with a fake `Submitter` + in-memory store: asserts online-sourced
|
||||
markers are skipped, sub-threshold auto markers skipped, duplicates skipped, a fresh manual
|
||||
marker submitted and recorded. Commit.
|
||||
|
||||
### Task 3.5 — Wire
|
||||
- [ ] `cmd/silo/main.go`: construct `ContributionService` (registry, resolver, config store,
|
||||
contribution store). Not yet called by any handler/task. `go build ./...` + tests green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 4 — Admin API (manual markers, provider config, contribute, history)
|
||||
|
||||
**Goal:** Expose the full backend contract under `RequireAdmin`. Mirror the structure of
|
||||
`internal/api/handlers/admin_intro.go` (interface-typed deps, `chi.URLParam`, `writeError`/
|
||||
`writeJSON`, `MarkerUpdateNotifier` after writes).
|
||||
|
||||
**Files:**
|
||||
- Create: `internal/api/handlers/admin_markers.go` (+ test), `internal/api/handlers/admin_marker_providers.go` (+ test)
|
||||
- Modify: `internal/api/router.go` (register routes in the `RequireAdmin` group)
|
||||
- Modify: `cmd/silo/main.go` (construct + inject the handlers)
|
||||
|
||||
### Task 4.1 — Manual marker endpoints (design §C3)
|
||||
- [ ] `admin_markers.go`:
|
||||
- `GET /admin/files/{fileId}/markers` → current per-segment values + provenance
|
||||
(`source`,`provider`,`confidence`,`algorithm`,`detected_at`) for intro/recap/credits/preview.
|
||||
- `PUT /admin/files/{fileId}/markers` → body per segment `{start,end}` (seconds) or `null` to
|
||||
clear; write `source = "manual"` via `scanner.FileRepository.UpsertMarkers` (priority 4);
|
||||
then `MarkerUpdateNotifier.MarkersUpdated(file)`. Validation: `end>start`; intro/recap may
|
||||
omit start; credits/preview may omit end; within `[0, duration]`.
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` → clear one segment.
|
||||
- `GET|PUT /admin/items/{id}/markers` → resolve to the item's primary file (convenience).
|
||||
- [ ] Handler test (libvips container): PUT writes manual markers and returns provenance;
|
||||
invalid range → 400; clear nulls the segment. Commit.
|
||||
|
||||
### Task 4.2 — Provider config + validate endpoints (design §C7)
|
||||
- [ ] `admin_marker_providers.go`:
|
||||
- `GET /admin/markers/providers` → for each registered provider: id, `isSubmitter`, its
|
||||
`marker_provider_config` row, and (best-effort cached) `UserStats` for submitters with a key.
|
||||
- `PUT /admin/markers/providers/{provider}` → update the config row (validated; unknown → 404).
|
||||
- `POST /admin/markers/providers/{provider}/validate` → `Submitter.FetchUserStats`; non-submitter → 400.
|
||||
- [ ] Test: list returns introdb with its config; update flips `contribute_enabled`; validate on
|
||||
a non-submitter → 400. Commit.
|
||||
|
||||
### Task 4.3 — Contribution endpoints (design §C7)
|
||||
- [ ] In `admin_markers.go` (or a small `admin_contributions.go`):
|
||||
- `POST /admin/files/{fileId}/contribute` (optional body `{provider?, segments?[]}`) →
|
||||
`ContributionService.ContributeFile(...)`; returns per (provider, segment) outcome.
|
||||
- `GET /admin/files/{fileId}/contributions` → `ContributionStore.ListByFile`.
|
||||
- [ ] Test with a fake service/store. Commit.
|
||||
|
||||
### Task 4.4 — Routing + wiring
|
||||
- [ ] Register all new routes inside the existing `RequireAdmin` group in `internal/api/router.go`
|
||||
(mirror the `redetect-intro` registration). Inject the handlers in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/api/handlers/ -run 'TestAdminMarkers|TestAdminMarkerProviders'`
|
||||
(libvips container) → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
# Phase 5 — Daily auto-contribution task
|
||||
|
||||
**Goal:** A scheduled task that, for each provider with `contribute_enabled &&
|
||||
contribute_auto_local`, submits high-confidence local intro detections that haven't been
|
||||
contributed. Mirror `internal/taskmanager/tasks/detect_intro_markers.go`.
|
||||
|
||||
**Files:** Create `internal/taskmanager/tasks/contribute_markers.go` (+ test); modify
|
||||
`cmd/silo/main.go` (register the task).
|
||||
|
||||
### Task 5.1 — `ContributeMarkersTask`
|
||||
- [ ] Implement the `taskmanager.Task` interface (`Key="contribute_markers"`, `Name`,
|
||||
`Description`, `Category=TaskCategoryLibrary`, `IsHidden() bool`, `DefaultTriggers()` daily e.g.
|
||||
`"04:00"` — after the 03:30 detection task, `Execute`).
|
||||
- [ ] `Execute`: short-circuit if no provider has `contribute_enabled && contribute_auto_local`
|
||||
(read `ProviderConfigStore`). Otherwise page through episode files with
|
||||
`intro_markers_source='scanner'` and `intro_markers_confidence ≥` the provider's
|
||||
`contribute_min_confidence` (a new `ContributionStore`/repo query, batched), and call
|
||||
`ContributionService.ContributeFile(file, ContributeOptions{Auto:true})`. Report progress; obey
|
||||
the introdb usage limit (the client already backs off on `X-UsageLimit-Reset`); idempotency
|
||||
makes interrupted runs resumable.
|
||||
- [ ] Test with fakes: enabled+auto submits eligible files; disabled → no-op; sub-threshold
|
||||
skipped. Commit.
|
||||
|
||||
### Task 5.2 — Register
|
||||
- [ ] Register the task where `DetectIntroMarkersTask` is registered in `cmd/silo/main.go`.
|
||||
- [ ] `go build ./... && go test ./internal/taskmanager/...` → green. Commit.
|
||||
|
||||
---
|
||||
|
||||
## Final integration verification
|
||||
|
||||
**Files:** none (build + runtime).
|
||||
|
||||
- [ ] **Build the image** (compiles backend + frontend):
|
||||
```bash
|
||||
docker build --build-arg BUILD_REVISION=$(git rev-parse --short HEAD) --build-arg BUILD_DIRTY=false -t silo-server:markers-test .
|
||||
```
|
||||
- [ ] **Recreate the container** (applies the new migrations on startup) and wait for health:
|
||||
```bash
|
||||
docker compose up -d silo
|
||||
for i in $(seq 1 24); do s=$(docker inspect -f '{{.State.Health.Status}}' silo-silo-1); echo "$s"; [ "$s" = healthy ] && break; sleep 5; done
|
||||
```
|
||||
- [ ] **Verify migrations applied + tables exist:**
|
||||
```bash
|
||||
docker compose exec -T postgres psql -U silo -d silo -c \
|
||||
"SELECT to_regclass('public.marker_provider_config'), to_regclass('public.marker_contributions');"
|
||||
docker compose exec -T postgres psql -U silo -d silo -c "SELECT * FROM marker_provider_config;"
|
||||
```
|
||||
Expected: both tables present; an `introdb` row with `fetch_enabled=t, contribute_enabled=f`.
|
||||
- [ ] **Smoke the read path (TVDB):** trigger playback (or call the marker fetch) for a
|
||||
TVDB-only-matched episode and confirm markers now populate where they previously did not.
|
||||
- [ ] **Smoke contribution (manual, dry):** enable contribution for introdb against a **test/staging**
|
||||
TheIntroDB key, `PUT` a manual marker on a file, `POST /admin/files/{id}/contribute`, then
|
||||
`GET /admin/files/{id}/contributions` shows a `pending` row; a second contribute is idempotent
|
||||
(no new row). Use a non-production key so test data isn't submitted to the real database.
|
||||
- [ ] `git status` → clean (all changes committed per phase).
|
||||
|
||||
---
|
||||
|
||||
## Self-review notes
|
||||
|
||||
- **Phase independence:** each phase compiles, tests, and ships alone; with only `introdb`
|
||||
enabled and contribution off (defaults), Phases 2–5 are observationally inert until an operator
|
||||
enables a provider — so they can land ahead of any behavior change.
|
||||
- **Per-provider, off by default:** `contribute_enabled`/`contribute_auto_local` default `false`
|
||||
in the migration; the service refuses to submit otherwise; auto additionally gated on
|
||||
`contribute_auto_local`. The `provider` column on `marker_contributions` and the per-provider
|
||||
loop make multi-target submission first-class.
|
||||
- **Circular-contribution guard:** `source='online'` is never contributed (C2 eligibility) — has
|
||||
a dedicated unit test in Task 3.4.
|
||||
- **Idempotency:** value `content_hash` + the unique constraint mean repeat/interrupted runs
|
||||
don't double-submit, while a corrected value (new hash) does submit — matching TheIntroDB's
|
||||
weighted-average model.
|
||||
- **Write-path accuracy:** Task 2.3 requires reading `applySegmentPatch` and the
|
||||
`MarkerUpdatePayload → scanner.MarkerUpdate` conversion before editing; the per-segment columns
|
||||
already exist in `media_files`, so this is a plumbing change, not a schema change.
|
||||
- **Migration numbers:** `marker_provider_config` and `marker_contributions` take the next two
|
||||
free numbers at implementation time (181+; latest on disk is 180). Re-check for collisions
|
||||
before writing.
|
||||
- **Out of scope (tracked in the design doc):** all web UI; per-user TheIntroDB accounts; a
|
||||
`marker_provider.v1` plugin capability. Clean seams left for each.
|
||||
@@ -1,13 +1,14 @@
|
||||
# Audiobooks Absorption — Discovery Findings
|
||||
|
||||
Produced by sub-plan 1, Task 1. Locks data-model and integration
|
||||
decisions for migrations 139–142 and downstream sub-plans.
|
||||
decisions for the audiobook foundation migrations and downstream sub-plans.
|
||||
|
||||
## D1 — Next migration number
|
||||
|
||||
Highest existing migration: 138_search_number_word_normalization.up.sql
|
||||
Next available number: 139
|
||||
Sub-plan 1 will use migrations 139 through 142.
|
||||
Original snapshot: `138_search_number_word_normalization.up.sql` was the
|
||||
highest existing migration. The landed implementation was renumbered to
|
||||
`147_abs_sessions`, `157_podcast_feeds`, `159_media_folders_kind_noop`, and
|
||||
`160_audiobooks_feature_flag`.
|
||||
|
||||
## D2 — `media_libraries` kind/type column
|
||||
|
||||
@@ -16,10 +17,10 @@ The spec refers to `media_libraries` but the actual table is `media_folders`.
|
||||
library content. Current values in production: `movies`, `series`, `mixed`.
|
||||
|
||||
Existing column 'type' (text) discriminates library content on `media_folders`.
|
||||
Task 4 (migration 141) should ADD the value `audiobooks` to the type vocabulary
|
||||
Task 4 (migration 159) should ADD the value `audiobooks` to the type vocabulary
|
||||
rather than add a new column. The audiobook scanner branch will set
|
||||
`media_folders.type = 'audiobooks'` for audiobook libraries. No schema change
|
||||
needed for the column itself; migration 141 becomes a no-op DDL migration that
|
||||
needed for the column itself; migration 159 becomes a no-op DDL migration that
|
||||
documents the new allowed value and adds any supporting indexes if needed.
|
||||
|
||||
No CHECK constraint or enum enforces the `type` column values, so adding
|
||||
|
||||
@@ -41,9 +41,10 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
### Out
|
||||
|
||||
- Audiobook requests flow (`silo-plugin-audiobook-requests` untouched).
|
||||
- Smart collections, share links, content restrictions, custom metadata providers.
|
||||
- Smart collections, share links, content restrictions, metadata-provider integration.
|
||||
- Embedding-powered "similar books" recommender.
|
||||
- External enrichment (OpenLibrary / Google Books) — deferred; file tags only for v1.
|
||||
- External enrichment through the audiobook metadata plugin/provider — deferred;
|
||||
scanner extracts local tags and identifiers for v1.
|
||||
- Group listening / cowatch parity for audiobooks (silo already has cowatch for
|
||||
video; revisit later if needed).
|
||||
|
||||
@@ -53,7 +54,7 @@ Go application. Eliminate it as a plugin. User-visible outcome:
|
||||
|
||||
New top-level Go package: `internal/audiobooks/`.
|
||||
|
||||
```
|
||||
```text
|
||||
internal/audiobooks/
|
||||
abs/ ← ported from plugin's internal/abs/ (~6.8k LOC)
|
||||
ABS-compatible REST handlers, mounted under /abs/* and the
|
||||
@@ -90,7 +91,9 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
- ~7.4k LOC ported from the plugin.
|
||||
- ~1.5–2k LOC new TypeScript/TSX in silo's SPA.
|
||||
- ~400 LOC of new Go glue (scanner branches, router wiring, service.go).
|
||||
- Two new SQL migrations (134, 135) and at most one column-add migration.
|
||||
- Three new SQL migrations in the current stack (`147_abs_sessions`,
|
||||
`157_podcast_feeds`, `160_audiobooks_feature_flag`) plus one documented
|
||||
no-op migration for the already-present media-folder type column.
|
||||
|
||||
## Data model
|
||||
|
||||
@@ -111,9 +114,9 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### New tables (two migrations)
|
||||
|
||||
1. **`abs_sessions`** (migration 134) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
1. **`abs_sessions`** (migration 147) — parallel to existing `jellycompat_sessions`. Tracks ABS client device, token hash, last-seen so ABS apps reconnect without re-auth. Columns: `id`, `user_id`, `token_hash`, `device_id`, `device_name`, `abs_client_version`, `created_at`, `last_seen_at`. Same shape as `jellycompat_sessions` plus the `abs_client_version` text.
|
||||
|
||||
2. **`podcast_feeds`** (migration 135) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
2. **`podcast_feeds`** (migration 157) — one row per subscribed podcast. Columns: `media_item_id` (FK to the podcast `media_items` row), `feed_url`, `etag`, `last_refreshed_at`, `refresh_interval_seconds`. Episode rows live in the reused `episodes` table.
|
||||
|
||||
### Schema-touch column add
|
||||
|
||||
@@ -124,8 +127,8 @@ media-token signing, CDN, streaming, event bus, migrations runner).
|
||||
|
||||
### Plugin migrations NOT ported
|
||||
|
||||
Smart collections, share links, embeddings, content restrictions, custom
|
||||
metadata providers, request provider, file cache, listening stats aggregates,
|
||||
Smart collections, share links, embeddings, content restrictions,
|
||||
metadata-provider integration, request provider, file cache, listening stats aggregates,
|
||||
reading goals, notification prefs, standalone-mode tables, recommender
|
||||
embeddings. All represent features dropped from scope.
|
||||
|
||||
@@ -225,8 +228,9 @@ and directors. No new code, just two more role string constants.
|
||||
|
||||
### External enrichment
|
||||
|
||||
Deferred. The plugin's `enrich/` package (OpenLibrary / Google Books lookup)
|
||||
is not ported in v1. File tags only. Add later if needed.
|
||||
Deferred. The scanner extracts local tags and identity hints only. Canonical
|
||||
enrichment remains the responsibility of the audiobook metadata plugin/provider
|
||||
path and is not ported in this foundation pass.
|
||||
|
||||
### Scheduled task
|
||||
|
||||
|
||||
@@ -0,0 +1,472 @@
|
||||
# Marker sources & TheIntroDB contribution — design
|
||||
|
||||
**Date:** 2026-06-06
|
||||
**Status:** Draft (rev 2), pending review
|
||||
**Scope:** Review the current TheIntroDB marker integration, formalize a multi-source
|
||||
marker-provider model (query-all / best-wins), and add the ability to contribute Silo's own
|
||||
markers back to TheIntroDB. This round designs the **server-side API and data model only** —
|
||||
web-admin and mobile surfaces are deferred to a UI-only follow-up. No client (Android/Apple)
|
||||
changes required.
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
**Rev 2 changes:** contribution is configured **per provider** (off by default); all new web
|
||||
UI is **deferred** and the backend API is specified in full instead.
|
||||
|
||||
## Decisions locked for this design
|
||||
|
||||
These were chosen up front and constrain everything below:
|
||||
|
||||
1. **Deliverable now:** design/plan only. No source changes land from this document; it is
|
||||
the blueprint a follow-up implementation plan expands.
|
||||
2. **What we contribute:** *both* local auto-detected markers *and* manual corrections. Local
|
||||
detection only produces **intro markers for episodes**, so credits/recap/preview and all
|
||||
movies can only be contributed via manual correction — this asymmetry drives the design.
|
||||
3. **Account model:** a single server-wide TheIntroDB account, reusing the existing
|
||||
`introdb.api_key` server setting. Per-user accounts are a noted future seam, not built now.
|
||||
4. **Multi-source dispatch:** query all enabled online sources and keep the highest-quality
|
||||
result per segment ("best wins"). This makes *real per-segment confidence* (gap #2 below)
|
||||
a hard prerequisite, not an optional nicety.
|
||||
5. **Contribution is configured per provider, off by default.** Every contribution-capable
|
||||
provider has its own enable switch and submit settings, so when more providers exist an
|
||||
operator chooses *which* ones to submit to. Both the master contribute switch and the
|
||||
auto-submit-local-detections switch default to off; a single setting flips each on.
|
||||
6. **API-first, UI deferred.** This round fully specifies the backend (manual marker
|
||||
read/write/clear, contribution trigger + status, per-provider provider config, key
|
||||
validation and stats) so any future surface — web admin or mobile — builds on a complete
|
||||
contract. No new web UI is designed here beyond noting where it will eventually attach.
|
||||
|
||||
## Problem
|
||||
|
||||
Silo currently detects intro/recap/credits/preview markers from two places — a local
|
||||
chromaprint/chapter analyzer (`internal/intromarkers`) and the online TheIntroDB provider
|
||||
(`internal/markers/introdb`) — reconciled by a source-priority ladder at write time. The
|
||||
online side is cleanly abstracted behind a `markers.Provider` interface and a
|
||||
`markers.Registry`, which is good, but three things are missing:
|
||||
|
||||
1. The current TheIntroDB provider has **correctness gaps** measured against the real v3 API
|
||||
contract (TVDB lookups dropped, real confidence discarded — see "Review findings").
|
||||
2. There is **no path to contribute** Silo's markers back to TheIntroDB, even though the v3
|
||||
API exposes a submission endpoint and the package was deliberately built read-only.
|
||||
3. The provider abstraction is **fetch-only and first-hit**, so it does not yet support
|
||||
"several online sources, keep the best answer" or "this source can also accept
|
||||
contributions, and I choose which sources to submit to."
|
||||
|
||||
## Review findings — current implementation
|
||||
|
||||
### What is already right (keep it)
|
||||
|
||||
- **Clean source abstraction.** `markers.Provider` (`ID()`, `FetchMarkers()`) + `markers.Registry`
|
||||
with `FetchFirstHit` (`internal/markers/types.go`). TheIntroDB is *not* hardcoded; a second
|
||||
provider is one `Register()` call (`cmd/silo/main.go`).
|
||||
- **Multi-source-aware schema.** `media_files` carries per-segment provenance for all four
|
||||
kinds: `{intro,credits,recap,preview}_markers_{source,provider,confidence,algorithm,detected_at}`
|
||||
(migrations 108/109/137).
|
||||
- **Write-time reconciliation across source classes.** `CanWriteMarker`
|
||||
(`internal/markers/write.go`) + `MarkerSourcePriority` (`internal/models/marker_source.go`):
|
||||
`manual(4) > online/plugin(3) > s3(2) > scanner(1)`, confidence breaks ties. Local detection
|
||||
uses its own priority-gated path (`intromarkers.Repository.PatchIntroMarker`).
|
||||
- **Solid HTTP hygiene** in `internal/markers/introdb/client.go`: conservative rate limiter
|
||||
(2 req/s, burst 5; documented limit is 30 req/10s), retry with exponential backoff,
|
||||
`Retry-After` handling, 24h TTL cache with negative caching, hot-reloadable API key over
|
||||
Redis pub/sub.
|
||||
|
||||
### Gaps (measured against TheIntroDB's own v3 client + Jellyfin plugin)
|
||||
|
||||
| # | Gap | Evidence | Impact |
|
||||
|---|-----|----------|--------|
|
||||
| 1 | **TVDB lookups silently dropped.** Provider reads only TMDB+IMDB; client can't send `tvdb_id`. | `internal/markers/introdb/provider.go` (reads `ExternalIDKeyTMDB`/`IMDB` only); `client.go` `FetchEpisode/FetchMovie` build `tmdb_id` else `imdb_id`. | Titles matched only by TVDB (very common for anime / TheTVDB-first libraries) get **zero** markers, even though the resolver already populates `TvdbID` and the v3 API + both official clients support `tvdb_id`. Correctness bug. |
|
||||
| 2 | **Real confidence + submission_count discarded; hardcoded 0.9.** | `internal/markers/introdb/types.go` `segmentTimestamps` has only `start_ms`/`end_ms`; `pickMarker` sets `Confidence: 0.9`. | The `/media` response carries per-segment `confidence` and `submission_count`. Without them, "best wins" merging can't rank sources, and write-time confidence tie-breaks are meaningless. Hard blocker for the chosen dispatch model. |
|
||||
| 3 | **Naive multi-candidate selection.** `pickMarker` takes the *first* usable entry. | `internal/markers/introdb/provider.go`. | When several candidates remain (e.g. no duration match), first-usable may not be the most-submitted/highest-confidence. Minor today (duration narrows the set), but compounds once we rank by confidence. |
|
||||
| 4 | **No contribution path** (by design). | `internal/markers/introdb/types.go` package doc: "Submissions are intentionally not supported." | The whole point of this effort. |
|
||||
| 5 | **Provider is fetch-only.** No submit capability in the interface. | `markers.Provider` in `internal/markers/types.go`. | Contribution needs an optional `Submitter` capability so non-contributing sources don't have to implement it, and so submission is per provider. |
|
||||
| 6 | **No key validation / contribution stats.** | `IntroDBCredentialCard` in `web/src/pages/admin-settings/IntegrationsSettings.tsx` only sets the key. | v3 exposes `GET /v3/user/stats` to validate a key and show contribution totals/streaks. Useful "connect & verify" probe, currently absent from the API. |
|
||||
| 7 | **Single collapsed confidence in the write payload.** `BuildUpdatePayload` promotes one max confidence across all segments. | `internal/markers/write.go`. | The DB has *per-segment* confidence columns, but a merged result (intro from source A, credits from source B) can't populate them independently until per-segment confidence is threaded through the payload. Coupled to gap #2. |
|
||||
|
||||
## Authoritative API contract (TheIntroDB v3)
|
||||
|
||||
Confirmed from TheIntroDB's own `theintrodb` npm client (`src/funcs.ts`, `src/types.ts`) and
|
||||
the official Jellyfin plugin (`TheIntroDbClient.cs`). Base URL: `https://api.theintrodb.org/v3`.
|
||||
|
||||
### `GET /media` — read (public; optional key)
|
||||
|
||||
Query: one of `tmdb_id` / `tvdb_id` / `imdb_id` (preference order tmdb → tvdb → imdb),
|
||||
plus `season` + `episode` together for TV, plus optional `duration_ms` (selects the closest
|
||||
release version). Optional `Authorization: Bearer <key>` additionally includes that user's
|
||||
*pending* submissions.
|
||||
|
||||
Response — each segment kind is an array of:
|
||||
```jsonc
|
||||
{ "start_ms": 30000|null, "end_ms": 90000|null, "confidence": 0.93|null, "submission_count": 7|null }
|
||||
```
|
||||
`start_ms:null` = "from the beginning" (intro/recap); `end_ms:null` = "to the end"
|
||||
(credits/preview). Arrays may hold multiple entries (multiple release versions) — pick by
|
||||
duration match, then by `submission_count`/`confidence`.
|
||||
|
||||
### `POST /submit` — contribute (requires key)
|
||||
|
||||
`Authorization: Bearer <user key>` required; submissions are credited to that account.
|
||||
Body (one segment per call):
|
||||
```jsonc
|
||||
{
|
||||
"tmdb_id": 1234, // required
|
||||
"imdb_id": "tt0903747", // optional
|
||||
"type": "tv" | "movie",
|
||||
"segment": "intro" | "recap" | "credits" | "preview",
|
||||
"season": 1, // TV only (number, or "1,2,3" multi-select)
|
||||
"episode": 1, // TV only
|
||||
"video_duration_ms": 2760000, // optional, strongly recommended (release-version match)
|
||||
"start_ms": 0|null, // intro/recap may be null (= beginning)
|
||||
"end_ms": 90000|null // credits/preview may be null (= end)
|
||||
}
|
||||
```
|
||||
Response: `{ "submissions": [ { "id": uuid, "status": "pending"|"accepted"|"rejected",
|
||||
"weight": number, ... } ] }`. Submissions enter as `pending`; the backend weight-averages
|
||||
timestamps across submissions and moderates to accepted/rejected.
|
||||
|
||||
Validation rules to mirror client-side: intro/recap require an **end**; credits/preview
|
||||
require a **start**; TV requires season+episode; movies omit them; max timestamp 6h
|
||||
(21,600,000 ms); `video_duration_ms` is 0 (unknown) or ≥ 300,000 ms.
|
||||
|
||||
### `GET /user/stats` — validate key + contribution stats
|
||||
|
||||
`Authorization: Bearer <key>`. Returns `{ total, accepted, pending, rejected,
|
||||
acceptance_rate, current_streak, best_streak, total_time_saved_ms, top_media }`. A 2xx with no
|
||||
`error` means the key is valid. This is both our key-validation probe and the stats display.
|
||||
|
||||
### Rate / usage limits
|
||||
|
||||
429 responses carry `X-RateLimit-{Limit,Remaining,Reset}` and `X-UsageLimit-{Limit,Remaining,Reset}`.
|
||||
Reads count against the rate limit; **submissions count against the usage limit**. The
|
||||
contribution pipeline must honor `X-UsageLimit-Reset` and back off — reuse the existing
|
||||
limiter and add usage-limit awareness.
|
||||
|
||||
## Target architecture
|
||||
|
||||
Three layers, each independently shippable. Layer A is pure correctness and unblocks B and C.
|
||||
|
||||
### Layer A — Correctness foundation (gaps #1, #2, #3, #7)
|
||||
|
||||
- **TVDB in the client + provider.** Add `tvdb_id` to the query builder in
|
||||
`internal/markers/introdb/client.go` (`FetchEpisode`/`FetchMovie` gain a `tvdbID` arg,
|
||||
preference tmdb → tvdb → imdb, cache keys extended). Provider reads
|
||||
`req.ExternalIDs[markers.ExternalIDKeyTVDB]`. Resolver already supplies it — no resolver
|
||||
change.
|
||||
- **Capture confidence + submission_count.** Extend `segmentTimestamps` with
|
||||
`Confidence *float64` and `SubmissionCount *int`; add `SubmissionCount int` to
|
||||
`markers.Marker`; `pickMarker` uses the real confidence (fallback to a named default
|
||||
constant, e.g. `0.9`, only when the field is null) instead of hardcoding.
|
||||
- **Best-candidate selection.** `pickMarker` ranks candidates by `(submission_count desc,
|
||||
confidence desc)` after duration filtering, instead of first-usable.
|
||||
- **Per-segment confidence in the payload.** Replace the single `Confidence *float64` on
|
||||
`markers.MarkerUpdatePayload` with per-segment confidence (e.g. `IntroConfidence`,
|
||||
`CreditsConfidence`, …) so a merged multi-source result writes the correct
|
||||
`{kind}_markers_confidence` per kind. Update `BuildUpdatePayload` and the
|
||||
`scanner.FileRepository.UpsertMarkers` column mapping accordingly.
|
||||
|
||||
### Layer B — Multi-source dispatch ("query all, best wins")
|
||||
|
||||
- **`Submitter` capability (optional interface).**
|
||||
```go
|
||||
// internal/markers/types.go
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
// FetchUserStats validates the configured key and returns contribution stats.
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
```
|
||||
Only `introdb.Provider` implements it; the contribution service type-asserts
|
||||
`provider.(markers.Submitter)`. This is also what makes contribution per-provider: the
|
||||
service only ever submits to registered providers that satisfy `Submitter` *and* are enabled
|
||||
in `marker_provider_config` (C6).
|
||||
|
||||
- **Merge dispatch.** Add `Registry.FetchMerged(ctx, req) (Result, bool, error)` that fans out
|
||||
to all *fetch-enabled* providers concurrently (bounded), then merges **per segment kind**
|
||||
keeping the candidate with the highest `(submission_count, confidence)`. `FetchFirstHit`
|
||||
stays as the trivial single-provider path / fallback. With only TheIntroDB enabled,
|
||||
`FetchMerged` behaves identically to today — zero behavior change until a second source
|
||||
exists.
|
||||
|
||||
- **Cross-source-class reconciliation is unchanged.** `FetchMerged` reconciles *within* the
|
||||
online class; `CanWriteMarker` still arbitrates online vs scanner vs manual at write time.
|
||||
No double logic.
|
||||
|
||||
- **Provider enable/order config.** Driven by the `marker_provider_config` table (C6):
|
||||
`fetch_enabled` selects which providers are queried, `fetch_priority` is the merge
|
||||
tiebreaker. The metadata `library_provider_chains` DB pattern (`internal/metadata/chain.go`)
|
||||
is the upgrade path *if* per-library control is ever needed — noted, not built.
|
||||
|
||||
- **Future plugin sources (deferred seam).** A `marker_provider.v1` plugin capability would let
|
||||
third-party sources plug in. The clean seam is a `PluginMarkerProvider` adapter implementing
|
||||
`markers.Provider` (mirroring `internal/metadata/plugin_provider.go`), so the registry never
|
||||
changes. Cross-repo (silo-plugin-sdk) — explicitly out of scope here.
|
||||
|
||||
### Layer C — Contribution to TheIntroDB
|
||||
|
||||
#### C1. Submission client
|
||||
|
||||
Add to `internal/markers/introdb/client.go`:
|
||||
- `SubmitSegment(ctx, SubmissionRequest) (SubmissionResult, error)` → `POST /v3/submit`,
|
||||
Bearer key **required** (error early if `introdb.api_key` is empty), usage-limit aware.
|
||||
- `FetchUserStats(ctx) (UserStats, error)` → `GET /v3/user/stats`.
|
||||
`introdb.Provider` implements `markers.Submitter` by delegating to these.
|
||||
|
||||
#### C2. What is eligible to contribute
|
||||
|
||||
A marker is contributable iff:
|
||||
- the target provider is a registered `Submitter` with `contribute_enabled = true` (C6), and
|
||||
- its segment source ∈ `{scanner, manual}` (**never** `online` — that came *from* an online
|
||||
provider; contributing it back is circular), and
|
||||
- the file resolves to a usable external ID (tmdb/tvdb/imdb) via the existing resolver, and
|
||||
- for **auto** contribution: provider `contribute_auto_local = true`, source = `scanner`,
|
||||
kind = `intro`, confidence ≥ that provider's `contribute_min_confidence`. (Local detection is
|
||||
intro-only/episodes-only.)
|
||||
- for **manual** contribution: any kind, episodes or movies, source = `manual`.
|
||||
|
||||
#### C3. Manual marker API (write path; UI deferred)
|
||||
|
||||
Markers live on `media_files`, and an item can have several versions, so the manual API is
|
||||
**file-scoped** for precision, with an item-level convenience resolver. All under the existing
|
||||
`RequireAdmin` group (`internal/api/middleware/auth.go`). New `internal/api/handlers/admin_markers.go`:
|
||||
|
||||
- `GET /admin/files/{fileId}/markers` — current per-segment values **plus full provenance**
|
||||
(`source`, `provider`, `confidence`, `algorithm`, `detected_at`) for each of
|
||||
intro/recap/credits/preview. The read side an editor (or a script) needs.
|
||||
- `PUT /admin/files/{fileId}/markers` — upsert the **manual** layer. Body carries, per segment,
|
||||
either `{start, end}` (seconds) or `null` to clear that segment's manual value. Writes
|
||||
`source = manual` (priority 4, beats every other source) through the existing priority-gated
|
||||
`scanner.FileRepository.UpsertMarkers` path, then notifies live sessions via
|
||||
`playback.MarkerUpdateNotifier.MarkersUpdated` (mirrors `admin_intro.go`).
|
||||
- `DELETE /admin/files/{fileId}/markers/{segment}` — explicit single-segment clear.
|
||||
- `GET|PUT /admin/items/{id}/markers` — item-level convenience that resolves to the primary
|
||||
file (and reports the per-version set), for callers that don't track file IDs.
|
||||
|
||||
**Clear semantics.** Clearing a manual segment nulls that segment's value and provenance;
|
||||
`markers_source` drops to the next-highest segment still present. A subsequent local
|
||||
re-detection or online lazy-fetch can repopulate it. (We can't "un-overwrite" a value a manual
|
||||
edit replaced, so clearing means "remove the manual value," not "restore the previous source.")
|
||||
|
||||
**Validation** mirrors the contribution rules so a manual edit can always be contributed:
|
||||
`end > start`; intro/recap may omit start (= 0); credits/preview may omit end (= file
|
||||
duration); values within [0, file duration].
|
||||
|
||||
A web editor (player-chrome control with a "use current time" helper) attaches to this API
|
||||
later; its UX is intentionally **out of scope** for this round.
|
||||
|
||||
#### C4. Contribution tracking (idempotency + audit)
|
||||
|
||||
The audit log that makes submission idempotent and lets us report status. Per-provider by
|
||||
construction (the `provider` column), so the same marker can be tracked/submitted to several
|
||||
providers independently.
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_contributions.up.sql (next free number at implementation time)
|
||||
CREATE TABLE public.marker_contributions (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
media_file_id integer NOT NULL REFERENCES media_files(id) ON DELETE CASCADE,
|
||||
provider text NOT NULL, -- 'introdb'
|
||||
segment_kind text NOT NULL, -- intro|recap|credits|preview
|
||||
source text NOT NULL, -- what we contributed: 'scanner'|'manual'
|
||||
submitted_start_ms bigint,
|
||||
submitted_end_ms bigint,
|
||||
video_duration_ms bigint,
|
||||
content_hash text NOT NULL, -- hash(segment_kind,start_ms,end_ms,duration_ms)
|
||||
submission_id uuid, -- returned by /submit
|
||||
status text NOT NULL, -- pending|accepted|rejected|error
|
||||
http_status integer,
|
||||
error text,
|
||||
submitted_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (media_file_id, provider, segment_kind, content_hash)
|
||||
);
|
||||
CREATE INDEX marker_contributions_file_idx ON public.marker_contributions(media_file_id);
|
||||
```
|
||||
|
||||
`content_hash` over the *value* gives idempotency: the same times are never resubmitted to the
|
||||
same provider, but a manual correction (different times → different hash) submits the new
|
||||
value — which is exactly how TheIntroDB's weighted averaging expects corrections to arrive.
|
||||
|
||||
#### C5. Contribution service + triggers
|
||||
|
||||
A single `internal/markers/contribute.go` `ContributionService` funnels both triggers:
|
||||
`resolve enabled submitter providers (C6) → per provider: eligibility (C2) → idempotency check
|
||||
(hash vs marker_contributions) → Submitter.SubmitMarker → persist result`. Triggers:
|
||||
- **Manual / on-demand:** `POST /admin/files/{fileId}/contribute` (C7) submits a file's eligible
|
||||
markers to enabled providers (optionally scoped to one provider/segments).
|
||||
- **Auto local detections:** a daily `ContributeMarkersTask` (`internal/taskmanager/tasks/`)
|
||||
that, for each provider with `contribute_enabled && contribute_auto_local`, scans for episode
|
||||
files with `intro_markers_source = 'scanner'` and `intro_markers_confidence ≥` that provider's
|
||||
`contribute_min_confidence` lacking a current-hash `marker_contributions` row, and submits
|
||||
them rate/usage-limited. Runs after the existing 03:30 detection task.
|
||||
|
||||
#### C6. Per-provider configuration
|
||||
|
||||
Contribution and fetch behavior is **per provider**, so a single global toggle won't do.
|
||||
Credentials stay in `server_settings` (sensitive handling: `introdb.api_key`); behavioral
|
||||
config moves to a new table keyed by provider id:
|
||||
|
||||
```sql
|
||||
-- migrations/<next>_marker_provider_config.up.sql
|
||||
CREATE TABLE public.marker_provider_config (
|
||||
provider text PRIMARY KEY, -- 'introdb'
|
||||
fetch_enabled boolean NOT NULL DEFAULT true, -- queried during reads (Layer B)
|
||||
fetch_priority integer NOT NULL DEFAULT 100, -- merge tiebreaker
|
||||
contribute_enabled boolean NOT NULL DEFAULT false, -- master submit switch (off)
|
||||
contribute_auto_local boolean NOT NULL DEFAULT false, -- auto-submit local detections (off)
|
||||
contribute_min_confidence double precision NOT NULL DEFAULT 0.95,
|
||||
updated_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
INSERT INTO public.marker_provider_config (provider, fetch_enabled) VALUES ('introdb', true);
|
||||
```
|
||||
|
||||
- A provider's submissions require `contribute_enabled = true`. With it on, **manual**
|
||||
corrections submit on demand; **auto** submission of local detections additionally requires
|
||||
`contribute_auto_local = true`. Both contribute flags default off — "easily enabled with a
|
||||
setting," exactly one row per provider.
|
||||
- This same table carries the Layer B fetch enable/order, so "which providers do we read from"
|
||||
and "which do we submit to" live in one place. New providers insert a row (contribute off).
|
||||
- Surfaced and edited through the provider API (C7), so the future admin surface lists
|
||||
providers generically rather than hard-coding `introdb`.
|
||||
|
||||
The `markers.mode` / `markers.lazy_playback` globals and the `introdb.api_key` credential are
|
||||
unchanged.
|
||||
|
||||
#### C7. Admin provider + contribution API (fully fledged)
|
||||
|
||||
The complete backend surface, all under `RequireAdmin`, so a UI is purely additive later:
|
||||
|
||||
- `GET /admin/markers/providers` — list every registered provider: id, whether it is a
|
||||
`Submitter`, its `marker_provider_config` row, and (best-effort, cached) live `/user/stats`
|
||||
for submitter providers that have a key.
|
||||
- `PUT /admin/markers/providers/{provider}` — update that provider's config row
|
||||
(`fetch_enabled`, `fetch_priority`, `contribute_enabled`, `contribute_auto_local`,
|
||||
`contribute_min_confidence`). Validated; unknown provider → 404.
|
||||
- `POST /admin/markers/providers/{provider}/validate` — validate the configured key and return
|
||||
fresh `UserStats` (calls `Submitter.FetchUserStats`). Non-submitter → 400.
|
||||
- `POST /admin/files/{fileId}/contribute` — submit this file's eligible markers (C2) to enabled
|
||||
submitter providers; optional body `{provider?, segments?[]}` to scope to one provider /
|
||||
specific kinds. Returns per (provider, segment) `SubmissionResult`. Honors idempotency (C4):
|
||||
unchanged values are skipped, not resubmitted.
|
||||
- `GET /admin/files/{fileId}/contributions` — contribution history for the file from
|
||||
`marker_contributions` (per provider/segment: status, submitted_at, error).
|
||||
|
||||
Plus the manual marker endpoints from C3. Together these cover read, edit, submit, and audit
|
||||
without any web code.
|
||||
|
||||
## Data model summary
|
||||
|
||||
- **New table `marker_contributions`** (C4) — idempotent submission audit, keyed per
|
||||
(file, provider, segment, value-hash).
|
||||
- **New table `marker_provider_config`** (C6) — per-provider fetch + contribute behavior,
|
||||
seeded with `introdb` (fetch on, contribute off).
|
||||
- **No new flat server settings.** `introdb.api_key`, `markers.mode`, `markers.lazy_playback`
|
||||
are unchanged; behavioral provider config lives in `marker_provider_config`, not string keys.
|
||||
- **No new marker columns.** Per-segment confidence columns already exist; Layer A only changes
|
||||
how they're *populated* (per-segment instead of collapsed). `submission_count` stays
|
||||
transient (merge-ranking only) unless a later need appears.
|
||||
|
||||
## Client / cross-repo impact
|
||||
|
||||
- **No Android/Apple changes required.** This adds no new marker *kinds* and no new
|
||||
client-facing read contract: markers still flow to clients via the existing `WatchDetail`
|
||||
fields, the jellycompat `GET /MediaSegments` endpoint, and the `markers_updated` realtime
|
||||
event. Manual editing and contribution are server-side (admin API); their eventual UI is
|
||||
deferred web-admin work.
|
||||
- **Plugin SDK:** untouched. `marker_provider.v1` is a deferred future capability (Layer B
|
||||
seam), not part of this work.
|
||||
- A future *per-user* contribution model (each user links their own TheIntroDB account) *would*
|
||||
be client work (a "Connect TheIntroDB" screen) — explicitly deferred.
|
||||
|
||||
## Build sequence
|
||||
|
||||
Each phase is shippable on its own; later phases depend only on earlier ones. Every phase is
|
||||
**backend-only** — no web UI.
|
||||
|
||||
1. **Phase 1 — Correctness foundation (Layer A).** TVDB lookups; capture
|
||||
confidence/submission_count; best-candidate selection; per-segment confidence in the
|
||||
payload. *Directly answers "are we doing this properly," low risk, no new surface.*
|
||||
2. **Phase 2 — Multi-source dispatch + provider config (Layer B + C6).** `Submitter` interface;
|
||||
`FetchMerged` query-all/best-wins; `marker_provider_config` table + repo driving fetch
|
||||
enable/order. *No behavior change with only TheIntroDB enabled.*
|
||||
3. **Phase 3 — Submission client + tracking + service (C1, C4, C5 core).** `SubmitSegment` /
|
||||
`FetchUserStats`; `marker_contributions` migration + repo; `ContributionService` (eligibility,
|
||||
idempotency, per-provider gating).
|
||||
4. **Phase 4 — Manual marker API (C3) + provider/contribution API (C7).** File- and item-level
|
||||
manual read/write/clear; provider list/config/validate; per-file contribute + history. The
|
||||
full admin contract, no UI.
|
||||
5. **Phase 5 — Auto-contribution (C5 task).** `ContributeMarkersTask`: per provider, when
|
||||
`contribute_enabled && contribute_auto_local`, submit episode intro markers above the
|
||||
provider's `contribute_min_confidence`, usage-limit aware, idempotent.
|
||||
|
||||
Deferred (separate UI-only follow-up): web admin surfaces over C3/C6/C7 — a marker editor in
|
||||
the player chrome and a provider/contribution panel in admin settings.
|
||||
|
||||
## Risks & trade-offs
|
||||
|
||||
- **Data-quality / pollution.** Auto-submitting chromaprint detections (confidence 0.65–0.90)
|
||||
risks polluting a provider's data. Mitigation: `contribute_enabled` and `contribute_auto_local`
|
||||
default **off** per provider, and the default `contribute_min_confidence` of 0.95 admits only
|
||||
chapter / chapter+silence detections even once enabled. Manual, high-trust corrections are the
|
||||
intended first contribution. Revisit chromaprint auto-submission after watching acceptance
|
||||
rates via `/user/stats`.
|
||||
- **Single-account weighting.** All contributions credit one server account; TheIntroDB's
|
||||
weighting may treat a single account's mass submissions differently than many users'. Note
|
||||
the per-user seam; acceptable for v1.
|
||||
- **Circular contribution.** Never contribute `source = online`. Enforced in C2 eligibility and
|
||||
worth a unit test.
|
||||
- **Usage limits.** Bulk auto-submission can hit the submission usage limit; the task must obey
|
||||
`X-UsageLimit-Reset` and resume next run. Idempotency means an interrupted run safely resumes.
|
||||
- **"Query all" cost.** `FetchMerged` issues one lookup per fetch-enabled provider; with only
|
||||
TheIntroDB it's identical to today, and per-provider caching bounds the cost as sources are
|
||||
added.
|
||||
- **Movies & non-intro segments** have no local detector, so their only contribution route is
|
||||
manual correction — acceptable and explicit.
|
||||
|
||||
## Open questions for review
|
||||
|
||||
1. **Contribute to all enabled providers, or require explicit per-call selection?** The design
|
||||
defaults `POST /admin/files/{id}/contribute` to "every enabled submitter," with an optional
|
||||
`provider` scope. Acceptable, or should submission always name its target provider?
|
||||
2. **Manual-clear behavior.** Clearing a manual segment removes the manual value and lets a
|
||||
later detection/fetch repopulate it (we can't restore the exact pre-edit value). Is
|
||||
"remove, then re-detect" the right contract, or do you want a marker history/undo stack?
|
||||
3. **Per-provider config home.** This design uses a `marker_provider_config` table (clean for N
|
||||
providers, future per-library override) rather than flat `server_settings` keys. Confirm the
|
||||
table is the direction you want.
|
||||
|
||||
## File-level blueprint
|
||||
|
||||
**Backend (Go) — Layer A**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `tvdb_id` query support; cache-key extension.
|
||||
- `internal/markers/introdb/types.go` — *modify*: `confidence`/`submission_count` fields.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: read TVDB; real confidence; best-candidate pick.
|
||||
- `internal/markers/types.go` — *modify*: per-segment confidence in `Marker`/Result plumbing.
|
||||
- `internal/markers/write.go` — *modify*: per-segment confidence in `MarkerUpdatePayload` + `BuildUpdatePayload`.
|
||||
- `internal/scanner/file_repo.go` — *modify*: map per-segment confidence in `UpsertMarkers`.
|
||||
|
||||
**Backend (Go) — Layer B + per-provider config**
|
||||
- `internal/markers/types.go` — *modify*: `Submitter` interface; `SubmissionRequest`/`Result`, `UserStats`; `Registry.FetchMerged`.
|
||||
- `internal/markers/provider_config.go` — *create*: `marker_provider_config` repo (load/update; drives fetch enable/order + contribute gating).
|
||||
- `migrations/<next>_marker_provider_config.{up,down}.sql` — *create*.
|
||||
|
||||
**Backend (Go) — Layer C**
|
||||
- `internal/markers/introdb/client.go` — *modify*: `SubmitSegment` (`POST /v3/submit`), `FetchUserStats` (`GET /v3/user/stats`), usage-limit handling.
|
||||
- `internal/markers/introdb/provider.go` — *modify*: implement `markers.Submitter`; drop the "submissions not supported" note in `types.go`.
|
||||
- `internal/markers/contribute.go` — *create*: `ContributionService` (eligibility, idempotency, per-provider gating, submit, persist).
|
||||
- `internal/markers/contribution_repo.go` — *create*: `marker_contributions` CRUD + value-hash check.
|
||||
- `migrations/<next>_marker_contributions.{up,down}.sql` — *create*.
|
||||
- `internal/api/handlers/admin_markers.go` — *create*: manual marker read/write/clear (C3) + per-file contribute + history (C7 file endpoints).
|
||||
- `internal/api/handlers/admin_marker_providers.go` — *create*: provider list/config/validate (C7 provider endpoints).
|
||||
- `internal/api/router.go` — *modify*: register the new `RequireAdmin` routes.
|
||||
- `internal/taskmanager/tasks/contribute_markers.go` — *create*: daily auto-contribution task.
|
||||
- `cmd/silo/main.go` — *modify*: wire `ContributionService`, provider-config repo, the task, and the `Submitter`.
|
||||
|
||||
**Frontend (React/TS) — deferred (not this round)**
|
||||
- Marker editor over `PUT /admin/files/{id}/markers`; provider/contribution panel over the C7
|
||||
endpoints; `IntroDBCredentialCard` "validate key & show stats". Listed for traceability only.
|
||||
|
||||
**Reference (read-only, not in this repo)**
|
||||
- TheIntroDB v3 contract verified against `github.com/TheIntroDB/theintrodb-npm`
|
||||
(`src/funcs.ts`, `src/types.ts`, `docs/`) and `github.com/TheIntroDB/jellyfin-plugin`
|
||||
(`TheIntroDB/Api/*.cs`).
|
||||
@@ -29,24 +29,28 @@ require (
|
||||
github.com/mmcdole/gofeed v1.3.0
|
||||
github.com/oklog/ulid/v2 v2.1.0
|
||||
github.com/pgvector/pgvector-go v0.3.0
|
||||
github.com/pressly/goose/v3 v3.27.1
|
||||
github.com/zishang520/socket.io/v2 v2.5.0
|
||||
go.n16f.net/thumbhash v1.1.0
|
||||
golang.org/x/image v0.39.0
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/PuerkitoBio/goquery v1.8.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/brotli v1.2.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.1 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/gookit/color v1.5.4 // indirect
|
||||
github.com/hashicorp/yamux v0.1.2 // indirect
|
||||
github.com/json-iterator/go v1.1.12 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/klauspost/compress v1.18.5 // indirect
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-isatty v0.0.21 // indirect
|
||||
github.com/mfridman/interpolate v0.0.2 // indirect
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -55,6 +59,7 @@ require (
|
||||
github.com/quic-go/quic-go v0.53.0 // indirect
|
||||
github.com/rogpeppe/go-internal v1.14.1 // indirect
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect
|
||||
github.com/sethvargo/go-retry v0.3.0 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
|
||||
@@ -64,14 +69,14 @@ require (
|
||||
github.com/zishang520/webtransport-go v0.9.1 // indirect
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 // indirect
|
||||
go.uber.org/mock v0.5.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
golang.org/x/mod v0.34.0 // indirect
|
||||
golang.org/x/net v0.53.0 // indirect
|
||||
golang.org/x/tools v0.43.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
|
||||
@@ -90,15 +95,14 @@ require (
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/prometheus/common v0.66.1 // indirect
|
||||
github.com/prometheus/procfs v0.16.1 // indirect
|
||||
github.com/prometheus/procfs v0.20.1 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/sony/sonyflake v1.3.0
|
||||
go.opentelemetry.io/otel v1.41.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v2 v2.4.2 // indirect
|
||||
golang.org/x/sync v0.20.0
|
||||
golang.org/x/text v0.36.0
|
||||
golang.org/x/time v0.14.0
|
||||
google.golang.org/grpc v1.79.3
|
||||
google.golang.org/grpc v1.80.0
|
||||
google.golang.org/protobuf v1.36.11
|
||||
)
|
||||
|
||||
@@ -2,12 +2,12 @@ entgo.io/ent v0.14.3 h1:wokAV/kIlH9TeklJWGGS7AYJdVckr0DloWjIcO9iIIQ=
|
||||
entgo.io/ent v0.14.3/go.mod h1:aDPE/OziPEu8+OWbzy4UlvWmD2/kbRuWfK2A40hcxJM=
|
||||
github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
|
||||
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220 h1:tiwfULdh9yOW7HO6fYf8v+lXIsrCrGfp3GM8ymhrBME=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.4.1-0.20260604220604-0d7865171220/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0 h1:0yJpD0RP1AetGsWVCXoPih2ggLjN8hVp8blS+2RkIWU=
|
||||
github.com/Silo-Server/silo-plugin-sdk v0.5.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
|
||||
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
|
||||
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
|
||||
github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
|
||||
github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro=
|
||||
github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
|
||||
github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
|
||||
github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
|
||||
@@ -44,6 +44,7 @@ github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/
|
||||
github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -51,6 +52,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
|
||||
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
|
||||
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
|
||||
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
|
||||
@@ -108,8 +111,8 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
|
||||
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE=
|
||||
github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
|
||||
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
@@ -125,10 +128,12 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs=
|
||||
github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4=
|
||||
github.com/mattn/go-sqlite3 v1.14.34 h1:3NtcvcUnFBPsuRcno8pUtupspG/GM+9nZ88zgJcp6Zk=
|
||||
github.com/mattn/go-sqlite3 v1.14.34/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY=
|
||||
github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg=
|
||||
github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
|
||||
github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
|
||||
github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
|
||||
@@ -140,6 +145,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
|
||||
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
|
||||
github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA=
|
||||
github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
|
||||
github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU=
|
||||
@@ -149,26 +156,32 @@ github.com/pgvector/pgvector-go v0.3.0 h1:Ij+Yt78R//uYqs3Zk35evZFvr+G0blW0OUN+Q2
|
||||
github.com/pgvector/pgvector-go v0.3.0/go.mod h1:duFy+PXWfW7QQd5ibqutBO4GxLsUZ9RVXhFZGIBsWSA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pressly/goose/v3 v3.27.1 h1:6uEvcprBybDmW4hcz3gYujhARhye+GoWKhEWyzD5sh4=
|
||||
github.com/pressly/goose/v3 v3.27.1/go.mod h1:maruOxsPnIG2yHHyo8UqKWXYKFcH7Q76csUV7+7KYoM=
|
||||
github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
|
||||
github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
|
||||
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
|
||||
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
|
||||
github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
|
||||
github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
|
||||
github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
|
||||
github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
|
||||
github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc=
|
||||
github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo=
|
||||
github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
|
||||
github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
|
||||
github.com/quic-go/quic-go v0.53.0 h1:QHX46sISpG2S03dPeZBgVIZp8dGagIaiu2FiVYvpCZI=
|
||||
github.com/quic-go/quic-go v0.53.0/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
|
||||
github.com/redis/go-redis/v9 v9.18.0 h1:pMkxYPkEbMPwRdenAzUNyFNrDgHx9U+DrBabWNfSRQs=
|
||||
github.com/redis/go-redis/v9 v9.18.0/go.mod h1:k3ufPphLU5YXwNTUcCRXGxUoF1fqxnhFQmscfkCoDA0=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
|
||||
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ=
|
||||
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU=
|
||||
github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE=
|
||||
github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas=
|
||||
github.com/sony/sonyflake v1.3.0 h1:tiB4Dlp0lnmKp/h6BLXA14P8Qi+LYS9+0QRpcrKHvg4=
|
||||
github.com/sony/sonyflake v1.3.0/go.mod h1:LORtCywH/cq10ZbyfhKrHYgAUGH7mOBa76enV9txy/Y=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -215,22 +228,24 @@ go.n16f.net/thumbhash v1.1.0 h1:aBEvuAd4yiwzeQ7Sm4BZoHJYbrQ1ewjrmrRlCE79snk=
|
||||
go.n16f.net/thumbhash v1.1.0/go.mod h1:mo9pP7WtfdV9ojIamGFR/Vc0PaPA2l0CUtmYQf/SweU=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
|
||||
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
|
||||
go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
|
||||
go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
|
||||
go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
|
||||
go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
|
||||
go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
|
||||
go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
|
||||
go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
|
||||
go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
|
||||
go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0 h1:siZQIYBAUd1rlIWQT2uCxWJxcCO7q3TriaMlf08rXw8=
|
||||
go.opentelemetry.io/otel/sdk/metric v1.41.0/go.mod h1:HNBuSvT7ROaGtGI50ArdRLUnvRTRGniSUZbxiWxSO8Y=
|
||||
go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
|
||||
go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
|
||||
go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
|
||||
go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
|
||||
go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
|
||||
go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
|
||||
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
|
||||
@@ -251,7 +266,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
|
||||
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
@@ -263,12 +277,12 @@ golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
|
||||
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
|
||||
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
|
||||
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
|
||||
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
|
||||
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
|
||||
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
|
||||
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
|
||||
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
|
||||
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
|
||||
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
|
||||
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
@@ -285,3 +299,11 @@ gorm.io/gorm v1.25.5 h1:zR9lOiiYf09VNh5Q1gphfyia1JpiClIWG9hQaxB/mls=
|
||||
gorm.io/gorm v1.25.5/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
|
||||
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
|
||||
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
|
||||
modernc.org/libc v1.72.1 h1:db1xwJ6u1kE3KHTFTTbe2GCrczHPKzlURP0aDC4NGD0=
|
||||
modernc.org/libc v1.72.1/go.mod h1:HRMiC/PhPGLIPM7GzAFCbI+oSgE3dhZ8FWftmRrHVlY=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
|
||||
modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/sqlite v1.49.1 h1:dYGHTKcX1sJ+EQDnUzvz4TJ5GbuvhNJa8Fg6ElGx73U=
|
||||
modernc.org/sqlite v1.49.1/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=
|
||||
|
||||
@@ -113,20 +113,34 @@ func NewAdminHandler(
|
||||
|
||||
// createUserRequest represents the JSON body for POST /admin/users.
|
||||
type createUserRequest struct {
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions []string `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
Username string `json:"username"`
|
||||
Email string `json:"email"`
|
||||
Password string `json:"password"`
|
||||
Role string `json:"role"`
|
||||
Permissions createStringSliceField `json:"permissions"`
|
||||
CreateDefaultProfile bool `json:"create_default_profile"`
|
||||
DefaultProfileName string `json:"default_profile_name,omitempty"`
|
||||
LibraryIDs []int `json:"library_ids"`
|
||||
MaxPlaybackQuality string `json:"max_playback_quality"`
|
||||
MaxStreams *int `json:"max_streams,omitempty"`
|
||||
MaxTranscodes *int `json:"max_transcodes,omitempty"`
|
||||
MaxProfiles *int `json:"max_profiles,omitempty"`
|
||||
DownloadAllowed *bool `json:"download_allowed,omitempty"`
|
||||
DownloadTranscodeAllowed *bool `json:"download_transcode_allowed,omitempty"`
|
||||
}
|
||||
|
||||
type createStringSliceField struct {
|
||||
Set bool
|
||||
Value []string
|
||||
}
|
||||
|
||||
func (f *createStringSliceField) UnmarshalJSON(data []byte) error {
|
||||
f.Set = true
|
||||
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
|
||||
f.Value = []string{}
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(data, &f.Value)
|
||||
}
|
||||
|
||||
type updateLibraryIDsField struct {
|
||||
@@ -391,7 +405,11 @@ func (h *AdminHandler) HandleCreateUser(w http.ResponseWriter, r *http.Request)
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "max_profiles must be at least 1")
|
||||
return
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(req.Permissions)
|
||||
permissions := auth.DefaultUserPermissions()
|
||||
if req.Permissions.Set {
|
||||
permissions = req.Permissions.Value
|
||||
}
|
||||
permissions, err := auth.NormalizePermissions(permissions)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
// AdminMarkerProvidersHandler serves the per-provider marker config + key
|
||||
// validation API under the RequireAdmin group.
|
||||
type AdminMarkerProvidersHandler struct {
|
||||
Registry *markers.Registry
|
||||
Config *markers.ProviderConfigStore
|
||||
EventBus cache.EventBus
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewAdminMarkerProvidersHandler constructs the handler.
|
||||
func NewAdminMarkerProvidersHandler(registry *markers.Registry, config *markers.ProviderConfigStore, eventBus cache.EventBus, logger *slog.Logger) *AdminMarkerProvidersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &AdminMarkerProvidersHandler{Registry: registry, Config: config, EventBus: eventBus, logger: logger}
|
||||
}
|
||||
|
||||
type providerConfigResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
DisplayName string `json:"display_name,omitempty"`
|
||||
SourceType string `json:"source_type,omitempty"`
|
||||
PluginID string `json:"plugin_id,omitempty"`
|
||||
PluginInstallationID int `json:"plugin_installation_id,omitempty"`
|
||||
CapabilityID string `json:"capability_id,omitempty"`
|
||||
IsSubmitter bool `json:"is_submitter"`
|
||||
FetchEnabled bool `json:"fetch_enabled"`
|
||||
FetchPriority int `json:"fetch_priority"`
|
||||
ContributeEnabled bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
|
||||
type markerUserStatsResponse struct {
|
||||
Total int `json:"total"`
|
||||
Accepted int `json:"accepted"`
|
||||
Pending int `json:"pending"`
|
||||
Rejected int `json:"rejected"`
|
||||
AcceptanceRate float64 `json:"acceptance_rate"`
|
||||
CurrentStreak int `json:"current_streak"`
|
||||
BestStreak int `json:"best_streak"`
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) submitterIDs() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if _, ok := p.(markers.Submitter); ok {
|
||||
out[p.ID()] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (h *AdminMarkerProvidersHandler) providerDescriptions() map[string]markers.ProviderDescriptor {
|
||||
out := map[string]markers.ProviderDescriptor{}
|
||||
if h.Registry == nil {
|
||||
return out
|
||||
}
|
||||
for _, p := range h.Registry.Providers() {
|
||||
desc := markers.ProviderDescriptor{ID: p.ID()}
|
||||
if described, ok := p.(markers.DescribedProvider); ok {
|
||||
desc = described.ProviderDescription()
|
||||
}
|
||||
if desc.ID == "" {
|
||||
desc.ID = p.ID()
|
||||
}
|
||||
out[p.ID()] = desc
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// HandleListProviders lists registered providers with their config + capability.
|
||||
func (h *AdminMarkerProvidersHandler) HandleListProviders(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
submitters := h.submitterIDs()
|
||||
descriptions := h.providerDescriptions()
|
||||
out := []providerConfigResponse{}
|
||||
if h.Registry != nil {
|
||||
for _, provider := range h.Registry.Providers() {
|
||||
c, ok := h.Config.Get(provider.ID())
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
} else {
|
||||
for _, c := range h.Config.List() {
|
||||
out = append(out, toProviderConfigResponse(c, submitters[c.Provider], descriptions[c.Provider]))
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"providers": out})
|
||||
}
|
||||
|
||||
// HandleUpdateProvider updates a provider's config row.
|
||||
func (h *AdminMarkerProvidersHandler) HandleUpdateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Config == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
existing, ok := h.Config.Get(provider)
|
||||
if !ok {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Unknown marker provider")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
FetchEnabled *bool `json:"fetch_enabled"`
|
||||
FetchPriority *int `json:"fetch_priority"`
|
||||
ContributeEnabled *bool `json:"contribute_enabled"`
|
||||
ContributeAutoLocal *bool `json:"contribute_auto_local"`
|
||||
ContributeMinConfidence *float64 `json:"contribute_min_confidence"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if body.FetchEnabled != nil {
|
||||
existing.FetchEnabled = *body.FetchEnabled
|
||||
}
|
||||
if body.FetchPriority != nil {
|
||||
existing.FetchPriority = *body.FetchPriority
|
||||
}
|
||||
if body.ContributeEnabled != nil {
|
||||
existing.ContributeEnabled = *body.ContributeEnabled
|
||||
}
|
||||
if body.ContributeAutoLocal != nil {
|
||||
existing.ContributeAutoLocal = *body.ContributeAutoLocal
|
||||
}
|
||||
if body.ContributeMinConfidence != nil {
|
||||
v := *body.ContributeMinConfidence
|
||||
if v < 0 || v > 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "contribute_min_confidence must be between 0 and 1")
|
||||
return
|
||||
}
|
||||
existing.ContributeMinConfidence = v
|
||||
}
|
||||
if err := h.Config.Update(r.Context(), existing); err != nil {
|
||||
h.logger.Error("admin markers: update provider config failed", "provider", provider, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update provider")
|
||||
return
|
||||
}
|
||||
if h.EventBus != nil {
|
||||
_ = h.EventBus.Publish(r.Context(), cache.ChannelAdmin, cache.Event{
|
||||
Type: cache.EventMarkerProviderConfigChanged,
|
||||
Payload: provider,
|
||||
})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, toProviderConfigResponse(existing, h.submitterIDs()[provider], h.providerDescriptions()[provider]))
|
||||
}
|
||||
|
||||
// HandleValidateProvider validates the provider's configured key and returns stats.
|
||||
func (h *AdminMarkerProvidersHandler) HandleValidateProvider(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Registry == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker providers are not configured")
|
||||
return
|
||||
}
|
||||
provider := chi.URLParam(r, "provider")
|
||||
var submitter markers.Submitter
|
||||
for _, p := range h.Registry.Providers() {
|
||||
if p.ID() == provider {
|
||||
if s, ok := p.(markers.Submitter); ok {
|
||||
submitter = s
|
||||
}
|
||||
}
|
||||
}
|
||||
if submitter == nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Provider does not support contribution")
|
||||
return
|
||||
}
|
||||
stats, err := submitter.FetchUserStats(r.Context())
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": false, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"valid": true, "stats": toMarkerUserStatsResponse(stats)})
|
||||
}
|
||||
|
||||
func toProviderConfigResponse(c markers.ProviderConfig, isSubmitter bool, desc markers.ProviderDescriptor) providerConfigResponse {
|
||||
return providerConfigResponse{
|
||||
Provider: c.Provider,
|
||||
DisplayName: desc.DisplayName,
|
||||
SourceType: desc.SourceType,
|
||||
PluginID: desc.PluginID,
|
||||
PluginInstallationID: desc.PluginInstallationID,
|
||||
CapabilityID: desc.CapabilityID,
|
||||
IsSubmitter: isSubmitter,
|
||||
FetchEnabled: c.FetchEnabled,
|
||||
FetchPriority: c.FetchPriority,
|
||||
ContributeEnabled: c.ContributeEnabled,
|
||||
ContributeAutoLocal: c.ContributeAutoLocal,
|
||||
ContributeMinConfidence: c.ContributeMinConfidence,
|
||||
}
|
||||
}
|
||||
|
||||
func toMarkerUserStatsResponse(s markers.UserStats) markerUserStatsResponse {
|
||||
return markerUserStatsResponse{
|
||||
Total: s.Total,
|
||||
Accepted: s.Accepted,
|
||||
Pending: s.Pending,
|
||||
Rejected: s.Rejected,
|
||||
AcceptanceRate: s.AcceptanceRate,
|
||||
CurrentStreak: s.CurrentStreak,
|
||||
BestStreak: s.BestStreak,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
)
|
||||
|
||||
type fakeMarkerStatsSubmitter struct{}
|
||||
|
||||
func (fakeMarkerStatsSubmitter) ID() string { return "introdb" }
|
||||
func (fakeMarkerStatsSubmitter) FetchMarkers(context.Context, markers.Request) (markers.Result, error) {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) SubmitMarker(context.Context, markers.SubmissionRequest) (markers.SubmissionResult, error) {
|
||||
return markers.SubmissionResult{}, nil
|
||||
}
|
||||
func (fakeMarkerStatsSubmitter) FetchUserStats(context.Context) (markers.UserStats, error) {
|
||||
return markers.UserStats{Total: 10, Accepted: 7, Pending: 2, Rejected: 1, AcceptanceRate: 0.7, CurrentStreak: 3, BestStreak: 5}, nil
|
||||
}
|
||||
|
||||
func TestValidateMarkerProviderUsesSnakeCaseStats(t *testing.T) {
|
||||
reg := markers.NewRegistry(nil)
|
||||
if err := reg.Register(fakeMarkerStatsSubmitter{}); err != nil {
|
||||
t.Fatalf("register provider: %v", err)
|
||||
}
|
||||
h := NewAdminMarkerProvidersHandler(reg, nil, nil, nil)
|
||||
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/markers/providers/introdb/validate", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("provider", "introdb")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleValidateProvider(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"acceptance_rate":0.7`) || strings.Contains(body, "AcceptanceRate") {
|
||||
t.Fatalf("unexpected stats response shape: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerProviderResponseIncludesPluginMetadata(t *testing.T) {
|
||||
resp := toProviderConfigResponse(
|
||||
markers.ProviderConfig{Provider: "plugin:4:markers", FetchEnabled: true, FetchPriority: 25},
|
||||
true,
|
||||
markers.ProviderDescriptor{
|
||||
DisplayName: "Plugin Markers",
|
||||
SourceType: markers.ProviderSourcePlugin,
|
||||
PluginID: "silo.plugin.markers",
|
||||
PluginInstallationID: 4,
|
||||
CapabilityID: "markers",
|
||||
},
|
||||
)
|
||||
if resp.DisplayName != "Plugin Markers" || resp.SourceType != markers.ProviderSourcePlugin {
|
||||
t.Fatalf("plugin metadata fields missing: %+v", resp)
|
||||
}
|
||||
if resp.PluginID != "silo.plugin.markers" ||
|
||||
resp.PluginInstallationID != 4 ||
|
||||
resp.CapabilityID != "markers" ||
|
||||
!resp.IsSubmitter {
|
||||
t.Fatalf("plugin identity fields missing: %+v", resp)
|
||||
}
|
||||
}
|
||||
@@ -3,13 +3,11 @@ package handlers
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
@@ -24,15 +22,16 @@ func (h *AudiobookHandler) HandleGetAudiobook(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
item, err := h.Items.GetByID(r.Context(), contentID)
|
||||
items, err := h.Items.GetByIDsWithAccess(r.Context(), []string{contentID}, requestAccessFilter(r))
|
||||
if err != nil {
|
||||
if errors.Is(err, catalog.ErrItemNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load audiobook failed")
|
||||
return
|
||||
}
|
||||
if len(items) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
item := items[0]
|
||||
if item == nil || item.Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
@@ -63,18 +62,18 @@ func (h *AudiobookHandler) HandleGetAudiobook(w http.ResponseWriter, r *http.Req
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterURL: h.presignPoster(r.Context(), item.PosterPath),
|
||||
PosterURL: h.presignAudiobookPoster(r.Context(), item.PosterPath),
|
||||
Publisher: pickFirstString(item.Studios),
|
||||
Genres: append([]string(nil), item.Genres...),
|
||||
},
|
||||
Author: author,
|
||||
Narrator: narrator,
|
||||
Files: audiobookDetailFiles(files),
|
||||
Progress: progress,
|
||||
AlsoByAuthor: alsoByAuthor,
|
||||
InSeries: inSeries,
|
||||
SimilarAudiobooks: similar,
|
||||
OtherNarrations: otherNarrations,
|
||||
Author: author,
|
||||
Narrator: narrator,
|
||||
Files: audiobookDetailFiles(files),
|
||||
Progress: progress,
|
||||
AlsoByAuthor: alsoByAuthor,
|
||||
InSeries: inSeries,
|
||||
SimilarAudiobooks: similar,
|
||||
OtherNarrations: otherNarrations,
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
@@ -483,7 +482,7 @@ type audiobookDetailItem struct {
|
||||
|
||||
func (h *AudiobookHandler) presignPoster(ctx context.Context, path string) string {
|
||||
if path == "" || h.Detail == nil {
|
||||
return path
|
||||
return ""
|
||||
}
|
||||
return h.Detail.PresignURL(ctx, path, "featured")
|
||||
}
|
||||
@@ -498,14 +497,14 @@ func pickFirstString(values []string) string {
|
||||
}
|
||||
|
||||
type audiobookDetailFile struct {
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
AudioChannels int `json:"audio_channels,omitempty"`
|
||||
Chapters []models.MediaChapter `json:"chapters,omitempty"`
|
||||
ID int `json:"id"`
|
||||
Path string `json:"path"`
|
||||
DurationSeconds int `json:"duration_seconds"`
|
||||
Container string `json:"container,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Bitrate int `json:"bitrate,omitempty"`
|
||||
AudioChannels int `json:"audio_channels,omitempty"`
|
||||
Chapters []models.MediaChapter `json:"chapters,omitempty"`
|
||||
}
|
||||
|
||||
type audiobookDetailProgress struct {
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
@@ -27,7 +30,8 @@ type AudiobookHandler struct {
|
||||
}
|
||||
|
||||
// HandleListAudiobooks serves GET /api/v1/audiobooks?limit=&offset=.
|
||||
// Paginated catalog of media_items with type='audiobook'.
|
||||
// Paginated catalog of media_items with type='audiobook' scoped to the
|
||||
// caller's accessible libraries.
|
||||
func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.Request) {
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
if limit <= 0 || limit > 200 {
|
||||
@@ -37,19 +41,31 @@ func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.R
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
access := requestAccessFilter(r)
|
||||
genre := r.URL.Query().Get("genre")
|
||||
conditions, args, argIdx, empty := audiobookListConditions(access, genre)
|
||||
if empty {
|
||||
writeAudiobookListResponse(w, nil, 0, limit, offset)
|
||||
return
|
||||
}
|
||||
|
||||
pool := h.Files.Pool()
|
||||
ctx := r.Context()
|
||||
var total int
|
||||
if err := pool.QueryRow(ctx, `
|
||||
SELECT COUNT(*) FROM media_items
|
||||
WHERE type='audiobook' AND ($1 = '' OR $1 = ANY(genres))
|
||||
`, genre).Scan(&total); err != nil {
|
||||
countSQL := "SELECT COUNT(*) FROM media_items mi WHERE " + strings.Join(conditions, " AND ")
|
||||
if err := pool.QueryRow(ctx, countSQL, args...).Scan(&total); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks count failed")
|
||||
return
|
||||
}
|
||||
rows, err := pool.Query(ctx, `
|
||||
|
||||
dataArgs := append([]any(nil), args...)
|
||||
limitIdx := argIdx
|
||||
dataArgs = append(dataArgs, limit)
|
||||
offsetIdx := argIdx + 1
|
||||
dataArgs = append(dataArgs, offset)
|
||||
|
||||
rows, err := pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT
|
||||
mi.content_id,
|
||||
mi.title,
|
||||
@@ -61,33 +77,83 @@ func (h *AudiobookHandler) HandleListAudiobooks(w http.ResponseWriter, r *http.R
|
||||
COUNT(mf.id) AS file_count
|
||||
FROM media_items mi
|
||||
LEFT JOIN media_files mf ON mf.content_id = mi.content_id
|
||||
WHERE mi.type='audiobook'
|
||||
AND ($3 = '' OR $3 = ANY(mi.genres))
|
||||
WHERE %s
|
||||
GROUP BY mi.content_id, mi.title, mi.year, mi.poster_path, mi.sort_title
|
||||
ORDER BY LOWER(mi.sort_title), LOWER(mi.title)
|
||||
LIMIT $1 OFFSET $2
|
||||
`, limit, offset, genre)
|
||||
LIMIT $%d OFFSET $%d
|
||||
`, strings.Join(conditions, " AND "), limitIdx, offsetIdx), dataArgs...)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "list audiobooks failed")
|
||||
return
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
items := make([]audiobookListItem, 0, limit)
|
||||
for rows.Next() {
|
||||
var it audiobookListItem
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &it.PosterURL, &it.DurationSeconds, &it.Container, &it.CodecAudio, &it.FileCount); err != nil {
|
||||
var posterPath string
|
||||
if err := rows.Scan(&it.ContentID, &it.Title, &it.Year, &posterPath, &it.DurationSeconds, &it.Container, &it.CodecAudio, &it.FileCount); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "scan audiobook row failed")
|
||||
return
|
||||
}
|
||||
if h.Detail != nil && it.PosterURL != "" {
|
||||
it.PosterURL = h.Detail.PresignURL(ctx, it.PosterURL, "card")
|
||||
}
|
||||
it.PosterURL = h.presignAudiobookPoster(ctx, posterPath)
|
||||
items = append(items, it)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "iterate audiobook rows failed")
|
||||
return
|
||||
}
|
||||
writeAudiobookListResponse(w, items, total, limit, offset)
|
||||
}
|
||||
|
||||
func audiobookListConditions(access catalog.AccessFilter, genre string) ([]string, []any, int, bool) {
|
||||
conditions := []string{"mi.type = 'audiobook'"}
|
||||
var args []any
|
||||
argIdx := 1
|
||||
|
||||
if genre != "" {
|
||||
conditions = append(conditions, fmt.Sprintf("$%d = ANY(mi.genres)", argIdx))
|
||||
args = append(args, genre)
|
||||
argIdx++
|
||||
}
|
||||
if access.AllowedLibraryIDs != nil {
|
||||
if len(access.AllowedLibraryIDs) == 0 {
|
||||
return nil, nil, 1, true
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf(`
|
||||
EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil
|
||||
WHERE mil.content_id = mi.content_id
|
||||
AND mil.media_folder_id = ANY($%d)
|
||||
)`, argIdx))
|
||||
args = append(args, access.AllowedLibraryIDs)
|
||||
argIdx++
|
||||
}
|
||||
if len(access.DisabledLibraryIDs) > 0 {
|
||||
if access.AllowedLibraryIDs == nil {
|
||||
conditions = append(conditions, `
|
||||
EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil
|
||||
WHERE mil.content_id = mi.content_id
|
||||
)`)
|
||||
}
|
||||
conditions = append(conditions, fmt.Sprintf(`
|
||||
NOT EXISTS (
|
||||
SELECT 1 FROM media_item_libraries mil
|
||||
WHERE mil.content_id = mi.content_id
|
||||
AND mil.media_folder_id = ANY($%d)
|
||||
)`, argIdx))
|
||||
args = append(args, access.DisabledLibraryIDs)
|
||||
argIdx++
|
||||
}
|
||||
catalog.ApplySectionAccessFilter("mi", catalog.AccessFilter{MaxContentRating: access.MaxContentRating}, &conditions, &args, &argIdx)
|
||||
return conditions, args, argIdx, false
|
||||
}
|
||||
|
||||
func writeAudiobookListResponse(w http.ResponseWriter, items []audiobookListItem, total, limit, offset int) {
|
||||
if items == nil {
|
||||
items = []audiobookListItem{}
|
||||
}
|
||||
resp := struct {
|
||||
Items []audiobookListItem `json:"items"`
|
||||
Total int `json:"total"`
|
||||
@@ -108,3 +174,10 @@ type audiobookListItem struct {
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
FileCount int `json:"file_count"`
|
||||
}
|
||||
|
||||
func (h *AudiobookHandler) presignAudiobookPoster(ctx context.Context, path string) string {
|
||||
if h == nil || h.Detail == nil || path == "" {
|
||||
return ""
|
||||
}
|
||||
return h.Detail.PresignURL(ctx, cardThumbnailPath(path), "card")
|
||||
}
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
)
|
||||
|
||||
@@ -17,8 +19,6 @@ type audiobookProgressRequest struct {
|
||||
|
||||
// HandleReportAudiobookProgress serves POST /api/v1/audiobooks/{id}/progress.
|
||||
// UPSERTs into user_watch_progress for the caller's (user_id, profile_id, content_id).
|
||||
// Duration is unknown at the audiobook level (it lives on the media_file), so we
|
||||
// pass 0 and let the store compute completion via thresholds against position only.
|
||||
func (h *AudiobookHandler) HandleReportAudiobookProgress(w http.ResponseWriter, r *http.Request) {
|
||||
contentID := chi.URLParam(r, "id")
|
||||
if contentID == "" {
|
||||
@@ -46,6 +46,26 @@ func (h *AudiobookHandler) HandleReportAudiobookProgress(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if h.Items == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "catalog unavailable")
|
||||
return
|
||||
}
|
||||
items, err := h.Items.GetByIDsWithAccess(r.Context(), []string{contentID}, requestAccessFilter(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "check access failed")
|
||||
return
|
||||
}
|
||||
if len(items) == 0 || items[0] == nil || items[0].Type != "audiobook" {
|
||||
writeError(w, http.StatusNotFound, "not_found", "audiobook not found")
|
||||
return
|
||||
}
|
||||
|
||||
durationSeconds, err := h.audiobookProgressDuration(r.Context(), contentID, req.MediaFileID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "load files failed")
|
||||
return
|
||||
}
|
||||
|
||||
if h.StoreProvider == nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "store unavailable")
|
||||
return
|
||||
@@ -56,13 +76,38 @@ func (h *AudiobookHandler) HandleReportAudiobookProgress(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
// Duration is not carried in the progress body; pass 0 so the store
|
||||
// uses position-only heuristics. ProgressThresholds zero value means
|
||||
// "use defaults" (90% watched, 5% min-resume).
|
||||
if err := store.SetProgress(r.Context(), profileID, contentID, req.PositionSeconds, 0, userstore.ProgressThresholds{}); err != nil {
|
||||
if err := store.SetProgress(r.Context(), profileID, contentID, req.PositionSeconds, durationSeconds, userstore.ProgressThresholds{}); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "save progress failed")
|
||||
return
|
||||
}
|
||||
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (h *AudiobookHandler) audiobookProgressDuration(ctx context.Context, contentID string, mediaFileID int) (float64, error) {
|
||||
if h == nil || h.Files == nil {
|
||||
return 0, nil
|
||||
}
|
||||
files, err := h.Files.GetByContentID(ctx, contentID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return audiobookDurationForProgress(files, mediaFileID), nil
|
||||
}
|
||||
|
||||
func audiobookDurationForProgress(files []*models.MediaFile, mediaFileID int) float64 {
|
||||
var total int
|
||||
for _, f := range files {
|
||||
if f == nil || f.Duration <= 0 {
|
||||
continue
|
||||
}
|
||||
if mediaFileID > 0 && f.ID == mediaFileID {
|
||||
return float64(f.Duration)
|
||||
}
|
||||
total += f.Duration
|
||||
}
|
||||
if total <= 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(total)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestPresignAudiobookPosterDoesNotExposeRawKeysWithoutPresigner(t *testing.T) {
|
||||
h := &AudiobookHandler{}
|
||||
got := h.presignAudiobookPoster(context.Background(), "metadata/audiobooks/book-1/poster/original.webp")
|
||||
if got != "" {
|
||||
t.Fatalf("PosterURL = %q, want empty without presigner", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudiobookListConditionsIncludeAccessPredicates(t *testing.T) {
|
||||
conditions, args, _, empty := audiobookListConditions(catalog.AccessFilter{
|
||||
AllowedLibraryIDs: []int{1, 2},
|
||||
DisabledLibraryIDs: []int{9},
|
||||
MaxContentRating: "PG-13",
|
||||
}, "Fantasy")
|
||||
if empty {
|
||||
t.Fatal("empty = true, want false")
|
||||
}
|
||||
sql := strings.Join(conditions, " AND ")
|
||||
for _, want := range []string{
|
||||
"mi.type = 'audiobook'",
|
||||
"ANY(mi.genres)",
|
||||
"EXISTS",
|
||||
"NOT EXISTS",
|
||||
"mi.content_rating = ANY",
|
||||
} {
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("conditions missing %q in:\n%s", want, sql)
|
||||
}
|
||||
}
|
||||
if len(args) != 4 {
|
||||
t.Fatalf("len(args) = %d, want 4", len(args))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudiobookDurationForProgress(t *testing.T) {
|
||||
files := []*models.MediaFile{
|
||||
{ID: 10, Duration: 120},
|
||||
{ID: 11, Duration: 180},
|
||||
}
|
||||
|
||||
if got := audiobookDurationForProgress(files, 10); got != 120 {
|
||||
t.Fatalf("selected file duration = %v, want 120", got)
|
||||
}
|
||||
if got := audiobookDurationForProgress(files, 0); got != 300 {
|
||||
t.Fatalf("total duration = %v, want 300", got)
|
||||
}
|
||||
if got := audiobookDurationForProgress([]*models.MediaFile{{ID: 1}}, 1); got != 0 {
|
||||
t.Fatalf("unknown duration = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -391,13 +391,17 @@ func (h *EventsHandler) writeSnapshotFrame(
|
||||
) error {
|
||||
data, err := h.snapshotForChannel(r, claims, channel)
|
||||
if err != nil {
|
||||
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
|
||||
_ = writeWebSocketControl(
|
||||
conn,
|
||||
websocket.CloseMessage,
|
||||
websocket.FormatCloseMessage(websocket.CloseInternalServerErr, "snapshot failed"),
|
||||
slog.Error(
|
||||
"events: failed to build initial snapshot",
|
||||
"channel",
|
||||
channel,
|
||||
"user_id",
|
||||
claims.UserID,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
return err
|
||||
writeWebSocketError(conn, "internal_error", "Failed to load snapshot")
|
||||
return nil
|
||||
}
|
||||
return writeWebSocketJSON(conn, evt.EventsSnapshotMessage{
|
||||
Type: "snapshot",
|
||||
|
||||
@@ -150,7 +150,7 @@ func (h *ItemsHandler) maybeRequestStaleEpisodeMetadataRefresh(ctx context.Conte
|
||||
if h == nil || h.metadataRefreshRequester == nil || episode == nil {
|
||||
return
|
||||
}
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, time.Now()) {
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, time.Now()) {
|
||||
h.requestStaleMetadataRefresh(ctx, "episode", episode.ContentID)
|
||||
}
|
||||
}
|
||||
@@ -161,7 +161,7 @@ func (h *ItemsHandler) maybeRequestStaleSeasonMetadataRefresh(ctx context.Contex
|
||||
}
|
||||
now := time.Now()
|
||||
for _, episode := range episodes {
|
||||
if metadata.EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if metadata.EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
h.requestStaleMetadataRefresh(ctx, "season", seasonID)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,849 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimw "github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
// MarkerFileResolver loads a media file by id for the manual-marker API.
|
||||
type MarkerFileResolver interface {
|
||||
GetByID(ctx context.Context, id int) (*models.MediaFile, error)
|
||||
GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error)
|
||||
GetByEpisodeID(ctx context.Context, episodeID string) ([]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
// ManualMarkerWriter persists/clears manual markers.
|
||||
type ManualMarkerWriter interface {
|
||||
UpsertMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate) (bool, error)
|
||||
ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error)
|
||||
UpsertAndClearMarkers(ctx context.Context, fileID int, update scanner.MarkerUpdate, clearSegments []string) (bool, error)
|
||||
}
|
||||
|
||||
// MarkerContributor submits a file's eligible markers to enabled providers.
|
||||
type MarkerContributor interface {
|
||||
ContributeFile(ctx context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error)
|
||||
}
|
||||
|
||||
// MarkerContributionLister reads contribution history for a file.
|
||||
type MarkerContributionLister interface {
|
||||
ListByFile(ctx context.Context, fileID int) ([]markers.ContributionRow, error)
|
||||
}
|
||||
|
||||
type MarkerAuditLister interface {
|
||||
ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
ListAllMarkerEditAudit(ctx context.Context, limit int) ([]scanner.MarkerEditAuditRow, error)
|
||||
}
|
||||
|
||||
type MarkerPermissionUserLoader interface {
|
||||
GetByID(ctx context.Context, id int) (*models.User, error)
|
||||
}
|
||||
|
||||
// MarkersHandler serves the manual-marker + contribution API. The marker
|
||||
// read/write/clear routes are mounted for any authenticated viewer (users fix
|
||||
// and create markers from the player); the contribution + history routes stay
|
||||
// admin-only. A successful manual write fires a background contribution run
|
||||
// (see maybeContribute) so corrected markers reach enabled providers.
|
||||
type MarkersHandler struct {
|
||||
Files MarkerFileResolver
|
||||
Writer ManualMarkerWriter
|
||||
Contributor MarkerContributor
|
||||
Contributions MarkerContributionLister
|
||||
AuditHistory MarkerAuditLister
|
||||
Notifier PlaybackMarkerUpdateNotifier
|
||||
Users MarkerPermissionUserLoader
|
||||
// Authorizer enforces per-item access on file lookups so a viewer can only
|
||||
// edit markers for content they can actually watch. When nil (tests) the
|
||||
// handler falls back to an unchecked lookup.
|
||||
Authorizer *MediaFileAuthorizer
|
||||
// BaseContext is the lifetime context for detached background work (set from
|
||||
// the app context) so contribution goroutines cancel on shutdown.
|
||||
BaseContext context.Context
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewMarkersHandler constructs the handler.
|
||||
func NewMarkersHandler(files MarkerFileResolver, writer ManualMarkerWriter, contributor MarkerContributor, contributions MarkerContributionLister, notifier PlaybackMarkerUpdateNotifier, logger *slog.Logger) *MarkersHandler {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &MarkersHandler{Files: files, Writer: writer, Contributor: contributor, Contributions: contributions, Notifier: notifier, logger: logger}
|
||||
}
|
||||
|
||||
var manualMarkerConfidence = 1.0
|
||||
|
||||
const manualMarkerAlgorithm = "manual:v1"
|
||||
|
||||
// markerContributeTimeout bounds the detached contribution run kicked off after
|
||||
// a manual save so a slow or rate-limited provider can't leak goroutines.
|
||||
const markerContributeTimeout = 30 * time.Second
|
||||
|
||||
var markerSegmentNames = []string{"intro", "credits", "recap", "preview"}
|
||||
|
||||
type segmentInput struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
}
|
||||
|
||||
type segmentMarker struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Source *string `json:"source"`
|
||||
Provider *string `json:"provider"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
Algorithm *string `json:"algorithm"`
|
||||
DetectedAt *time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
type fileMarkersResponse struct {
|
||||
FileID int `json:"file_id"`
|
||||
Intro segmentMarker `json:"intro"`
|
||||
Credits segmentMarker `json:"credits"`
|
||||
Recap segmentMarker `json:"recap"`
|
||||
Preview segmentMarker `json:"preview"`
|
||||
}
|
||||
|
||||
type contributionOutcomeResponse struct {
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Status string `json:"status"`
|
||||
SubmissionID string `json:"submission_id,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RetryAfterSeconds int `json:"retry_after_seconds,omitempty"`
|
||||
}
|
||||
|
||||
type contributionRowResponse struct {
|
||||
ID string `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
Provider string `json:"provider"`
|
||||
Segment string `json:"segment"`
|
||||
Source string `json:"source"`
|
||||
SubmittedStartMS *int64 `json:"submitted_start_ms,omitempty"`
|
||||
SubmittedEndMS *int64 `json:"submitted_end_ms,omitempty"`
|
||||
VideoDurationMS *int64 `json:"video_duration_ms,omitempty"`
|
||||
ContentHash string `json:"content_hash"`
|
||||
SubmissionID *string `json:"submission_id,omitempty"`
|
||||
Status string `json:"status"`
|
||||
HTTPStatus *int `json:"http_status,omitempty"`
|
||||
Error *string `json:"error,omitempty"`
|
||||
SubmittedAt *time.Time `json:"submitted_at,omitempty"`
|
||||
UpdatedAt *time.Time `json:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type markerEditAuditResponse struct {
|
||||
ID int64 `json:"id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
ItemID *string `json:"item_id,omitempty"`
|
||||
ItemType *string `json:"item_type,omitempty"`
|
||||
MediaTitle *string `json:"media_title,omitempty"`
|
||||
FilePath *string `json:"file_path,omitempty"`
|
||||
Segment string `json:"segment"`
|
||||
Action string `json:"action"`
|
||||
Before *segmentMarker `json:"before"`
|
||||
After *segmentMarker `json:"after"`
|
||||
UserID *int `json:"user_id,omitempty"`
|
||||
Username *string `json:"username,omitempty"`
|
||||
ImpersonatorUserID *int `json:"impersonator_user_id,omitempty"`
|
||||
ImpersonatorUsername *string `json:"impersonator_username,omitempty"`
|
||||
APIKeyID *int64 `json:"api_key_id,omitempty"`
|
||||
RequestID *string `json:"request_id,omitempty"`
|
||||
ClientIP *string `json:"client_ip,omitempty"`
|
||||
UserAgent *string `json:"user_agent,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
func fileMarkers(file *models.MediaFile) fileMarkersResponse {
|
||||
return fileMarkersResponse{
|
||||
FileID: file.ID,
|
||||
Intro: segmentMarker{file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersProvider,
|
||||
file.IntroMarkersConfidence, file.IntroMarkersAlgorithm, file.IntroMarkersDetectedAt},
|
||||
Credits: segmentMarker{file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersProvider,
|
||||
file.CreditsMarkersConfidence, file.CreditsMarkersAlgorithm, file.CreditsMarkersDetectedAt},
|
||||
Recap: segmentMarker{file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersProvider,
|
||||
file.RecapMarkersConfidence, file.RecapMarkersAlgorithm, file.RecapMarkersDetectedAt},
|
||||
Preview: segmentMarker{file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersProvider,
|
||||
file.PreviewMarkersConfidence, file.PreviewMarkersAlgorithm, file.PreviewMarkersDetectedAt},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
fileID, err := strconv.Atoi(chi.URLParam(r, "fileId"))
|
||||
if err != nil || fileID <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid file id is required")
|
||||
return nil, false
|
||||
}
|
||||
return h.authorizeFile(w, r, fileID)
|
||||
}
|
||||
|
||||
// authorizeFile loads a file, enforcing per-item access when an authorizer is
|
||||
// configured. The 404-on-denial mirrors the playback/subtitle paths so marker
|
||||
// editing can't be used to probe content outside a profile's library scope.
|
||||
func (h *MarkersHandler) authorizeFile(w http.ResponseWriter, r *http.Request, fileID int) (*models.MediaFile, bool) {
|
||||
if h.Authorizer != nil {
|
||||
file, err := h.Authorizer.Authorize(r, fileID)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, catalog.ErrItemNotFound), errors.Is(err, catalog.ErrEpisodeNotFound):
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
default:
|
||||
h.logger.Error("markers: authorize failed", "file_id", fileID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
file, err := h.Files.GetByID(r.Context(), fileID)
|
||||
if err != nil || file == nil {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return nil, false
|
||||
}
|
||||
return file, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) loadItemPrimaryFile(w http.ResponseWriter, r *http.Request) (*models.MediaFile, bool) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker editing is not configured")
|
||||
return nil, false
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
if len(files) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
// Re-validate the resolved primary file through the authorizer so item-id
|
||||
// edits are access-checked the same way file-id edits are.
|
||||
if h.Authorizer != nil {
|
||||
for _, file := range files {
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
authorized, err := h.Authorizer.Authorize(r, file.ID)
|
||||
if err == nil {
|
||||
return authorized, true
|
||||
}
|
||||
if errors.Is(err, catalog.ErrItemNotFound) || errors.Is(err, catalog.ErrEpisodeNotFound) {
|
||||
continue
|
||||
}
|
||||
h.logger.Error("markers: authorize item file failed", "item_id", itemID, "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file")
|
||||
return nil, false
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
return file, true
|
||||
}
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) requireMarkerEdit(w http.ResponseWriter, r *http.Request) bool {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
|
||||
return false
|
||||
}
|
||||
if claims.Role == "admin" {
|
||||
return true
|
||||
}
|
||||
if h == nil || h.Users == nil {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
user, err := h.Users.GetByID(r.Context(), claims.UserID)
|
||||
if err != nil || user == nil || !auth.HasEffectivePermission(user, auth.PermissionMarkerEdit) {
|
||||
writeError(w, http.StatusForbidden, "forbidden", "Marker editing permission required")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) auditContext(r *http.Request) context.Context {
|
||||
claims := apimw.GetClaims(r.Context())
|
||||
if claims == nil {
|
||||
return r.Context()
|
||||
}
|
||||
audit := scanner.MarkerAuditContext{
|
||||
UserID: &claims.UserID,
|
||||
ImpersonatorUserID: claims.ImpersonatorUserID,
|
||||
RequestID: chimw.GetReqID(r.Context()),
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
UserAgent: r.UserAgent(),
|
||||
}
|
||||
if claims.APIKeyID > 0 {
|
||||
apiKeyID := claims.APIKeyID
|
||||
audit.APIKeyID = &apiKeyID
|
||||
}
|
||||
return scanner.WithMarkerAuditContext(r.Context(), audit)
|
||||
}
|
||||
|
||||
// HandleGetFileMarkers returns the current markers + provenance for a file.
|
||||
func (h *MarkersHandler) HandleGetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleGetItemMarkers returns markers for the item's primary file.
|
||||
func (h *MarkersHandler) HandleGetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(file))
|
||||
}
|
||||
|
||||
// HandleSetFileMarkers upserts the manual marker layer. Each segment key may be
|
||||
// an object {start, end} to set, or null to clear; absent keys are unchanged.
|
||||
func (h *MarkersHandler) HandleSetFileMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
// HandleSetItemMarkers upserts manual markers on the item's primary file.
|
||||
func (h *MarkersHandler) HandleSetItemMarkers(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadItemPrimaryFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
h.setMarkersForFile(w, r, file)
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) setMarkersForFile(w http.ResponseWriter, r *http.Request, file *models.MediaFile) {
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
|
||||
var raw map[string]json.RawMessage
|
||||
if err := json.NewDecoder(r.Body).Decode(&raw); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
duration := float64(file.Duration)
|
||||
update := scanner.MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceManual,
|
||||
MarkersConfidence: &manualMarkerConfidence,
|
||||
MarkersAlgorithm: manualMarkerAlgorithm,
|
||||
}
|
||||
var clears []string
|
||||
var setSegs []string
|
||||
|
||||
for _, seg := range markerSegmentNames {
|
||||
val, present := raw[seg]
|
||||
if !present {
|
||||
continue
|
||||
}
|
||||
if isJSONNull(val) {
|
||||
clears = append(clears, seg)
|
||||
continue
|
||||
}
|
||||
var in segmentInput
|
||||
if err := json.Unmarshal(val, &in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid "+seg+" marker")
|
||||
return
|
||||
}
|
||||
start, end, err := normalizeManualSegment(seg, in, duration)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
applyManualSegment(&update, seg, start, end)
|
||||
setSegs = append(setSegs, seg)
|
||||
}
|
||||
|
||||
if _, err := h.Writer.UpsertAndClearMarkers(h.auditContext(r), file.ID, update, clears); err != nil {
|
||||
h.logger.Error("markers: save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save markers")
|
||||
return
|
||||
}
|
||||
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after save failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Markers saved but failed to reload")
|
||||
return
|
||||
}
|
||||
h.maybeContribute(refreshed, setSegs)
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleClearFileSegment clears a single segment.
|
||||
func (h *MarkersHandler) HandleClearFileSegment(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Writer == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker writing is not configured")
|
||||
return
|
||||
}
|
||||
if !h.requireMarkerEdit(w, r) {
|
||||
return
|
||||
}
|
||||
segment := chi.URLParam(r, "segment")
|
||||
if !isMarkerSegment(segment) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown marker segment")
|
||||
return
|
||||
}
|
||||
if _, err := h.Writer.ClearMarkers(h.auditContext(r), file.ID, []string{segment}); err != nil {
|
||||
h.logger.Error("markers: clear segment failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to clear marker")
|
||||
return
|
||||
}
|
||||
refreshed, err := h.reloadAndNotify(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: reload after clear failed", "file_id", file.ID, "segment", segment, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Marker cleared but failed to reload")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, fileMarkers(refreshed))
|
||||
}
|
||||
|
||||
// HandleContributeFile submits the file's eligible markers to enabled providers.
|
||||
func (h *MarkersHandler) HandleContributeFile(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributor == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Contribution is not configured")
|
||||
return
|
||||
}
|
||||
var body struct {
|
||||
Provider string `json:"provider"`
|
||||
Segments []string `json:"segments"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil && !errors.Is(err, io.EOF) {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
var kinds []markers.MarkerKind
|
||||
for _, name := range body.Segments {
|
||||
kind, ok := markerKindForName(name)
|
||||
if !ok {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Unknown segment "+name)
|
||||
return
|
||||
}
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
outcomes, err := h.Contributor.ContributeFile(r.Context(), file, markers.ContributeOptions{Provider: body.Provider, Segments: kinds})
|
||||
if err != nil {
|
||||
h.logger.Error("markers: contribute failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Contribution failed")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"outcomes": contributionOutcomeResponses(outcomes)})
|
||||
}
|
||||
|
||||
// HandleListFileContributions returns the contribution history for a file.
|
||||
func (h *MarkersHandler) HandleListFileContributions(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if h.Contributions == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": []contributionRowResponse{}})
|
||||
return
|
||||
}
|
||||
rows, err := h.Contributions.ListByFile(r.Context(), file.ID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list contributions failed", "file_id", file.ID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load contributions")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"contributions": contributionRowResponses(rows)})
|
||||
}
|
||||
|
||||
// HandleListFileMarkerHistory returns recent semantic marker edit audit rows for one file.
|
||||
func (h *MarkersHandler) HandleListFileMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
file, ok := h.loadFile(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, []int{file.ID})
|
||||
}
|
||||
|
||||
// HandleListItemMarkerHistory returns recent marker edit audit rows for every file version on an item.
|
||||
func (h *MarkersHandler) HandleListItemMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h == nil || h.Files == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Marker history is not configured")
|
||||
return
|
||||
}
|
||||
itemID := strings.TrimSpace(chi.URLParam(r, "id"))
|
||||
if itemID == "" {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid item id is required")
|
||||
return
|
||||
}
|
||||
|
||||
files, err := h.Files.GetByEpisodeID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: episode history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
if len(files) == 0 {
|
||||
files, err = h.Files.GetByContentID(r.Context(), itemID)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: content history file lookup failed", "item_id", itemID, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load item files")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
fileIDs := make([]int, 0, len(files))
|
||||
for _, file := range files {
|
||||
if file != nil {
|
||||
fileIDs = append(fileIDs, file.ID)
|
||||
}
|
||||
}
|
||||
if len(fileIDs) == 0 {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found for item")
|
||||
return
|
||||
}
|
||||
h.listMarkerHistory(w, r, fileIDs)
|
||||
}
|
||||
|
||||
// HandleListMarkerHistory returns recent semantic marker edit audit rows across all files.
|
||||
func (h *MarkersHandler) HandleListMarkerHistory(w http.ResponseWriter, r *http.Request) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListAllMarkerEditAudit(r.Context(), limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list all history failed", "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) listMarkerHistory(w http.ResponseWriter, r *http.Request, fileIDs []int) {
|
||||
if h.AuditHistory == nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": []markerEditAuditResponse{}})
|
||||
return
|
||||
}
|
||||
limit, ok := markerHistoryLimit(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rows, err := h.AuditHistory.ListMarkerEditAudit(r.Context(), fileIDs, limit)
|
||||
if err != nil {
|
||||
h.logger.Error("markers: list history failed", "file_ids", fileIDs, "error", err)
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load marker history")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"history": markerEditAuditResponses(rows)})
|
||||
}
|
||||
|
||||
func markerHistoryLimit(w http.ResponseWriter, r *http.Request) (int, bool) {
|
||||
limit := 25
|
||||
if raw := strings.TrimSpace(r.URL.Query().Get("limit")); raw != "" {
|
||||
parsed, err := strconv.Atoi(raw)
|
||||
if err != nil || parsed <= 0 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "A valid limit is required")
|
||||
return 0, false
|
||||
}
|
||||
limit = parsed
|
||||
}
|
||||
return limit, true
|
||||
}
|
||||
|
||||
func (h *MarkersHandler) reloadAndNotify(ctx context.Context, fileID int) (*models.MediaFile, error) {
|
||||
refreshed, err := h.Files.GetByID(ctx, fileID)
|
||||
if err != nil || refreshed == nil {
|
||||
if err == nil {
|
||||
err = scanner.ErrFileNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if h.Notifier != nil {
|
||||
h.Notifier.MarkersUpdated(ctx, refreshed)
|
||||
}
|
||||
return refreshed, nil
|
||||
}
|
||||
|
||||
// maybeContribute fires a best-effort, detached contribution run for the
|
||||
// segments just set by a manual save. ContributeFile self-gates on each
|
||||
// provider's contribute_enabled flag, only submits scanner/manual-sourced
|
||||
// segments, and is idempotent — so this is a no-op when contribution is off or
|
||||
// the marker was already submitted. It runs in the background so the save
|
||||
// responds immediately and is never blocked by provider rate limits.
|
||||
func (h *MarkersHandler) maybeContribute(file *models.MediaFile, segments []string) {
|
||||
if h == nil || h.Contributor == nil || file == nil || len(segments) == 0 {
|
||||
return
|
||||
}
|
||||
kinds := make([]markers.MarkerKind, 0, len(segments))
|
||||
for _, seg := range segments {
|
||||
if kind, ok := markerKindForName(seg); ok {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
}
|
||||
if len(kinds) == 0 {
|
||||
return
|
||||
}
|
||||
base := h.BaseContext
|
||||
if base == nil {
|
||||
base = context.Background()
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(base, markerContributeTimeout)
|
||||
defer cancel()
|
||||
if _, err := h.Contributor.ContributeFile(ctx, file, markers.ContributeOptions{Segments: kinds}); err != nil {
|
||||
h.logger.Warn("markers: background contribution failed", "file_id", file.ID, "error", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func isJSONNull(raw json.RawMessage) bool {
|
||||
return string(bytes.TrimSpace(raw)) == "null"
|
||||
}
|
||||
|
||||
func isMarkerSegment(seg string) bool {
|
||||
_, ok := markerKindForName(seg)
|
||||
return ok
|
||||
}
|
||||
|
||||
func markerKindForName(name string) (markers.MarkerKind, bool) {
|
||||
switch name {
|
||||
case "intro":
|
||||
return markers.MarkerKindIntro, true
|
||||
case "credits":
|
||||
return markers.MarkerKindCredits, true
|
||||
case "recap":
|
||||
return markers.MarkerKindRecap, true
|
||||
case "preview":
|
||||
return markers.MarkerKindPreview, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func markerNameForKind(kind markers.MarkerKind) string {
|
||||
switch kind {
|
||||
case markers.MarkerKindIntro:
|
||||
return "intro"
|
||||
case markers.MarkerKindCredits:
|
||||
return "credits"
|
||||
case markers.MarkerKindRecap:
|
||||
return "recap"
|
||||
case markers.MarkerKindPreview:
|
||||
return "preview"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func contributionOutcomeResponses(outcomes []markers.ContributionOutcome) []contributionOutcomeResponse {
|
||||
resp := make([]contributionOutcomeResponse, 0, len(outcomes))
|
||||
for _, o := range outcomes {
|
||||
item := contributionOutcomeResponse{
|
||||
Provider: o.Provider,
|
||||
Segment: markerNameForKind(o.Segment),
|
||||
Status: o.Status,
|
||||
SubmissionID: o.SubmissionID,
|
||||
Reason: o.Reason,
|
||||
}
|
||||
if o.RetryAfter > 0 {
|
||||
item.RetryAfterSeconds = int(o.RetryAfter.Seconds())
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func contributionRowResponses(rows []markers.ContributionRow) []contributionRowResponse {
|
||||
resp := make([]contributionRowResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
item := contributionRowResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
Provider: row.Provider,
|
||||
Segment: row.SegmentKind,
|
||||
Source: row.Source,
|
||||
SubmittedStartMS: row.SubmittedStartMs,
|
||||
SubmittedEndMS: row.SubmittedEndMs,
|
||||
VideoDurationMS: row.VideoDurationMs,
|
||||
ContentHash: row.ContentHash,
|
||||
SubmissionID: row.SubmissionID,
|
||||
Status: row.Status,
|
||||
HTTPStatus: row.HTTPStatus,
|
||||
Error: row.Error,
|
||||
}
|
||||
if !row.SubmittedAt.IsZero() {
|
||||
submittedAt := row.SubmittedAt
|
||||
item.SubmittedAt = &submittedAt
|
||||
}
|
||||
if !row.UpdatedAt.IsZero() {
|
||||
updatedAt := row.UpdatedAt
|
||||
item.UpdatedAt = &updatedAt
|
||||
}
|
||||
resp = append(resp, item)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerEditAuditResponses(rows []scanner.MarkerEditAuditRow) []markerEditAuditResponse {
|
||||
resp := make([]markerEditAuditResponse, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
resp = append(resp, markerEditAuditResponse{
|
||||
ID: row.ID,
|
||||
MediaFileID: row.MediaFileID,
|
||||
ItemID: row.ItemID,
|
||||
ItemType: row.ItemType,
|
||||
MediaTitle: row.MediaTitle,
|
||||
FilePath: row.FilePath,
|
||||
Segment: row.SegmentKind,
|
||||
Action: row.Action,
|
||||
Before: markerAuditSegmentResponse(row.Before),
|
||||
After: markerAuditSegmentResponse(row.After),
|
||||
UserID: row.UserID,
|
||||
Username: row.Username,
|
||||
ImpersonatorUserID: row.ImpersonatorUserID,
|
||||
ImpersonatorUsername: row.ImpersonatorUsername,
|
||||
APIKeyID: row.APIKeyID,
|
||||
RequestID: row.RequestID,
|
||||
ClientIP: row.ClientIP,
|
||||
UserAgent: row.UserAgent,
|
||||
CreatedAt: row.CreatedAt,
|
||||
})
|
||||
}
|
||||
return resp
|
||||
}
|
||||
|
||||
func markerAuditSegmentResponse(segment *scanner.MarkerAuditSegment) *segmentMarker {
|
||||
if segment == nil {
|
||||
return nil
|
||||
}
|
||||
return &segmentMarker{
|
||||
Start: segment.Start,
|
||||
End: segment.End,
|
||||
Source: segment.Source,
|
||||
Provider: segment.Provider,
|
||||
Confidence: segment.Confidence,
|
||||
Algorithm: segment.Algorithm,
|
||||
DetectedAt: segment.DetectedAt,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeManualSegment applies the start/end defaults and validation, mirroring
|
||||
// the contribution rules: intro/recap may omit start (=0); credits/preview may
|
||||
// omit end (=duration); end must exceed start and stay within the file.
|
||||
func normalizeManualSegment(seg string, in segmentInput, duration float64) (start, end float64, err error) {
|
||||
switch seg {
|
||||
case "intro", "recap":
|
||||
if in.Start != nil {
|
||||
start = *in.Start
|
||||
}
|
||||
if in.End == nil {
|
||||
return 0, 0, errSegment(seg, "end is required")
|
||||
}
|
||||
end = *in.End
|
||||
default: // credits, preview
|
||||
if in.Start == nil {
|
||||
return 0, 0, errSegment(seg, "start is required")
|
||||
}
|
||||
start = *in.Start
|
||||
if in.End != nil {
|
||||
end = *in.End
|
||||
} else if duration > 0 {
|
||||
end = duration
|
||||
} else {
|
||||
return 0, 0, errSegment(seg, "end is required when duration is unknown")
|
||||
}
|
||||
}
|
||||
if start < 0 || end <= start {
|
||||
return 0, 0, errSegment(seg, "end must be greater than start")
|
||||
}
|
||||
if duration > 0 && end > duration+1 {
|
||||
return 0, 0, errSegment(seg, "end exceeds the file duration")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
func applyManualSegment(update *scanner.MarkerUpdate, seg string, start, end float64) {
|
||||
s, e := start, end
|
||||
switch seg {
|
||||
case "intro":
|
||||
update.IntroStart, update.IntroEnd = &s, &e
|
||||
case "credits":
|
||||
update.CreditsStart, update.CreditsEnd = &s, &e
|
||||
case "recap":
|
||||
update.RecapStart, update.RecapEnd = &s, &e
|
||||
case "preview":
|
||||
update.PreviewStart, update.PreviewEnd = &s, &e
|
||||
}
|
||||
}
|
||||
|
||||
func errSegment(seg, msg string) error {
|
||||
return &markerValidationError{seg: seg, msg: msg}
|
||||
}
|
||||
|
||||
type markerValidationError struct {
|
||||
seg string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *markerValidationError) Error() string { return e.seg + " marker: " + e.msg }
|
||||
@@ -0,0 +1,524 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
)
|
||||
|
||||
type fakeMarkerFiles struct {
|
||||
file *models.MediaFile
|
||||
byID map[int]*models.MediaFile
|
||||
contentFiles []*models.MediaFile
|
||||
episodeFiles []*models.MediaFile
|
||||
}
|
||||
|
||||
func (f fakeMarkerFiles) GetByID(_ context.Context, id int) (*models.MediaFile, error) {
|
||||
if f.byID != nil {
|
||||
return f.byID[id], nil
|
||||
}
|
||||
return f.file, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByContentID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.contentFiles, nil
|
||||
}
|
||||
func (f fakeMarkerFiles) GetByEpisodeID(context.Context, string) ([]*models.MediaFile, error) {
|
||||
return f.episodeFiles, nil
|
||||
}
|
||||
|
||||
type fakeMarkerWriter struct {
|
||||
upserts []scanner.MarkerUpdate
|
||||
clears [][]string
|
||||
audits []scanner.MarkerAuditContext
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) captureAudit(ctx context.Context) {
|
||||
if audit, ok := scanner.MarkerAuditContextFromContext(ctx); ok {
|
||||
f.audits = append(f.audits, audit)
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeMarkerWriter) UpsertMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.upserts = append(f.upserts, u)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) ClearMarkers(ctx context.Context, _ int, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
f.clears = append(f.clears, segs)
|
||||
return true, nil
|
||||
}
|
||||
func (f *fakeMarkerWriter) UpsertAndClearMarkers(ctx context.Context, _ int, u scanner.MarkerUpdate, segs []string) (bool, error) {
|
||||
f.captureAudit(ctx)
|
||||
if u.HasAnySegment() {
|
||||
f.upserts = append(f.upserts, u)
|
||||
}
|
||||
if len(segs) > 0 {
|
||||
f.clears = append(f.clears, segs)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func markerPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/files/5", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func markerItemPutRequest(body string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodPut, "/markers/items/episode-1", strings.NewReader(body))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
return withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
}
|
||||
|
||||
func withMarkerAdminClaims(req *http.Request) *http.Request {
|
||||
return req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{
|
||||
UserID: 1,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
SessionID: "session-1",
|
||||
}))
|
||||
}
|
||||
|
||||
func newMarkersHandler(writer ManualMarkerWriter) *MarkersHandler {
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
return NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
}
|
||||
|
||||
func TestSetFileMarkersWritesManual(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
u := writer.upserts[0]
|
||||
if u.MarkersSource != models.MarkerSourceManual {
|
||||
t.Errorf("source = %q, want manual", u.MarkersSource)
|
||||
}
|
||||
if u.IntroStart == nil || *u.IntroStart != 0 || u.IntroEnd == nil || *u.IntroEnd != 60 {
|
||||
t.Errorf("intro = %v..%v, want 0..60", u.IntroStart, u.IntroEnd)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetItemMarkersWritesPrimaryEpisodeFile(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
file := &models.MediaFile{ID: 8, Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{8: file},
|
||||
episodeFiles: []*models.MediaFile{file},
|
||||
}
|
||||
h := NewMarkersHandler(files, writer, nil, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetItemMarkers(rec, markerItemPutRequest(`{"recap":{"start":0,"end":45}}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected 1 upsert, got %d", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].RecapEnd == nil || *writer.upserts[0].RecapEnd != 45 {
|
||||
t.Errorf("recap end = %v, want 45", writer.upserts[0].RecapEnd)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerItemAccess map[string]error
|
||||
|
||||
func (f fakeMarkerItemAccess) EnsureAccessible(_ context.Context, contentID string, _ catalog.AccessFilter) error {
|
||||
return f[contentID]
|
||||
}
|
||||
|
||||
type fakeMarkerEpisodeLookup map[string]*models.Episode
|
||||
|
||||
func (f fakeMarkerEpisodeLookup) GetByID(_ context.Context, contentID string) (*models.Episode, error) {
|
||||
return f[contentID], nil
|
||||
}
|
||||
|
||||
type fakeMarkerUsers map[int]*models.User
|
||||
|
||||
func (f fakeMarkerUsers) GetByID(_ context.Context, id int) (*models.User, error) {
|
||||
return f[id], nil
|
||||
}
|
||||
|
||||
func TestGetItemMarkersUsesFirstAuthorizedFile(t *testing.T) {
|
||||
denied := &models.MediaFile{ID: 8, EpisodeID: "episode-denied", Duration: 1800}
|
||||
allowed := &models.MediaFile{ID: 9, EpisodeID: "episode-allowed", Duration: 1800}
|
||||
files := fakeMarkerFiles{
|
||||
byID: map[int]*models.MediaFile{
|
||||
8: denied,
|
||||
9: allowed,
|
||||
},
|
||||
episodeFiles: []*models.MediaFile{denied, allowed},
|
||||
}
|
||||
h := NewMarkersHandler(files, nil, nil, nil, nil, nil)
|
||||
h.Authorizer = &MediaFileAuthorizer{
|
||||
FileResolver: files,
|
||||
ItemAccess: fakeMarkerItemAccess{
|
||||
"series-denied": catalog.ErrItemNotFound,
|
||||
},
|
||||
EpisodeLookup: fakeMarkerEpisodeLookup{
|
||||
"episode-denied": {ContentID: "episode-denied", SeriesID: "series-denied"},
|
||||
"episode-allowed": {ContentID: "episode-allowed", SeriesID: "series-allowed"},
|
||||
},
|
||||
}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/items/episode-1", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("id", "episode-1")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetItemMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body fileMarkersResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if body.FileID != allowed.ID {
|
||||
t.Fatalf("file_id = %d, want first authorized file %d", body.FileID, allowed.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetFileMarkersDoesNotRequireMarkerEditPermission(t *testing.T) {
|
||||
h := newMarkersHandler(nil)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/markers/files/5", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsUserWithoutMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: nil},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d, want 403; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("permission failure wrote markers: %+v", writer.upserts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersAllowsUserWithMarkerEditPermission(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
h.Users = fakeMarkerUsers{
|
||||
7: &models.User{ID: 7, Role: "user", Enabled: true, Permissions: []string{"marker_edit"}},
|
||||
}
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), &auth.Claims{UserID: 7, Role: "user", TokenType: auth.TokenTypeAccess}))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("expected marker write, got %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersPassesAuditContextToWriter(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
apiKeyID := int64(99)
|
||||
req := markerPutRequest(`{"intro":{"start":0,"end":60}}`)
|
||||
req.Header.Set("User-Agent", "marker-test-agent")
|
||||
ctx := clientip.SetContext(req.Context(), "203.0.113.10")
|
||||
ctx = apimw.SetClaims(ctx, &auth.Claims{
|
||||
UserID: 7,
|
||||
Role: "admin",
|
||||
TokenType: auth.TokenTypeAPIKey,
|
||||
APIKeyID: apiKeyID,
|
||||
})
|
||||
req = req.WithContext(ctx)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if len(writer.audits) != 1 {
|
||||
t.Fatalf("audit contexts = %d, want 1", len(writer.audits))
|
||||
}
|
||||
audit := writer.audits[0]
|
||||
if audit.UserID == nil || *audit.UserID != 7 {
|
||||
t.Fatalf("audit user id = %v, want 7", audit.UserID)
|
||||
}
|
||||
if audit.APIKeyID == nil || *audit.APIKeyID != apiKeyID {
|
||||
t.Fatalf("audit api key id = %v, want %d", audit.APIKeyID, apiKeyID)
|
||||
}
|
||||
if audit.ClientIP != "203.0.113.10" || audit.UserAgent != "marker-test-agent" {
|
||||
t.Fatalf("audit request metadata = ip %q ua %q", audit.ClientIP, audit.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearsOnNull(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("expected no upsert for a null clear, got %d", len(writer.upserts))
|
||||
}
|
||||
if len(writer.clears) != 1 || len(writer.clears[0]) != 1 || writer.clears[0][0] != "credits" {
|
||||
t.Errorf("clears = %v, want [[credits]]", writer.clears)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersRejectsInvalidRange(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":60,"end":10}}`))
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Errorf("invalid range must not write, got %d upserts", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearFileSegmentRejectsUnknown(t *testing.T) {
|
||||
writer := &fakeMarkerWriter{}
|
||||
h := newMarkersHandler(writer)
|
||||
|
||||
req := httptest.NewRequest(http.MethodDelete, "/markers/files/5/bogus", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
rctx.URLParams.Add("segment", "bogus")
|
||||
req = withMarkerAdminClaims(req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleClearFileSegment(rec, req)
|
||||
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status = %d, want 400", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeMarkerContributor struct{ outcomes []markers.ContributionOutcome }
|
||||
|
||||
func (f fakeMarkerContributor) ContributeFile(context.Context, *models.MediaFile, markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
return f.outcomes, nil
|
||||
}
|
||||
|
||||
type captureMarkerContributor struct {
|
||||
opts markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (c *captureMarkerContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
c.opts = opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// signalContributor reports every ContributeFile call on a channel so a test
|
||||
// can wait for the detached background contribution kicked off by a save.
|
||||
type signalContributor struct {
|
||||
called chan markers.ContributeOptions
|
||||
}
|
||||
|
||||
func (s signalContributor) ContributeFile(_ context.Context, _ *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
s.called <- opts
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func TestSetFileMarkersTriggersBackgroundContribution(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"intro":{"start":0,"end":60}}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
select {
|
||||
case opts := <-called:
|
||||
if len(opts.Segments) != 1 || opts.Segments[0] != markers.MarkerKindIntro {
|
||||
t.Fatalf("contribute segments = %v, want [intro]", opts.Segments)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("expected a background contribution for the saved intro segment")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetFileMarkersClearOnlyDoesNotContribute(t *testing.T) {
|
||||
called := make(chan markers.ContributeOptions, 1)
|
||||
files := fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}}
|
||||
h := NewMarkersHandler(files, &fakeMarkerWriter{}, signalContributor{called: called}, nil, nil, nil)
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleSetFileMarkers(rec, markerPutRequest(`{"credits":null}`))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-called:
|
||||
t.Fatal("a clear-only save must not trigger contribution")
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
type fakeContributionLister struct{ rows []markers.ContributionRow }
|
||||
|
||||
func (f fakeContributionLister) ListByFile(context.Context, int) ([]markers.ContributionRow, error) {
|
||||
return f.rows, nil
|
||||
}
|
||||
|
||||
func TestContributeFileUsesSnakeCaseResponse(t *testing.T) {
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
fakeMarkerContributor{outcomes: []markers.ContributionOutcome{{
|
||||
Provider: "introdb", Segment: markers.MarkerKindCredits, Status: markers.OutcomeStatusRateLimited,
|
||||
Reason: "usage limited", RetryAfter: 30 * time.Second,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{}`))
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Outcomes []map[string]any `json:"outcomes"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(body.Outcomes) != 1 || body.Outcomes[0]["segment"] != "credits" {
|
||||
t.Fatalf("outcomes = %+v, want credits segment name", body.Outcomes)
|
||||
}
|
||||
if _, ok := body.Outcomes[0]["Segment"]; ok {
|
||||
t.Fatalf("response leaked internal Segment field: %s", rec.Body.String())
|
||||
}
|
||||
if body.Outcomes[0]["retry_after_seconds"] != float64(30) {
|
||||
t.Fatalf("retry_after_seconds = %v, want 30", body.Outcomes[0]["retry_after_seconds"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeFileDecodesUnknownLengthBody(t *testing.T) {
|
||||
contributor := &captureMarkerContributor{}
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
contributor,
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodPost, "/admin/files/5/contribute", strings.NewReader(`{"provider":"introdb","segments":["credits"]}`))
|
||||
req.ContentLength = -1
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleContributeFile(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if contributor.opts.Provider != "introdb" {
|
||||
t.Fatalf("provider = %q, want introdb", contributor.opts.Provider)
|
||||
}
|
||||
if len(contributor.opts.Segments) != 1 || contributor.opts.Segments[0] != markers.MarkerKindCredits {
|
||||
t.Fatalf("segments = %+v, want [credits]", contributor.opts.Segments)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListFileContributionsUsesSnakeCaseResponse(t *testing.T) {
|
||||
now := time.Unix(100, 0).UTC()
|
||||
h := NewMarkersHandler(
|
||||
fakeMarkerFiles{file: &models.MediaFile{ID: 5, Duration: 1800}},
|
||||
nil,
|
||||
nil,
|
||||
fakeContributionLister{rows: []markers.ContributionRow{{
|
||||
ID: "row1", MediaFileID: 5, Provider: "introdb", SegmentKind: "intro",
|
||||
Source: "manual", ContentHash: "hash", Status: "pending", SubmittedAt: now, UpdatedAt: now,
|
||||
}}},
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
req := httptest.NewRequest(http.MethodGet, "/admin/files/5/contributions", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("fileId", "5")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListFileContributions(rec, req)
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `"media_file_id":5`) || strings.Contains(body, "MediaFileID") {
|
||||
t.Fatalf("unexpected contribution response shape: %s", body)
|
||||
}
|
||||
}
|
||||
@@ -2257,6 +2257,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2270,6 +2271,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2284,6 +2286,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"session", sessionID,
|
||||
"playback_session_id", sessionID,
|
||||
@@ -2295,7 +2298,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
// active encode range), either restart at the exact manifest-derived
|
||||
// timeline position or return 404 for copy-mode segments outside the
|
||||
// current manifest window.
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
|
||||
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
|
||||
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
|
||||
slog.Error("resolve transcode seek target", "error", seekErr, "segment", segmentName, "session", sessionID, "playback_session_id", sessionID)
|
||||
@@ -2310,6 +2313,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
"start_segment_number", decision.Progress.StartSegmentNumber,
|
||||
"last_produced_age_ms", lastProducedAgeMS,
|
||||
"wait_timeout_ms", decision.WaitTimeout.Milliseconds(),
|
||||
"restart_on_timeout", decision.RestartOnTimeout,
|
||||
"reason", decision.Reason,
|
||||
"seek_seconds", seekSeconds,
|
||||
"session", sessionID,
|
||||
|
||||
@@ -272,7 +272,7 @@ func (h *PlaybackHandler) fetchOnlineMarkersForPlayback(ctx context.Context, fil
|
||||
EpisodeNumber: ids.EpisodeNumber,
|
||||
Duration: time.Duration(file.Duration) * time.Second,
|
||||
}
|
||||
result, ok, err := h.MarkerRegistry.FetchFirstHit(ctx, req)
|
||||
result, ok, err := h.MarkerRegistry.FetchMerged(ctx, req)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -373,17 +373,31 @@ func hasLocalDetectionMarkers(file *models.MediaFile) bool {
|
||||
// each other.
|
||||
func markerUpdateFromPayload(p markers.MarkerUpdatePayload) scanner.MarkerUpdate {
|
||||
return scanner.MarkerUpdate{
|
||||
IntroStart: p.IntroStart,
|
||||
IntroEnd: p.IntroEnd,
|
||||
CreditsStart: p.CreditsStart,
|
||||
CreditsEnd: p.CreditsEnd,
|
||||
RecapStart: p.RecapStart,
|
||||
RecapEnd: p.RecapEnd,
|
||||
PreviewStart: p.PreviewStart,
|
||||
PreviewEnd: p.PreviewEnd,
|
||||
MarkersSource: p.Source,
|
||||
MarkersProvider: p.Provider,
|
||||
MarkersConfidence: p.Confidence,
|
||||
MarkersAlgorithm: p.Algorithm,
|
||||
IntroStart: p.Intro.Start,
|
||||
IntroEnd: p.Intro.End,
|
||||
CreditsStart: p.Credits.Start,
|
||||
CreditsEnd: p.Credits.End,
|
||||
RecapStart: p.Recap.Start,
|
||||
RecapEnd: p.Recap.End,
|
||||
PreviewStart: p.Preview.Start,
|
||||
PreviewEnd: p.Preview.End,
|
||||
MarkersSource: p.SummarySource(),
|
||||
MarkersConfidence: p.SummaryConfidence(),
|
||||
IntroProvenance: segmentProvenance(p.Intro),
|
||||
CreditsProvenance: segmentProvenance(p.Credits),
|
||||
RecapProvenance: segmentProvenance(p.Recap),
|
||||
PreviewProvenance: segmentProvenance(p.Preview),
|
||||
}
|
||||
}
|
||||
|
||||
func segmentProvenance(s markers.SegmentPayload) *scanner.SegmentProvenance {
|
||||
if !s.Present() {
|
||||
return nil
|
||||
}
|
||||
return &scanner.SegmentProvenance{
|
||||
Source: s.Source,
|
||||
Provider: s.Provider,
|
||||
Confidence: s.Confidence,
|
||||
Algorithm: s.Algorithm,
|
||||
}
|
||||
}
|
||||
|
||||
+70
-1
@@ -123,6 +123,9 @@ type Dependencies struct {
|
||||
IntroAnalyzer *intromarkers.Analyzer
|
||||
MarkerRegistry *markers.Registry
|
||||
MarkerResolver markers.ExternalIDResolver
|
||||
MarkerProviderConfig *markers.ProviderConfigStore
|
||||
MarkerContributionStore *markers.ContributionStore
|
||||
MarkerContributionService *markers.ContributionService
|
||||
WatchProviderService handlers.WatchProviderService
|
||||
PluginService *plugins.Service
|
||||
PluginHTTPProxy *plugins.HTTPProxy
|
||||
@@ -785,6 +788,42 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
}
|
||||
|
||||
var markersHandler *handlers.MarkersHandler
|
||||
if deps.FileRepo != nil {
|
||||
var notifier handlers.PlaybackMarkerUpdateNotifier
|
||||
if playbackHandler != nil {
|
||||
notifier = playbackHandler.MarkerUpdateNotifier
|
||||
}
|
||||
var contributor handlers.MarkerContributor
|
||||
if deps.MarkerContributionService != nil {
|
||||
contributor = deps.MarkerContributionService
|
||||
}
|
||||
var contributions handlers.MarkerContributionLister
|
||||
if deps.MarkerContributionStore != nil {
|
||||
contributions = deps.MarkerContributionStore
|
||||
}
|
||||
markersHandler = handlers.NewMarkersHandler(
|
||||
deps.FileRepo, deps.FileRepo, contributor, contributions, notifier, slog.Default(),
|
||||
)
|
||||
markersHandler.BaseContext = deps.AppContext
|
||||
markersHandler.AuditHistory = deps.FileRepo
|
||||
markersHandler.Users = userRepo
|
||||
if itemRepo != nil {
|
||||
markersHandler.Authorizer = &handlers.MediaFileAuthorizer{
|
||||
FileResolver: deps.FileRepo,
|
||||
ItemAccess: itemRepo,
|
||||
EpisodeLookup: episodeRepo,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var adminMarkerProvidersHandler *handlers.AdminMarkerProvidersHandler
|
||||
if deps.MarkerRegistry != nil && deps.MarkerProviderConfig != nil {
|
||||
adminMarkerProvidersHandler = handlers.NewAdminMarkerProvidersHandler(
|
||||
deps.MarkerRegistry, deps.MarkerProviderConfig, deps.EventBus, slog.Default(),
|
||||
)
|
||||
}
|
||||
|
||||
// Admin subtitle config handler only needs the DB repo — no S3 required.
|
||||
var adminSubtitleHandler *handlers.AdminSubtitleHandler
|
||||
var subtitleManager *subtitles.Manager
|
||||
@@ -1320,6 +1359,21 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Get("/events/ws", eventsHandler.HandleWebSocket)
|
||||
}
|
||||
|
||||
// Marker read/write/clear for any authenticated viewer: users
|
||||
// fix and create intro/recap/credits/preview markers from the
|
||||
// player. Writes are stamped source="manual" and contributed to
|
||||
// enabled providers in the background. Contribution + provider
|
||||
// config stay admin-only (see the /admin group below).
|
||||
if markersHandler != nil {
|
||||
r.Route("/markers", func(r chi.Router) {
|
||||
r.Get("/items/{id}", markersHandler.HandleGetItemMarkers)
|
||||
r.Put("/items/{id}", markersHandler.HandleSetItemMarkers)
|
||||
r.Get("/files/{fileId}", markersHandler.HandleGetFileMarkers)
|
||||
r.Put("/files/{fileId}", markersHandler.HandleSetFileMarkers)
|
||||
r.Delete("/files/{fileId}/{segment}", markersHandler.HandleClearFileSegment)
|
||||
})
|
||||
}
|
||||
|
||||
// Library management routes (admin-only).
|
||||
if libraryHandler != nil {
|
||||
r.Group(func(r chi.Router) {
|
||||
@@ -1760,8 +1814,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
audiobookHandler := &handlers.AudiobookHandler{
|
||||
Items: itemRepo,
|
||||
Files: deps.FileRepo,
|
||||
StoreProvider: deps.UserStoreProvider,
|
||||
Detail: detailSvc,
|
||||
StoreProvider: deps.UserStoreProvider,
|
||||
Recs: recsRepoForStale,
|
||||
}
|
||||
r.Get("/audiobooks", audiobookHandler.HandleListAudiobooks)
|
||||
@@ -1879,6 +1933,21 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Post("/items/{id}/refresh-markers", adminIntroHandler.HandleRefreshEpisodeMarkers)
|
||||
r.Post("/items/{id}/redetect-intro", adminIntroHandler.HandleRedetectEpisodeIntro)
|
||||
}
|
||||
if markersHandler != nil {
|
||||
// Marker read/write/clear live on the authenticated
|
||||
// /markers routes; writes require marker_edit.
|
||||
// Contribution and audit history stay admin operations.
|
||||
r.Post("/files/{fileId}/contribute", markersHandler.HandleContributeFile)
|
||||
r.Get("/files/{fileId}/contributions", markersHandler.HandleListFileContributions)
|
||||
r.Get("/markers/history", markersHandler.HandleListMarkerHistory)
|
||||
r.Get("/markers/files/{fileId}/history", markersHandler.HandleListFileMarkerHistory)
|
||||
r.Get("/markers/items/{id}/history", markersHandler.HandleListItemMarkerHistory)
|
||||
}
|
||||
if adminMarkerProvidersHandler != nil {
|
||||
r.Get("/markers/providers", adminMarkerProvidersHandler.HandleListProviders)
|
||||
r.Put("/markers/providers/{provider}", adminMarkerProvidersHandler.HandleUpdateProvider)
|
||||
r.Post("/markers/providers/{provider}/validate", adminMarkerProvidersHandler.HandleValidateProvider)
|
||||
}
|
||||
if peopleHandler != nil {
|
||||
r.Post("/people/{id}/refresh", peopleHandler.HandleAdminRefreshPerson)
|
||||
r.Patch("/people/{id}", peopleHandler.HandleAdminUpdatePerson)
|
||||
|
||||
@@ -2,6 +2,8 @@ package audiobooks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
@@ -14,9 +16,9 @@ import (
|
||||
)
|
||||
|
||||
// ABSSessionStore implements abs.TokenStore on the abs_sessions table
|
||||
// (migration 139). Each JTI is stored as the `token` column; revocation sets
|
||||
// `revoked_at`. The table stores integer user_id, so we parse the string
|
||||
// UserID from ABSToken back to int.
|
||||
// (migration 147). Each JTI is stored as a SHA-256 `token_hash`; revocation
|
||||
// sets `revoked_at`. The table stores integer user_id, so we parse the
|
||||
// string UserID from ABSToken back to int.
|
||||
type ABSSessionStore struct {
|
||||
Pool *pgxpool.Pool
|
||||
}
|
||||
@@ -31,12 +33,12 @@ func (s *ABSSessionStore) InsertToken(ctx context.Context, tok abs.ABSToken) err
|
||||
}
|
||||
_, err = s.Pool.Exec(ctx, `
|
||||
INSERT INTO abs_sessions
|
||||
(user_id, profile_id, token, token_type, expires_at, device_id, device_name, client_name, client_version, last_seen_at)
|
||||
(user_id, profile_id, token_hash, token_type, expires_at, device_id, device_name, client_name, client_version, last_seen_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, now())
|
||||
ON CONFLICT (token) DO NOTHING`,
|
||||
ON CONFLICT (token_hash) DO NOTHING`,
|
||||
uid,
|
||||
tok.ProfileID,
|
||||
tok.JTI,
|
||||
absTokenHash(tok.JTI),
|
||||
tok.Type,
|
||||
tok.ExpiresAt,
|
||||
tok.JTI, // device_id: use the JTI as a stable device key
|
||||
@@ -50,7 +52,7 @@ func (s *ABSSessionStore) InsertToken(ctx context.Context, tok abs.ABSToken) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTokenByJTI looks up an abs_sessions row by its JTI (stored in `token`).
|
||||
// GetTokenByJTI looks up an abs_sessions row by the SHA-256 hash of its JTI.
|
||||
// Returns abs.ErrNotFound when the row doesn't exist.
|
||||
func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.ABSToken, error) {
|
||||
var (
|
||||
@@ -60,17 +62,18 @@ func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.AB
|
||||
tok abs.ABSToken
|
||||
)
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
SELECT user_id, profile_id, token, token_type, expires_at, revoked_at
|
||||
SELECT user_id, profile_id, token_type, expires_at, revoked_at
|
||||
FROM abs_sessions
|
||||
WHERE token = $1`, jti)
|
||||
err := row.Scan(&uid, &profileID, &tok.JTI, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
WHERE token_hash = $1`, absTokenHash(jti))
|
||||
err := row.Scan(&uid, &profileID, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return abs.ABSToken{}, abs.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return abs.ABSToken{}, fmt.Errorf("abs_session_store: get token: %w", err)
|
||||
}
|
||||
tok.ID = tok.JTI
|
||||
tok.JTI = jti
|
||||
tok.ID = jti
|
||||
tok.UserID = strconv.Itoa(uid)
|
||||
tok.ProfileID = profileID
|
||||
if expiresAt != nil {
|
||||
@@ -84,7 +87,7 @@ func (s *ABSSessionStore) GetTokenByJTI(ctx context.Context, jti string) (abs.AB
|
||||
func (s *ABSSessionStore) RevokeTokenByJTI(ctx context.Context, jti string) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE abs_sessions SET revoked_at = now()
|
||||
WHERE token = $1 AND revoked_at IS NULL`, jti)
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL`, absTokenHash(jti))
|
||||
if err != nil {
|
||||
return fmt.Errorf("abs_session_store: revoke token: %w", err)
|
||||
}
|
||||
@@ -101,16 +104,17 @@ func (s *ABSSessionStore) RevokeTokenIfActive(ctx context.Context, jti string) (
|
||||
row := s.Pool.QueryRow(ctx, `
|
||||
UPDATE abs_sessions
|
||||
SET revoked_at = now()
|
||||
WHERE token = $1 AND revoked_at IS NULL
|
||||
RETURNING user_id, profile_id, token, token_type, expires_at, revoked_at`, jti)
|
||||
err := row.Scan(&uid, &profileID, &tok.JTI, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL
|
||||
RETURNING user_id, profile_id, token_type, expires_at, revoked_at`, absTokenHash(jti))
|
||||
err := row.Scan(&uid, &profileID, &tok.Type, &expiresAt, &tok.RevokedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return abs.ABSToken{}, abs.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return abs.ABSToken{}, fmt.Errorf("abs_session_store: revoke active token: %w", err)
|
||||
}
|
||||
tok.ID = tok.JTI
|
||||
tok.JTI = jti
|
||||
tok.ID = jti
|
||||
tok.UserID = strconv.Itoa(uid)
|
||||
tok.ProfileID = profileID
|
||||
if expiresAt != nil {
|
||||
@@ -141,9 +145,14 @@ func (s *ABSSessionStore) RevokeTokensForPrincipal(ctx context.Context, userID,
|
||||
func (s *ABSSessionStore) TouchToken(ctx context.Context, jti string) error {
|
||||
_, err := s.Pool.Exec(ctx, `
|
||||
UPDATE abs_sessions SET last_seen_at = now()
|
||||
WHERE token = $1`, jti)
|
||||
WHERE token_hash = $1`, absTokenHash(jti))
|
||||
if err != nil {
|
||||
return fmt.Errorf("abs_session_store: touch token: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func absTokenHash(token string) string {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
@@ -181,9 +181,9 @@ func buildABSRecommender(deps ABSHandlerDeps) abs.Recommender {
|
||||
return &ABSRecommender{Pool: deps.Pool, Recs: deps.Recs}
|
||||
}
|
||||
|
||||
// Enabled reports whether the audiobooks feature flag (set by migration
|
||||
// 142 and toggled by operators) is currently true. Any value other than
|
||||
// the literal string "true" reads as false; this matches how silo
|
||||
// Enabled reports whether the audiobooks feature flag (set by
|
||||
// 160_audiobooks_feature_flag and toggled by operators) is currently true.
|
||||
// Any value other than the literal string "true" reads as false; this matches how silo
|
||||
// treats other boolean server_settings rows.
|
||||
func (s *Service) Enabled(ctx context.Context) (bool, error) {
|
||||
if s == nil || s.settings == nil {
|
||||
|
||||
@@ -52,3 +52,25 @@ func TestServiceEnabledPropagatesError(t *testing.T) {
|
||||
t.Fatalf("Enabled error = %v, want %v wrapped", err, wantErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledNilReceiverReturnsFalse(t *testing.T) {
|
||||
var svc *Service
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("Enabled = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceEnabledNilSettingsReturnsFalse(t *testing.T) {
|
||||
svc := New(nil)
|
||||
got, err := svc.Enabled(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Enabled returned error: %v", err)
|
||||
}
|
||||
if got {
|
||||
t.Fatal("Enabled = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,9 +10,13 @@ import (
|
||||
|
||||
type Permission string
|
||||
|
||||
const PermissionMetadataCuration Permission = "metadata_curation"
|
||||
const (
|
||||
PermissionMarkerEdit Permission = "marker_edit"
|
||||
PermissionMetadataCuration Permission = "metadata_curation"
|
||||
)
|
||||
|
||||
var assignablePermissions = map[Permission]struct{}{
|
||||
PermissionMarkerEdit: {},
|
||||
PermissionMetadataCuration: {},
|
||||
}
|
||||
|
||||
@@ -56,6 +60,10 @@ func NormalizePermissions(values []string) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func DefaultUserPermissions() []string {
|
||||
return []string{string(PermissionMarkerEdit)}
|
||||
}
|
||||
|
||||
func HasAssignedPermission(user *models.User, permission Permission) bool {
|
||||
if user == nil {
|
||||
return false
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
|
||||
func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
got, err := NormalizePermissions([]string{
|
||||
"marker_edit",
|
||||
" metadata_curation ",
|
||||
"metadata_curation",
|
||||
"",
|
||||
@@ -16,7 +17,7 @@ func TestNormalizePermissions_DeduplicatesAndSorts(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("NormalizePermissions returned error: %v", err)
|
||||
}
|
||||
want := []string{"metadata_curation"}
|
||||
want := []string{"marker_edit", "metadata_curation"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
@@ -28,11 +29,14 @@ func TestNormalizePermissions_RejectsUnknownPermission(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_AdminImpliesMetadataCuration(t *testing.T) {
|
||||
func TestHasEffectivePermission_AdminImpliesAssignablePermissions(t *testing.T) {
|
||||
user := &models.User{Role: "admin", Enabled: true}
|
||||
if !HasEffectivePermission(user, PermissionMetadataCuration) {
|
||||
t.Fatal("admin should have metadata curation")
|
||||
}
|
||||
if !HasEffectivePermission(user, PermissionMarkerEdit) {
|
||||
t.Fatal("admin should have marker edit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
@@ -45,3 +49,11 @@ func TestHasEffectivePermission_UserRequiresAssignedPermission(t *testing.T) {
|
||||
t.Fatal("assigned user should have metadata curation")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultUserPermissionsIncludesMarkerEditOnly(t *testing.T) {
|
||||
got := DefaultUserPermissions()
|
||||
want := []string{"marker_edit"}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("default permissions = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,11 @@ func (r *UserRepository) Create(ctx context.Context, input models.CreateUserInpu
|
||||
localPasswordLoginEnabled = *input.LocalPasswordLoginEnabled
|
||||
}
|
||||
|
||||
permissions, err := NormalizePermissions(input.Permissions)
|
||||
permissions := append([]string(nil), input.Permissions...)
|
||||
if input.Permissions == nil && input.Role != "admin" {
|
||||
permissions = DefaultUserPermissions()
|
||||
}
|
||||
permissions, err = NormalizePermissions(permissions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Vendored
+12
-11
@@ -40,17 +40,18 @@ const (
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const (
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
EventScanComplete = "scan_complete"
|
||||
EventMetadataUpdated = "metadata_updated"
|
||||
EventAdminStatsInvalidated = "admin_stats_invalidated"
|
||||
EventPlaybackSessionsChanged = "playback_sessions_changed"
|
||||
EventUserDisabled = "user_disabled"
|
||||
EventUserDeleted = "user_deleted"
|
||||
EventSettingsChanged = "settings_changed"
|
||||
EventMarkerProviderConfigChanged = "marker_provider_config_changed"
|
||||
EventNodePoolChanged = "node_pool_changed"
|
||||
EventOperationalLogAppended = "operational_log_appended"
|
||||
EventAuditLogAppended = "audit_log_appended"
|
||||
EventEventsNotification = "events_notification"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -634,7 +634,7 @@ func (s *Service) exportFolders(ctx context.Context, libraryIDs []int) ([]Librar
|
||||
|
||||
func (s *Service) schemaVersion(ctx context.Context) (int, error) {
|
||||
var version int
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version), 0) FROM schema_versions").Scan(&version); err != nil {
|
||||
if err := s.pool.QueryRow(ctx, "SELECT COALESCE(MAX(version_id), 0) FROM public.goose_db_version").Scan(&version); err != nil {
|
||||
return 0, fmt.Errorf("loading schema version: %w", err)
|
||||
}
|
||||
return version, nil
|
||||
|
||||
+210
-199
@@ -2,233 +2,244 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/pressly/goose/v3"
|
||||
"github.com/pressly/goose/v3/lock"
|
||||
)
|
||||
|
||||
// MigrationFile represents a parsed migration file with its version number,
|
||||
// filename, and SQL content.
|
||||
type MigrationFile struct {
|
||||
Version int
|
||||
Filename string
|
||||
SQL string
|
||||
}
|
||||
const (
|
||||
schemaMigrationsLockID int64 = 8_034_219_741
|
||||
gooseVersionTable = "public.goose_db_version"
|
||||
)
|
||||
|
||||
const schemaMigrationsLockID int64 = 8_034_219_741
|
||||
|
||||
type migrationDB interface {
|
||||
Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error)
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error)
|
||||
}
|
||||
|
||||
// RunMigrations applies all pending .up.sql migrations from the given embed.FS.
|
||||
// It creates a schema_versions table to track which migrations have been applied,
|
||||
// parses migration filenames to extract version numbers, and applies each
|
||||
// pending migration within its own transaction.
|
||||
//
|
||||
// The dir parameter specifies the subdirectory within the fs.FS that contains
|
||||
// the migration files.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) (err error) {
|
||||
conn, err := pool.Acquire(ctx)
|
||||
// RunMigrations applies all pending Goose migrations from fsys/dir.
|
||||
func RunMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) error {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("acquiring migration connection: %w", err)
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
defer provider.Close()
|
||||
|
||||
if err := lockMigrations(ctx, conn); err != nil {
|
||||
return fmt.Errorf("acquiring migration lock: %w", err)
|
||||
if _, err := provider.Up(ctx); err != nil {
|
||||
return fmt.Errorf("running goose migrations: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if unlockErr := unlockMigrations(conn); unlockErr != nil {
|
||||
if err != nil {
|
||||
err = fmt.Errorf("%w; releasing migration lock: %v", err, unlockErr)
|
||||
return
|
||||
}
|
||||
err = fmt.Errorf("releasing migration lock: %w", unlockErr)
|
||||
}
|
||||
}()
|
||||
|
||||
if err := ensureSchemaVersionsTable(ctx, conn); err != nil {
|
||||
return fmt.Errorf("ensuring schema_versions table: %w", err)
|
||||
}
|
||||
|
||||
migrations, err := loadMigrations(fsys, dir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading migrations: %w", err)
|
||||
}
|
||||
|
||||
applied, err := getAppliedVersions(ctx, conn)
|
||||
if err != nil {
|
||||
return fmt.Errorf("getting applied versions: %w", err)
|
||||
}
|
||||
|
||||
for _, m := range migrations {
|
||||
if applied[m.Version] {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := applyMigration(ctx, conn, m); err != nil {
|
||||
return fmt.Errorf("applying migration %s (version %d): %w", m.Filename, m.Version, err)
|
||||
}
|
||||
applied[m.Version] = true
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockMigrations(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_lock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
// MigrationStatus describes a migration source and whether Goose has applied it.
|
||||
type MigrationStatus struct {
|
||||
Version int64
|
||||
Source string
|
||||
State string
|
||||
AppliedAt time.Time
|
||||
}
|
||||
|
||||
func unlockMigrations(db migrationDB) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := db.Exec(ctx, "SELECT pg_advisory_unlock($1)", schemaMigrationsLockID)
|
||||
return err
|
||||
}
|
||||
|
||||
// ensureSchemaVersionsTable creates the schema_versions table if it does not
|
||||
// already exist.
|
||||
func ensureSchemaVersionsTable(ctx context.Context, db migrationDB) error {
|
||||
_, err := db.Exec(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.schema_versions (
|
||||
version INT PRIMARY KEY,
|
||||
filename TEXT NOT NULL,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
return err
|
||||
}
|
||||
|
||||
// loadMigrations reads all .up.sql files from the specified directory within
|
||||
// the filesystem and returns them sorted by version number.
|
||||
func loadMigrations(fsys fs.FS, dir string) ([]MigrationFile, error) {
|
||||
entries, err := fs.ReadDir(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading migrations directory %q: %w", dir, err)
|
||||
}
|
||||
|
||||
var migrations []MigrationFile
|
||||
seenVersions := make(map[int]string)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
name := entry.Name()
|
||||
if !strings.HasSuffix(name, ".up.sql") {
|
||||
continue
|
||||
}
|
||||
|
||||
version, err := parseVersion(name)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parsing version from %q: %w", name, err)
|
||||
}
|
||||
if existing, ok := seenVersions[version]; ok {
|
||||
return nil, fmt.Errorf("duplicate migration version %d: %s and %s", version, existing, name)
|
||||
}
|
||||
seenVersions[version] = name
|
||||
|
||||
content, err := fs.ReadFile(fsys, filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading migration file %q: %w", name, err)
|
||||
}
|
||||
|
||||
migrations = append(migrations, MigrationFile{
|
||||
Version: version,
|
||||
Filename: name,
|
||||
SQL: string(content),
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(migrations, func(i, j int) bool {
|
||||
return migrations[i].Version < migrations[j].Version
|
||||
})
|
||||
|
||||
return migrations, nil
|
||||
}
|
||||
|
||||
// parseVersion extracts the numeric version prefix from a migration filename.
|
||||
// For example, "001_initial_schema.up.sql" returns 1.
|
||||
func parseVersion(filename string) (int, error) {
|
||||
// Split on the first underscore to get the version prefix.
|
||||
parts := strings.SplitN(filename, "_", 2)
|
||||
if len(parts) < 2 {
|
||||
return 0, fmt.Errorf("migration filename %q must be in format NNN_description.up.sql", filename)
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(parts[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid version number in %q: %w", filename, err)
|
||||
}
|
||||
|
||||
if version <= 0 {
|
||||
return 0, fmt.Errorf("version number must be positive, got %d in %q", version, filename)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
// getAppliedVersions returns a set of migration versions that have already
|
||||
// been applied.
|
||||
func getAppliedVersions(ctx context.Context, db migrationDB) (map[int]bool, error) {
|
||||
rows, err := db.Query(ctx, "SELECT version FROM public.schema_versions")
|
||||
// MigrationStatuses returns Goose status using the same legacy bootstrap and locking path as RunMigrations.
|
||||
func MigrationStatuses(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, dir string) ([]MigrationStatus, error) {
|
||||
provider, err := newMigrationProvider(pool, fsys, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer provider.Close()
|
||||
|
||||
gooseStatuses, err := provider.Status(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading goose migration status: %w", err)
|
||||
}
|
||||
|
||||
statuses := make([]MigrationStatus, 0, len(gooseStatuses))
|
||||
for _, status := range gooseStatuses {
|
||||
var version int64
|
||||
var source string
|
||||
if status.Source != nil {
|
||||
version = status.Source.Version
|
||||
source = status.Source.Path
|
||||
}
|
||||
statuses = append(statuses, MigrationStatus{
|
||||
Version: version,
|
||||
Source: source,
|
||||
State: string(status.State),
|
||||
AppliedAt: status.AppliedAt,
|
||||
})
|
||||
}
|
||||
return statuses, nil
|
||||
}
|
||||
|
||||
func newMigrationProvider(pool *pgxpool.Pool, fsys fs.FS, dir string) (*goose.Provider, error) {
|
||||
migrationFS, err := migrationSubFS(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sqlDB := stdlib.OpenDBFromPool(pool)
|
||||
|
||||
locker, err := lock.NewPostgresSessionLocker(lock.WithLockID(schemaMigrationsLockID))
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration lock: %w", err)
|
||||
}
|
||||
|
||||
provider, err := goose.NewProvider(
|
||||
goose.DialectPostgres,
|
||||
sqlDB,
|
||||
migrationFS,
|
||||
goose.WithTableName(gooseVersionTable),
|
||||
goose.WithAllowOutofOrder(true),
|
||||
goose.WithSessionLocker(&legacyBootstrapLocker{delegate: locker}),
|
||||
)
|
||||
if err != nil {
|
||||
_ = sqlDB.Close()
|
||||
return nil, fmt.Errorf("creating goose migration provider: %w", err)
|
||||
}
|
||||
return provider, nil
|
||||
}
|
||||
|
||||
func migrationSubFS(fsys fs.FS, dir string) (fs.FS, error) {
|
||||
if dir == "" || dir == "." {
|
||||
return fsys, nil
|
||||
}
|
||||
|
||||
sub, err := fs.Sub(fsys, dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("opening migration directory %q: %w", dir, err)
|
||||
}
|
||||
return sub, nil
|
||||
}
|
||||
|
||||
type legacyBootstrapLocker struct {
|
||||
delegate lock.SessionLocker
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionLock(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := l.delegate.SessionLock(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := bootstrapLegacyGooseVersions(ctx, conn); err != nil {
|
||||
_ = l.delegate.SessionUnlock(context.WithoutCancel(ctx), conn)
|
||||
return fmt.Errorf("bootstrapping goose migration versions: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (l *legacyBootstrapLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error {
|
||||
return l.delegate.SessionUnlock(ctx, conn)
|
||||
}
|
||||
|
||||
func bootstrapLegacyGooseVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
if err := ensureGooseVersionTable(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ensureGooseZeroVersion(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasLegacyVersions, err := tableExists(ctx, conn, "public.schema_versions")
|
||||
if err != nil {
|
||||
return fmt.Errorf("checking legacy schema_versions table: %w", err)
|
||||
}
|
||||
if !hasLegacyVersions {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := copyLegacyVersions(ctx, conn); err != nil {
|
||||
return err
|
||||
}
|
||||
return verifyLegacyVersions(ctx, conn)
|
||||
}
|
||||
|
||||
func ensureGooseVersionTable(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
CREATE TABLE IF NOT EXISTS public.goose_db_version (
|
||||
id integer PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY,
|
||||
version_id bigint NOT NULL,
|
||||
is_applied boolean NOT NULL,
|
||||
tstamp timestamp NOT NULL DEFAULT now()
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose version table: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ensureGooseZeroVersion(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied)
|
||||
SELECT 0, true
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version
|
||||
WHERE version_id = 0 AND is_applied
|
||||
)
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ensuring goose zero version: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
_, err := conn.ExecContext(ctx, `
|
||||
INSERT INTO public.goose_db_version (version_id, is_applied, tstamp)
|
||||
SELECT sv.version, true, sv.applied_at::timestamp
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func verifyLegacyVersions(ctx context.Context, conn *sql.Conn) error {
|
||||
rows, err := conn.QueryContext(ctx, `
|
||||
SELECT sv.version
|
||||
FROM public.schema_versions sv
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM public.goose_db_version gv
|
||||
WHERE gv.version_id = sv.version AND gv.is_applied
|
||||
)
|
||||
ORDER BY sv.version
|
||||
LIMIT 10
|
||||
`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("verifying legacy schema_versions rows: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
applied := make(map[int]bool)
|
||||
var missing []int
|
||||
for rows.Next() {
|
||||
var version int
|
||||
if err := rows.Scan(&version); err != nil {
|
||||
return nil, err
|
||||
return fmt.Errorf("scanning missing legacy version: %w", err)
|
||||
}
|
||||
applied[version] = true
|
||||
missing = append(missing, version)
|
||||
}
|
||||
|
||||
return applied, rows.Err()
|
||||
}
|
||||
|
||||
// applyMigration runs a single migration within a transaction and records the
|
||||
// version in schema_versions.
|
||||
func applyMigration(ctx context.Context, db migrationDB, m MigrationFile) error {
|
||||
tx, err := db.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("beginning transaction: %w", err)
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("reading missing legacy versions: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
// Rollback is a no-op if the tx has already been committed.
|
||||
_ = tx.Rollback(ctx)
|
||||
}()
|
||||
|
||||
if _, err := tx.Exec(ctx, m.SQL); err != nil {
|
||||
return fmt.Errorf("executing SQL: %w", err)
|
||||
if len(missing) > 0 {
|
||||
return fmt.Errorf("legacy schema_versions rows missing from goose_db_version: %v", missing)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
"INSERT INTO public.schema_versions (version, filename) VALUES ($1, $2)",
|
||||
m.Version, m.Filename,
|
||||
); err != nil {
|
||||
return fmt.Errorf("recording version: %w", err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return fmt.Errorf("committing transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func tableExists(ctx context.Context, conn *sql.Conn, qualifiedName string) (bool, error) {
|
||||
var exists bool
|
||||
if err := conn.QueryRowContext(ctx, "SELECT to_regclass($1) IS NOT NULL", qualifiedName).Scan(&exists); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
@@ -12,11 +12,12 @@ import (
|
||||
)
|
||||
|
||||
type Analyzer struct {
|
||||
repo introRepository
|
||||
extractor fingerprintExtractor
|
||||
refiner boundaryRefiner
|
||||
config Config
|
||||
logger *slog.Logger
|
||||
repo introRepository
|
||||
extractor fingerprintExtractor
|
||||
refiner boundaryRefiner
|
||||
chromaprintRefiner chromaprintStartRefiner
|
||||
config Config
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
type introRepository interface {
|
||||
@@ -43,11 +44,12 @@ func NewAnalyzer(repo *Repository, config Config, logger *slog.Logger) *Analyzer
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &Analyzer{
|
||||
repo: repo,
|
||||
extractor: NewChromaprintExtractor(config),
|
||||
refiner: NewSilenceBoundaryRefiner(config),
|
||||
config: config,
|
||||
logger: logger,
|
||||
repo: repo,
|
||||
extractor: NewChromaprintExtractor(config),
|
||||
refiner: NewSilenceBoundaryRefiner(config),
|
||||
chromaprintRefiner: NewDialogueBoundaryRefiner(config),
|
||||
config: config,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +146,9 @@ func (a *Analyzer) Run(ctx context.Context, progress ProgressFunc) (RunSummary,
|
||||
summary.FingerprintsComputed += groupSummary.FingerprintsComputed
|
||||
summary.FingerprintCacheHits += groupSummary.FingerprintCacheHits
|
||||
summary.ChromaprintMarkersWritten += groupSummary.ChromaprintMarkersWritten
|
||||
summary.DialogueRefinementsAttempted += groupSummary.DialogueRefinementsAttempted
|
||||
summary.DialogueRefinementsApplied += groupSummary.DialogueRefinementsApplied
|
||||
summary.DialogueRefinementErrors += groupSummary.DialogueRefinementErrors
|
||||
summary.GroupsNotFound += groupSummary.GroupsNotFound
|
||||
summary.GroupsSkipped += groupSummary.GroupsSkipped
|
||||
summary.Errors = append(summary.Errors, groupSummary.Errors...)
|
||||
@@ -249,6 +254,9 @@ func (a *Analyzer) AnalyzeEpisode(ctx context.Context, episodeID string) (RunSum
|
||||
summary.FingerprintsComputed += groupSummary.FingerprintsComputed
|
||||
summary.FingerprintCacheHits += groupSummary.FingerprintCacheHits
|
||||
summary.ChromaprintMarkersWritten += groupSummary.ChromaprintMarkersWritten
|
||||
summary.DialogueRefinementsAttempted += groupSummary.DialogueRefinementsAttempted
|
||||
summary.DialogueRefinementsApplied += groupSummary.DialogueRefinementsApplied
|
||||
summary.DialogueRefinementErrors += groupSummary.DialogueRefinementErrors
|
||||
summary.GroupsNotFound += groupSummary.GroupsNotFound
|
||||
summary.GroupsSkipped += groupSummary.GroupsSkipped
|
||||
summary.Errors = append(summary.Errors, groupSummary.Errors...)
|
||||
@@ -277,6 +285,8 @@ type chapterSourceMarker struct {
|
||||
segment Segment
|
||||
}
|
||||
|
||||
const episodeVersionCopyDurationToleranceSeconds = 3.0
|
||||
|
||||
func (a *Analyzer) processChapterCandidates(ctx context.Context, candidates []Candidate, opts chapterProcessingOptions) ([]Candidate, RunSummary) {
|
||||
summary := RunSummary{}
|
||||
remaining := make([]Candidate, 0, len(candidates))
|
||||
@@ -438,7 +448,7 @@ func compatibleEpisodeVersionDuration(source, target Candidate) bool {
|
||||
if diff < 0 {
|
||||
diff = -diff
|
||||
}
|
||||
return diff <= 1.0
|
||||
return diff <= episodeVersionCopyDurationToleranceSeconds
|
||||
}
|
||||
|
||||
func ownDetectionCandidates(candidates []Candidate) []Candidate {
|
||||
@@ -594,6 +604,7 @@ func (a *Analyzer) analyzeGroup(ctx context.Context, group candidateGroup, opts
|
||||
continue
|
||||
}
|
||||
candidate := byFileID[fileID]
|
||||
segment = a.refineChromaprintSegment(ctx, candidate, segment, &summary)
|
||||
applied, patchErr := a.repo.PatchIntroMarker(ctx, IntroMarkerPatch{
|
||||
FileID: fileID,
|
||||
Start: segment.Start,
|
||||
@@ -624,6 +635,24 @@ func (a *Analyzer) analyzeGroup(ctx context.Context, group candidateGroup, opts
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func (a *Analyzer) refineChromaprintSegment(ctx context.Context, candidate Candidate, segment Segment, summary *RunSummary) Segment {
|
||||
if a.chromaprintRefiner == nil || !a.config.normalized().DialogueRefinementEnabled {
|
||||
return segment
|
||||
}
|
||||
summary.DialogueRefinementsAttempted++
|
||||
refined, ok, err := a.chromaprintRefiner.RefineChromaprintStart(ctx, candidate, segment)
|
||||
if err != nil {
|
||||
summary.DialogueRefinementErrors++
|
||||
a.logger.Warn("intro marker dialogue refinement failed", "file_id", candidate.FileID, "path", candidate.FilePath, "error", err)
|
||||
return segment
|
||||
}
|
||||
if ok {
|
||||
summary.DialogueRefinementsApplied++
|
||||
return refined
|
||||
}
|
||||
return segment
|
||||
}
|
||||
|
||||
func candidateFileIDs(candidates []Candidate) map[int]struct{} {
|
||||
fileIDs := make(map[int]struct{}, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
@@ -758,4 +787,7 @@ func mergeRunSummary(dst *RunSummary, src RunSummary) {
|
||||
dst.SilenceRefinementErrors += src.SilenceRefinementErrors
|
||||
dst.EpisodeVersionMarkersCopied += src.EpisodeVersionMarkersCopied
|
||||
dst.SilenceBackfillConsidered += src.SilenceBackfillConsidered
|
||||
dst.DialogueRefinementsAttempted += src.DialogueRefinementsAttempted
|
||||
dst.DialogueRefinementsApplied += src.DialogueRefinementsApplied
|
||||
dst.DialogueRefinementErrors += src.DialogueRefinementErrors
|
||||
}
|
||||
|
||||
@@ -102,6 +102,20 @@ func (f *fakeBoundaryRefiner) RefineChapterEnd(_ context.Context, candidate Cand
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
type fakeChromaprintStartRefiner struct {
|
||||
calls int
|
||||
segments map[int]Segment
|
||||
}
|
||||
|
||||
func (f *fakeChromaprintStartRefiner) RefineChromaprintStart(_ context.Context, candidate Candidate, segment Segment) (Segment, bool, error) {
|
||||
f.calls++
|
||||
refined, ok := f.segments[candidate.FileID]
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
func TestAnalyzeEpisodeNoCandidatesIsNoOp(t *testing.T) {
|
||||
repo := &fakeIntroRepository{episodeCandidates: map[string][]Candidate{}}
|
||||
extractor := &fakeFingerprintExtractor{}
|
||||
@@ -260,7 +274,7 @@ func TestAnalyzeEpisodeCopiesMarkerToCompatibleEpisodeVersion(t *testing.T) {
|
||||
{Index: 2, Title: "Part 1", StartSeconds: 120, EndSeconds: 900},
|
||||
},
|
||||
}
|
||||
target := Candidate{FileID: 11, EpisodeID: "ep1", DurationSeconds: 1200.5}
|
||||
target := Candidate{FileID: 11, EpisodeID: "ep1", DurationSeconds: 1202.5}
|
||||
repo := &fakeIntroRepository{episodeCandidates: map[string][]Candidate{"ep1": []Candidate{source, target}}}
|
||||
analyzer := &Analyzer{repo: repo, extractor: &fakeFingerprintExtractor{}, config: DefaultConfig("ffmpeg")}
|
||||
|
||||
@@ -420,6 +434,65 @@ func TestAnalyzeEpisodeChromaprintOnlyPatchesRequestedEpisode(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeEpisodePersistsRefinedChromaprintSegment(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
target := Candidate{
|
||||
FileID: 1,
|
||||
EpisodeID: "ep1",
|
||||
SeasonID: "season1",
|
||||
MediaFolderID: 7,
|
||||
FileHash: "hash1",
|
||||
FileSize: 100,
|
||||
DurationSeconds: 1200,
|
||||
}
|
||||
sibling := Candidate{
|
||||
FileID: 2,
|
||||
EpisodeID: "ep2",
|
||||
SeasonID: "season1",
|
||||
MediaFolderID: 7,
|
||||
FileHash: "hash2",
|
||||
FileSize: 200,
|
||||
DurationSeconds: 1200,
|
||||
}
|
||||
groupCandidates := []Candidate{target, sibling}
|
||||
group := groupKey(target.MediaFolderID, target.SeasonID, target.AnalysisGroupKey())
|
||||
repo := &fakeIntroRepository{
|
||||
episodeCandidates: map[string][]Candidate{"ep1": []Candidate{target}},
|
||||
groupCandidates: map[string][]Candidate{group: groupCandidates},
|
||||
fingerprints: map[int]*Fingerprint{
|
||||
target.FileID: cachedFingerprint(target, cfg, sharedIntroPoints(1000)),
|
||||
sibling.FileID: cachedFingerprint(sibling, cfg, sharedIntroPoints(5000)),
|
||||
},
|
||||
}
|
||||
refiner := &fakeChromaprintStartRefiner{segments: map[int]Segment{
|
||||
target.FileID: {Start: 12.5, End: 36.5, Confidence: 0.85, Algorithm: ChromaprintDialogueAlgorithm},
|
||||
}}
|
||||
analyzer := &Analyzer{
|
||||
repo: repo,
|
||||
extractor: &fakeFingerprintExtractor{},
|
||||
chromaprintRefiner: refiner,
|
||||
config: cfg,
|
||||
}
|
||||
|
||||
summary, err := analyzer.AnalyzeEpisode(context.Background(), "ep1")
|
||||
if err != nil {
|
||||
t.Fatalf("AnalyzeEpisode returned error: %v", err)
|
||||
}
|
||||
if refiner.calls != 1 {
|
||||
t.Fatalf("expected one refinement call for requested file, got %d", refiner.calls)
|
||||
}
|
||||
if summary.DialogueRefinementsAttempted != 1 || summary.DialogueRefinementsApplied != 1 {
|
||||
t.Fatalf("expected one applied dialogue refinement, got attempted=%d applied=%d",
|
||||
summary.DialogueRefinementsAttempted, summary.DialogueRefinementsApplied)
|
||||
}
|
||||
if len(repo.patches) != 1 {
|
||||
t.Fatalf("expected one patch, got %d", len(repo.patches))
|
||||
}
|
||||
if repo.patches[0].Start != 12.5 || repo.patches[0].Algorithm != ChromaprintDialogueAlgorithm {
|
||||
t.Fatalf("patch = %+v, want refined chromaprint marker", repo.patches[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunBackfillsExistingChapterMarkerWithSilenceBudget(t *testing.T) {
|
||||
source := models.MarkerSourceScanner
|
||||
algorithm := ChapterAlgorithm
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
package intromarkers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"unicode"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/lang"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
)
|
||||
|
||||
type chromaprintStartRefiner interface {
|
||||
RefineChromaprintStart(ctx context.Context, candidate Candidate, segment Segment) (Segment, bool, error)
|
||||
}
|
||||
|
||||
type DialogueBoundaryRefiner struct {
|
||||
config Config
|
||||
readFile func(string) ([]byte, error)
|
||||
}
|
||||
|
||||
var htmlTagPattern = regexp.MustCompile(`<[^>]+>`)
|
||||
|
||||
func NewDialogueBoundaryRefiner(config Config) *DialogueBoundaryRefiner {
|
||||
return &DialogueBoundaryRefiner{
|
||||
config: config.normalized(),
|
||||
readFile: os.ReadFile,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *DialogueBoundaryRefiner) RefineChromaprintStart(ctx context.Context, candidate Candidate, segment Segment) (Segment, bool, error) {
|
||||
cfg := r.config.normalized()
|
||||
if !cfg.DialogueRefinementEnabled || segment.End <= segment.Start || segment.Algorithm != ChromaprintAlgorithm {
|
||||
return segment, false, nil
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return segment, false, err
|
||||
}
|
||||
subtitle, ok := selectDialogueSubtitle(candidate)
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
readFile := r.readFile
|
||||
if readFile == nil {
|
||||
readFile = os.ReadFile
|
||||
}
|
||||
data, err := readFile(subtitle.Path)
|
||||
if err != nil {
|
||||
return segment, false, fmt.Errorf("read dialogue subtitle %q for file %d: %w", subtitle.Path, candidate.FileID, err)
|
||||
}
|
||||
cues, err := subtitles.ParseCues(data)
|
||||
if err != nil {
|
||||
return segment, false, fmt.Errorf("parse dialogue subtitle %q for file %d: %w", subtitle.Path, candidate.FileID, err)
|
||||
}
|
||||
refinedStart, ok := dialogueRefinedStart(cues, segment, cfg)
|
||||
if !ok {
|
||||
return segment, false, nil
|
||||
}
|
||||
refined := segment
|
||||
refined.Start = refinedStart
|
||||
refined.Algorithm = ChromaprintDialogueAlgorithm
|
||||
return refined, true, nil
|
||||
}
|
||||
|
||||
func selectDialogueSubtitle(candidate Candidate) (models.ExternalSubtitle, bool) {
|
||||
audioLanguage := lang.Canonical(candidate.AudioLanguage)
|
||||
bestScore := -1
|
||||
var best models.ExternalSubtitle
|
||||
for _, subtitle := range candidate.ExternalSubtitles {
|
||||
if strings.TrimSpace(subtitle.Path) == "" ||
|
||||
!isDialogueSubtitleFormat(subtitle.Format) ||
|
||||
subtitle.Forced ||
|
||||
subtitle.HearingImpaired {
|
||||
continue
|
||||
}
|
||||
score := 0
|
||||
subtitleLanguage := lang.Canonical(subtitle.Language)
|
||||
if audioLanguage != "" && subtitleLanguage == audioLanguage {
|
||||
score += 100
|
||||
} else if audioLanguage == "" && subtitleLanguage == "en" {
|
||||
score += 20
|
||||
}
|
||||
if subtitle.Default {
|
||||
score += 10
|
||||
}
|
||||
if subtitleLanguage != "" {
|
||||
score += 1
|
||||
}
|
||||
if score > bestScore {
|
||||
bestScore = score
|
||||
best = subtitle
|
||||
}
|
||||
}
|
||||
if bestScore < 0 {
|
||||
return models.ExternalSubtitle{}, false
|
||||
}
|
||||
return best, true
|
||||
}
|
||||
|
||||
func isDialogueSubtitleFormat(format string) bool {
|
||||
switch strings.ToLower(strings.TrimSpace(format)) {
|
||||
case "srt", "subrip", "vtt", "webvtt":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func dialogueRefinedStart(cues []subtitles.SubtitleCue, segment Segment, cfg Config) (float64, bool) {
|
||||
if segment.End <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
windowEnd := math.Min(segment.Start+cfg.DialogueRefinementWindowSeconds, segment.End)
|
||||
if windowEnd <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
lastDialogueEnd := segment.Start
|
||||
for _, cue := range cues {
|
||||
if !cueLooksLikeDialogue(cue) {
|
||||
continue
|
||||
}
|
||||
start := cue.Start.Seconds()
|
||||
end := cue.End.Seconds()
|
||||
if end <= segment.Start {
|
||||
continue
|
||||
}
|
||||
if start > windowEnd {
|
||||
break
|
||||
}
|
||||
if end > lastDialogueEnd {
|
||||
lastDialogueEnd = end
|
||||
}
|
||||
}
|
||||
|
||||
if lastDialogueEnd <= segment.Start {
|
||||
return 0, false
|
||||
}
|
||||
if lastDialogueEnd-segment.Start > cfg.DialogueRefinementMaxShiftSeconds {
|
||||
return 0, false
|
||||
}
|
||||
if segment.End-lastDialogueEnd < cfg.DialogueRefinementMinimumRemainingSeconds {
|
||||
return 0, false
|
||||
}
|
||||
return lastDialogueEnd, true
|
||||
}
|
||||
|
||||
func cueLooksLikeDialogue(cue subtitles.SubtitleCue) bool {
|
||||
for _, line := range cue.Lines {
|
||||
if lineLooksLikeDialogue(line) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func lineLooksLikeDialogue(line string) bool {
|
||||
cleaned := strings.TrimSpace(stripSubtitleDialogueMarkup(line))
|
||||
if cleaned == "" || strings.Contains(cleaned, "♪") {
|
||||
return false
|
||||
}
|
||||
lower := strings.ToLower(cleaned)
|
||||
if isBracketedCue(lower, "[", "]") ||
|
||||
isBracketedCue(lower, "(", ")") ||
|
||||
isBracketedCue(lower, "{", "}") {
|
||||
return false
|
||||
}
|
||||
for _, r := range cleaned {
|
||||
if unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func stripSubtitleDialogueMarkup(line string) string {
|
||||
line = stripSubtitleASSTags(line)
|
||||
line = htmlTagPattern.ReplaceAllString(line, "")
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
func isBracketedCue(line, open, close string) bool {
|
||||
return strings.HasPrefix(line, open) && strings.HasSuffix(line, close)
|
||||
}
|
||||
|
||||
func stripSubtitleASSTags(line string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(line))
|
||||
inTag := false
|
||||
for _, r := range line {
|
||||
switch {
|
||||
case r == '{':
|
||||
inTag = true
|
||||
case r == '}':
|
||||
inTag = false
|
||||
case !inTag:
|
||||
b.WriteRune(r)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package intromarkers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestDialogueBoundaryRefinerMovesStartPastEarlyDialogue(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
candidate := Candidate{
|
||||
FileID: 636600,
|
||||
AudioLanguage: "eng",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte(swatS05E02IntroLeadInSRT), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected dialogue refinement to apply")
|
||||
}
|
||||
if math.Abs(refined.Start-330.852) > 0.001 {
|
||||
t.Fatalf("refined start = %.3f, want 330.852", refined.Start)
|
||||
}
|
||||
if refined.End != segment.End {
|
||||
t.Fatalf("refined end = %.3f, want %.3f", refined.End, segment.End)
|
||||
}
|
||||
if refined.Algorithm != ChromaprintDialogueAlgorithm {
|
||||
t.Fatalf("algorithm = %q, want %q", refined.Algorithm, ChromaprintDialogueAlgorithm)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialogueBoundaryRefinerIgnoresMusicCues(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{Start: 100, End: 150, Confidence: 0.85, Algorithm: ChromaprintAlgorithm}
|
||||
candidate := Candidate{
|
||||
FileID: 1,
|
||||
AudioLanguage: "en",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte("1\n00:01:40,000 --> 00:01:48,000\n♪ Opening theme ♪\n\n"), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("music cue should not refine segment, got %+v", refined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialogueBoundaryRefinerKeepsMinimumRemainingDuration(t *testing.T) {
|
||||
cfg := DefaultConfig("ffmpeg")
|
||||
segment := Segment{Start: 100, End: 112, Confidence: 0.85, Algorithm: ChromaprintAlgorithm}
|
||||
candidate := Candidate{
|
||||
FileID: 1,
|
||||
AudioLanguage: "en",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
refiner := &DialogueBoundaryRefiner{
|
||||
config: cfg,
|
||||
readFile: func(path string) ([]byte, error) {
|
||||
return []byte("1\n00:01:40,000 --> 00:01:45,000\nStill talking.\n\n"), nil
|
||||
},
|
||||
}
|
||||
|
||||
refined, ok, err := refiner.RefineChromaprintStart(context.Background(), candidate, segment)
|
||||
if err != nil {
|
||||
t.Fatalf("RefineChromaprintStart returned error: %v", err)
|
||||
}
|
||||
if ok {
|
||||
t.Fatalf("refinement should not leave too-short intro, got %+v", refined)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectDialogueSubtitlePrefersAudioLanguage(t *testing.T) {
|
||||
candidate := Candidate{
|
||||
AudioLanguage: "eng",
|
||||
ExternalSubtitles: []models.ExternalSubtitle{
|
||||
{Path: "/episode.es.srt", Language: "es", Format: "srt"},
|
||||
{Path: "/episode.en.srt", Language: "en", Format: "srt"},
|
||||
},
|
||||
}
|
||||
|
||||
subtitle, ok := selectDialogueSubtitle(candidate)
|
||||
if !ok {
|
||||
t.Fatal("expected subtitle selection")
|
||||
}
|
||||
if subtitle.Path != "/episode.en.srt" {
|
||||
t.Fatalf("selected %q, want English sidecar", subtitle.Path)
|
||||
}
|
||||
}
|
||||
|
||||
const swatS05E02IntroLeadInSRT = `142
|
||||
00:05:11,006 --> 00:05:13,051
|
||||
Okay.
|
||||
|
||||
143
|
||||
00:05:13,095 --> 00:05:15,358
|
||||
I'm in.
|
||||
|
||||
144
|
||||
00:05:15,402 --> 00:05:19,144
|
||||
Never be in a hurry to die.
|
||||
|
||||
145
|
||||
00:05:19,188 --> 00:05:20,972
|
||||
Weapons are nice,
|
||||
but we need intel.
|
||||
|
||||
146
|
||||
00:05:21,016 --> 00:05:22,539
|
||||
We don't know where Delfina is
|
||||
|
||||
147
|
||||
00:05:22,583 --> 00:05:24,106
|
||||
or how many people
|
||||
are holding her.
|
||||
|
||||
148
|
||||
00:05:24,149 --> 00:05:25,499
|
||||
It won't be easy.
|
||||
|
||||
149
|
||||
00:05:25,542 --> 00:05:27,196
|
||||
Senor Novak has
|
||||
friends everywhere.
|
||||
|
||||
150
|
||||
00:05:27,239 --> 00:05:28,806
|
||||
I got an idea
|
||||
of where we can start.
|
||||
|
||||
151
|
||||
00:05:28,850 --> 00:05:30,852
|
||||
Follow me.
|
||||
|
||||
152
|
||||
00:06:20,771 --> 00:06:22,643
|
||||
Oh, keep screaming, chica.
|
||||
`
|
||||
@@ -45,6 +45,8 @@ const baseCandidateSelect = `
|
||||
COALESCE(mf.edition_key, ''),
|
||||
mf.chapters,
|
||||
mf.audio_tracks,
|
||||
mf.subtitle_tracks,
|
||||
mf.external_subtitles,
|
||||
mf.intro_start,
|
||||
mf.intro_end,
|
||||
mf.intro_markers_source,
|
||||
@@ -200,7 +202,7 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
var candidates []Candidate
|
||||
for rows.Next() {
|
||||
var c Candidate
|
||||
var chaptersJSON, audioTracksJSON []byte
|
||||
var chaptersJSON, audioTracksJSON, subtitleTracksJSON, externalSubtitlesJSON []byte
|
||||
if err := rows.Scan(
|
||||
&c.FileID,
|
||||
&c.EpisodeID,
|
||||
@@ -214,6 +216,8 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
&c.EditionKey,
|
||||
&chaptersJSON,
|
||||
&audioTracksJSON,
|
||||
&subtitleTracksJSON,
|
||||
&externalSubtitlesJSON,
|
||||
&c.IntroStart,
|
||||
&c.IntroEnd,
|
||||
&c.IntroMarkersSource,
|
||||
@@ -235,6 +239,16 @@ func scanCandidates(rows pgx.Rows) ([]Candidate, error) {
|
||||
}
|
||||
c.AudioLanguage = effectiveAudioLanguage(tracks)
|
||||
}
|
||||
if len(subtitleTracksJSON) > 0 {
|
||||
if err := json.Unmarshal(subtitleTracksJSON, &c.SubtitleTracks); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling subtitle tracks for file %d: %w", c.FileID, err)
|
||||
}
|
||||
}
|
||||
if len(externalSubtitlesJSON) > 0 {
|
||||
if err := json.Unmarshal(externalSubtitlesJSON, &c.ExternalSubtitles); err != nil {
|
||||
return nil, fmt.Errorf("unmarshaling external subtitles for file %d: %w", c.FileID, err)
|
||||
}
|
||||
}
|
||||
candidates = append(candidates, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -484,7 +498,7 @@ func (r *Repository) LoadSeasonState(ctx context.Context, state SeasonState, cfg
|
||||
state.MediaFolderID,
|
||||
state.AnalysisGroupKey,
|
||||
AlgorithmVersion,
|
||||
cfg.ConfigHash(),
|
||||
cfg.AnalysisConfigHash(),
|
||||
).Scan(
|
||||
&existing.SeasonID,
|
||||
&existing.MediaFolderID,
|
||||
@@ -535,7 +549,7 @@ func (r *Repository) UpsertSeasonState(ctx context.Context, state SeasonState, c
|
||||
state.MediaFolderID,
|
||||
state.AnalysisGroupKey,
|
||||
AlgorithmVersion,
|
||||
cfg.ConfigHash(),
|
||||
cfg.AnalysisConfigHash(),
|
||||
state.InputSignature,
|
||||
state.EpisodeCount,
|
||||
state.FileCount,
|
||||
@@ -576,21 +590,57 @@ func shouldApplyIntroPatch(row markerRow, patch IntroMarkerPatch) bool {
|
||||
if models.MarkerSourcePriority(source) < models.MarkerSourcePriority(patch.Source) {
|
||||
return true
|
||||
}
|
||||
if row.IntroMarkersAlgorithm == nil || *row.IntroMarkersAlgorithm != patch.Algorithm {
|
||||
return true
|
||||
}
|
||||
if row.IntroMarkersConfidence == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
existingAlgorithm := ""
|
||||
if row.IntroMarkersAlgorithm != nil {
|
||||
existingAlgorithm = *row.IntroMarkersAlgorithm
|
||||
}
|
||||
if existingAlgorithm == "" {
|
||||
return true
|
||||
}
|
||||
patchPriority := scannerAlgorithmPriority(patch.Algorithm)
|
||||
existingPriority := scannerAlgorithmPriority(existingAlgorithm)
|
||||
if patchPriority > existingPriority {
|
||||
return true
|
||||
}
|
||||
if patchPriority < existingPriority {
|
||||
return false
|
||||
}
|
||||
if patch.Confidence > *row.IntroMarkersConfidence {
|
||||
return true
|
||||
}
|
||||
if patch.Confidence >= *row.IntroMarkersConfidence && introRangeDiffers(row, patch, 0.5) {
|
||||
if patch.Confidence < *row.IntroMarkersConfidence {
|
||||
return false
|
||||
}
|
||||
if existingAlgorithm != patch.Algorithm && patchPriority == 0 {
|
||||
return true
|
||||
}
|
||||
if existingAlgorithm == patch.Algorithm && introRangeDiffers(row, patch, 0.5) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func scannerAlgorithmPriority(algorithm string) int {
|
||||
switch algorithm {
|
||||
case ChapterSilenceAlgorithm:
|
||||
return 40
|
||||
case ChapterAlgorithm:
|
||||
return 30
|
||||
case EpisodeVersionCopyAlgorithm:
|
||||
return 20
|
||||
case ChromaprintDialogueAlgorithm:
|
||||
return 15
|
||||
case ChromaprintAlgorithm:
|
||||
return 10
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
func introRangeDiffers(row markerRow, patch IntroMarkerPatch, tolerance float64) bool {
|
||||
if row.IntroStart == nil || row.IntroEnd == nil {
|
||||
return true
|
||||
@@ -622,9 +672,42 @@ func effectiveAudioLanguage(tracks []models.AudioTrack) string {
|
||||
func InputSignature(candidates []Candidate) string {
|
||||
parts := make([]string, 0, len(candidates))
|
||||
for _, c := range candidates {
|
||||
parts = append(parts, fmt.Sprintf("%d:%s:%d:%.3f:%s", c.FileID, c.FileHash, c.FileSize, c.DurationSeconds, c.EpisodeID))
|
||||
parts = append(parts, fmt.Sprintf("%d:%s:%d:%.3f:%s:%s",
|
||||
c.FileID,
|
||||
c.FileHash,
|
||||
c.FileSize,
|
||||
c.DurationSeconds,
|
||||
c.EpisodeID,
|
||||
subtitleInventorySignature(c),
|
||||
))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
sum := sha256.Sum256([]byte(strings.Join(parts, "\n")))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func subtitleInventorySignature(candidate Candidate) string {
|
||||
parts := make([]string, 0, len(candidate.ExternalSubtitles)+len(candidate.SubtitleTracks))
|
||||
for _, sub := range candidate.ExternalSubtitles {
|
||||
parts = append(parts, fmt.Sprintf("ext:%s:%s:%s:%t:%t:%t",
|
||||
sub.Path,
|
||||
sub.Language,
|
||||
sub.Format,
|
||||
sub.Forced,
|
||||
sub.Default,
|
||||
sub.HearingImpaired,
|
||||
))
|
||||
}
|
||||
for _, track := range candidate.SubtitleTracks {
|
||||
parts = append(parts, fmt.Sprintf("emb:%d:%s:%s:%t:%t:%t",
|
||||
track.Index,
|
||||
track.Language,
|
||||
track.Codec,
|
||||
track.Forced,
|
||||
track.Default,
|
||||
track.HearingImpaired,
|
||||
))
|
||||
}
|
||||
sort.Strings(parts)
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
@@ -57,3 +57,107 @@ func TestShouldApplyIntroPatchRejectsLowerPrioritySource(t *testing.T) {
|
||||
t.Fatal("scanner patch should not overwrite manual marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsLowerConfidenceAlgorithmChange(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.95
|
||||
algorithm := ChapterAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("lower-confidence chromaprint patch should not overwrite chapter marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsEqualConfidenceLowerRankAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.85
|
||||
algorithm := EpisodeVersionCopyAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("equal-confidence chromaprint patch should not overwrite copied chapter marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchRejectsHigherConfidenceLowerRankAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.85
|
||||
algorithm := EpisodeVersionCopyAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.90,
|
||||
Algorithm: ChromaprintAlgorithm,
|
||||
}
|
||||
|
||||
if shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("higher-confidence chromaprint patch should not replace copied marker")
|
||||
}
|
||||
}
|
||||
|
||||
func TestShouldApplyIntroPatchAllowsHigherRankLowerConfidenceAlgorithm(t *testing.T) {
|
||||
start := 331.5
|
||||
end := 362.5
|
||||
source := models.MarkerSourceScanner
|
||||
confidence := 0.95
|
||||
algorithm := ChromaprintAlgorithm
|
||||
row := markerRow{
|
||||
IntroStart: &start,
|
||||
IntroEnd: &end,
|
||||
IntroMarkersSource: &source,
|
||||
IntroMarkersConfidence: &confidence,
|
||||
IntroMarkersAlgorithm: &algorithm,
|
||||
}
|
||||
patch := IntroMarkerPatch{
|
||||
Start: 322.014,
|
||||
End: 363.465,
|
||||
Source: models.MarkerSourceScanner,
|
||||
Confidence: 0.85,
|
||||
Algorithm: ChapterAlgorithm,
|
||||
}
|
||||
|
||||
if !shouldApplyIntroPatch(row, patch) {
|
||||
t.Fatal("higher-rank chapter patch should replace lower-rank chromaprint marker")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,31 +11,37 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
AlgorithmVersion = 1
|
||||
ChapterAlgorithm = "chapter:v1"
|
||||
ChapterSilenceAlgorithm = "chapter:silence:v1"
|
||||
EpisodeVersionCopyAlgorithm = "episode-version-copy:v1"
|
||||
ChromaprintAlgorithm = "chromaprint:v1"
|
||||
ChromaprintFormat = "chromaprint:raw:uint32le"
|
||||
DefaultPointHopSeconds = 0.123
|
||||
AlgorithmVersion = 1
|
||||
AnalysisBehaviorVersion = 2
|
||||
ChapterAlgorithm = "chapter:v1"
|
||||
ChapterSilenceAlgorithm = "chapter:silence:v1"
|
||||
EpisodeVersionCopyAlgorithm = "episode-version-copy:v1"
|
||||
ChromaprintAlgorithm = "chromaprint:v1"
|
||||
ChromaprintDialogueAlgorithm = "chromaprint:dialogue:v1"
|
||||
ChromaprintFormat = "chromaprint:raw:uint32le"
|
||||
DefaultPointHopSeconds = 0.123
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
FFmpegPath string
|
||||
MaxParallelFFmpeg int
|
||||
AnalysisPercent int
|
||||
AnalysisLengthLimitMinutes int
|
||||
MinimumIntroDurationSeconds int
|
||||
MaximumIntroDurationSeconds int
|
||||
SilenceRefinementEnabled bool
|
||||
SilenceWindowBeforeSeconds float64
|
||||
SilenceWindowAfterSeconds float64
|
||||
SilenceMinimumDurationSeconds float64
|
||||
SilenceNoiseThresholdDB *int
|
||||
SilenceMinimumExtensionSeconds float64
|
||||
SilenceMaximumExtensionSeconds float64
|
||||
SilenceBackfillLimit int
|
||||
SilenceBackfillMaxDuration time.Duration
|
||||
FFmpegPath string
|
||||
MaxParallelFFmpeg int
|
||||
AnalysisPercent int
|
||||
AnalysisLengthLimitMinutes int
|
||||
MinimumIntroDurationSeconds int
|
||||
MaximumIntroDurationSeconds int
|
||||
SilenceRefinementEnabled bool
|
||||
SilenceWindowBeforeSeconds float64
|
||||
SilenceWindowAfterSeconds float64
|
||||
SilenceMinimumDurationSeconds float64
|
||||
SilenceNoiseThresholdDB *int
|
||||
SilenceMinimumExtensionSeconds float64
|
||||
SilenceMaximumExtensionSeconds float64
|
||||
SilenceBackfillLimit int
|
||||
SilenceBackfillMaxDuration time.Duration
|
||||
DialogueRefinementEnabled bool
|
||||
DialogueRefinementWindowSeconds float64
|
||||
DialogueRefinementMaxShiftSeconds float64
|
||||
DialogueRefinementMinimumRemainingSeconds float64
|
||||
}
|
||||
|
||||
func DefaultConfig(ffmpegPath string) Config {
|
||||
@@ -43,21 +49,25 @@ func DefaultConfig(ffmpegPath string) Config {
|
||||
ffmpegPath = "ffmpeg"
|
||||
}
|
||||
return Config{
|
||||
FFmpegPath: ffmpegPath,
|
||||
MaxParallelFFmpeg: 1,
|
||||
AnalysisPercent: 25,
|
||||
AnalysisLengthLimitMinutes: 10,
|
||||
MinimumIntroDurationSeconds: 15,
|
||||
MaximumIntroDurationSeconds: 120,
|
||||
SilenceRefinementEnabled: true,
|
||||
SilenceWindowBeforeSeconds: 3,
|
||||
SilenceWindowAfterSeconds: 30,
|
||||
SilenceMinimumDurationSeconds: 0.33,
|
||||
SilenceNoiseThresholdDB: intPtr(-50),
|
||||
SilenceMinimumExtensionSeconds: 0.5,
|
||||
SilenceMaximumExtensionSeconds: 30,
|
||||
SilenceBackfillLimit: 2000,
|
||||
SilenceBackfillMaxDuration: 45 * time.Minute,
|
||||
FFmpegPath: ffmpegPath,
|
||||
MaxParallelFFmpeg: 1,
|
||||
AnalysisPercent: 25,
|
||||
AnalysisLengthLimitMinutes: 10,
|
||||
MinimumIntroDurationSeconds: 15,
|
||||
MaximumIntroDurationSeconds: 120,
|
||||
SilenceRefinementEnabled: true,
|
||||
SilenceWindowBeforeSeconds: 3,
|
||||
SilenceWindowAfterSeconds: 30,
|
||||
SilenceMinimumDurationSeconds: 0.33,
|
||||
SilenceNoiseThresholdDB: intPtr(-50),
|
||||
SilenceMinimumExtensionSeconds: 0.5,
|
||||
SilenceMaximumExtensionSeconds: 30,
|
||||
SilenceBackfillLimit: 2000,
|
||||
SilenceBackfillMaxDuration: 45 * time.Minute,
|
||||
DialogueRefinementEnabled: true,
|
||||
DialogueRefinementWindowSeconds: 15,
|
||||
DialogueRefinementMaxShiftSeconds: 20,
|
||||
DialogueRefinementMinimumRemainingSeconds: 15,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -104,6 +114,15 @@ func (c Config) normalized() Config {
|
||||
if c.SilenceBackfillMaxDuration <= 0 {
|
||||
c.SilenceBackfillMaxDuration = 45 * time.Minute
|
||||
}
|
||||
if c.DialogueRefinementWindowSeconds <= 0 {
|
||||
c.DialogueRefinementWindowSeconds = 15
|
||||
}
|
||||
if c.DialogueRefinementMaxShiftSeconds <= 0 {
|
||||
c.DialogueRefinementMaxShiftSeconds = 20
|
||||
}
|
||||
if c.DialogueRefinementMinimumRemainingSeconds <= 0 {
|
||||
c.DialogueRefinementMinimumRemainingSeconds = float64(c.MinimumIntroDurationSeconds)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -122,6 +141,19 @@ func (c Config) ConfigHash() string {
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
func (c Config) AnalysisConfigHash() string {
|
||||
c = c.normalized()
|
||||
sum := sha256.Sum256([]byte(fmt.Sprintf("%s:%d:%t:%.3f:%.3f:%.3f",
|
||||
c.ConfigHash(),
|
||||
AnalysisBehaviorVersion,
|
||||
c.DialogueRefinementEnabled,
|
||||
c.DialogueRefinementWindowSeconds,
|
||||
c.DialogueRefinementMaxShiftSeconds,
|
||||
c.DialogueRefinementMinimumRemainingSeconds,
|
||||
)))
|
||||
return hex.EncodeToString(sum[:])[:16]
|
||||
}
|
||||
|
||||
type Candidate struct {
|
||||
FileID int
|
||||
EpisodeID string
|
||||
@@ -135,6 +167,8 @@ type Candidate struct {
|
||||
EditionKey string
|
||||
AudioLanguage string
|
||||
Chapters []models.MediaChapter
|
||||
SubtitleTracks []models.SubtitleTrack
|
||||
ExternalSubtitles []models.ExternalSubtitle
|
||||
IntroStart *float64
|
||||
IntroEnd *float64
|
||||
IntroMarkersSource *string
|
||||
@@ -220,21 +254,24 @@ type SeasonState struct {
|
||||
}
|
||||
|
||||
type RunSummary struct {
|
||||
LibrariesScanned int `json:"libraries_scanned"`
|
||||
FilesConsidered int `json:"files_considered"`
|
||||
SeasonGroupsConsidered int `json:"season_groups_considered"`
|
||||
FingerprintsComputed int `json:"fingerprints_computed"`
|
||||
FingerprintCacheHits int `json:"fingerprint_cache_hits"`
|
||||
ChapterMarkersWritten int `json:"chapter_markers_written"`
|
||||
ChromaprintMarkersWritten int `json:"chromaprint_markers_written"`
|
||||
GroupsNotFound int `json:"groups_not_found"`
|
||||
GroupsSkipped int `json:"groups_skipped"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
ChromaprintSupported bool `json:"chromaprint_supported"`
|
||||
ChromaprintSupportMessage string `json:"chromaprint_support_message,omitempty"`
|
||||
SilenceRefinementsAttempted int `json:"silence_refinements_attempted"`
|
||||
SilenceRefinementsApplied int `json:"silence_refinements_applied"`
|
||||
SilenceRefinementErrors int `json:"silence_refinement_errors"`
|
||||
EpisodeVersionMarkersCopied int `json:"episode_version_markers_copied"`
|
||||
SilenceBackfillConsidered int `json:"silence_backfill_considered"`
|
||||
LibrariesScanned int `json:"libraries_scanned"`
|
||||
FilesConsidered int `json:"files_considered"`
|
||||
SeasonGroupsConsidered int `json:"season_groups_considered"`
|
||||
FingerprintsComputed int `json:"fingerprints_computed"`
|
||||
FingerprintCacheHits int `json:"fingerprint_cache_hits"`
|
||||
ChapterMarkersWritten int `json:"chapter_markers_written"`
|
||||
ChromaprintMarkersWritten int `json:"chromaprint_markers_written"`
|
||||
GroupsNotFound int `json:"groups_not_found"`
|
||||
GroupsSkipped int `json:"groups_skipped"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
ChromaprintSupported bool `json:"chromaprint_supported"`
|
||||
ChromaprintSupportMessage string `json:"chromaprint_support_message,omitempty"`
|
||||
SilenceRefinementsAttempted int `json:"silence_refinements_attempted"`
|
||||
SilenceRefinementsApplied int `json:"silence_refinements_applied"`
|
||||
SilenceRefinementErrors int `json:"silence_refinement_errors"`
|
||||
EpisodeVersionMarkersCopied int `json:"episode_version_markers_copied"`
|
||||
SilenceBackfillConsidered int `json:"silence_backfill_considered"`
|
||||
DialogueRefinementsAttempted int `json:"dialogue_refinements_attempted"`
|
||||
DialogueRefinementsApplied int `json:"dialogue_refinements_applied"`
|
||||
DialogueRefinementErrors int `json:"dialogue_refinement_errors"`
|
||||
}
|
||||
|
||||
@@ -337,7 +337,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
|
||||
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) && decision.RestartOnTimeout {
|
||||
seekSeconds, ok, seekErr := transcodeSession.RestartSeekTarget(segNum)
|
||||
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
|
||||
slog.Error("resolve transcode seek target",
|
||||
|
||||
@@ -0,0 +1,299 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// OutcomeStatus values for a contribution attempt, in addition to the provider
|
||||
// SubmissionStatus* values.
|
||||
const (
|
||||
OutcomeStatusError = "error"
|
||||
OutcomeStatusRateLimited = "rate_limited"
|
||||
OutcomeStatusSkipped = "skipped"
|
||||
)
|
||||
|
||||
// ContributeOptions scopes a contribution run.
|
||||
type ContributeOptions struct {
|
||||
// Provider restricts the run to a single provider id ("" = all enabled).
|
||||
Provider string
|
||||
// Segments restricts the run to specific kinds (nil = every present segment).
|
||||
Segments []MarkerKind
|
||||
// Auto marks a background run: a segment is only contributed when the
|
||||
// provider has contribute_auto_local enabled and the segment is a local
|
||||
// (scanner) intro at or above the provider's confidence threshold.
|
||||
Auto bool
|
||||
}
|
||||
|
||||
// ContributionOutcome is the per (provider, segment) result of a run.
|
||||
type ContributionOutcome struct {
|
||||
Provider string
|
||||
Segment MarkerKind
|
||||
Status string // pending | accepted | rejected | error | rate_limited | skipped
|
||||
SubmissionID string
|
||||
Reason string // skip reason or error message
|
||||
RetryAfter time.Duration
|
||||
}
|
||||
|
||||
// providerConfigReader is the read surface ContributionService needs from the
|
||||
// provider config store (satisfied by *ProviderConfigStore).
|
||||
type providerConfigReader interface {
|
||||
Get(provider string) (ProviderConfig, bool)
|
||||
}
|
||||
|
||||
// contributionRecorder is the audit surface ContributionService needs
|
||||
// (satisfied by *ContributionStore).
|
||||
type contributionRecorder interface {
|
||||
AlreadySubmitted(ctx context.Context, fileID int, provider, segmentKind, contentHash string) (bool, error)
|
||||
Record(ctx context.Context, row ContributionRow) error
|
||||
}
|
||||
|
||||
// ContributionService submits a file's eligible markers to enabled submitter
|
||||
// providers, idempotently and audited. Both the on-demand admin path and the
|
||||
// daily auto-submission task funnel through ContributeFile.
|
||||
type ContributionService struct {
|
||||
registry *Registry
|
||||
resolver ExternalIDResolver
|
||||
config providerConfigReader
|
||||
store contributionRecorder
|
||||
logger *slog.Logger
|
||||
}
|
||||
|
||||
// NewContributionService constructs the service.
|
||||
func NewContributionService(registry *Registry, resolver ExternalIDResolver, config providerConfigReader, store contributionRecorder, logger *slog.Logger) *ContributionService {
|
||||
if logger == nil {
|
||||
logger = slog.Default()
|
||||
}
|
||||
return &ContributionService{registry: registry, resolver: resolver, config: config, store: store, logger: logger}
|
||||
}
|
||||
|
||||
type segmentData struct {
|
||||
kind MarkerKind
|
||||
name string
|
||||
start *float64
|
||||
end *float64
|
||||
source *string
|
||||
confidence *float64
|
||||
}
|
||||
|
||||
func fileSegments(file *models.MediaFile) []segmentData {
|
||||
return []segmentData{
|
||||
{MarkerKindIntro, "intro", file.IntroStart, file.IntroEnd, file.IntroMarkersSource, file.IntroMarkersConfidence},
|
||||
{MarkerKindCredits, "credits", file.CreditsStart, file.CreditsEnd, file.CreditsMarkersSource, file.CreditsMarkersConfidence},
|
||||
{MarkerKindRecap, "recap", file.RecapStart, file.RecapEnd, file.RecapMarkersSource, file.RecapMarkersConfidence},
|
||||
{MarkerKindPreview, "preview", file.PreviewStart, file.PreviewEnd, file.PreviewMarkersSource, file.PreviewMarkersConfidence},
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ContributionService) submitters() []Submitter {
|
||||
if s == nil || s.registry == nil {
|
||||
return nil
|
||||
}
|
||||
var out []Submitter
|
||||
for _, p := range s.registry.Providers() {
|
||||
if sub, ok := p.(Submitter); ok {
|
||||
out = append(out, sub)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ContributeFile submits the file's eligible markers and returns one outcome
|
||||
// per (provider, segment) attempted (including explicit skips).
|
||||
func (s *ContributionService) ContributeFile(ctx context.Context, file *models.MediaFile, opts ContributeOptions) ([]ContributionOutcome, error) {
|
||||
if s == nil || file == nil || s.registry == nil || s.resolver == nil || s.config == nil || s.store == nil {
|
||||
return nil, nil
|
||||
}
|
||||
ids, err := s.resolver.ResolveForFile(ctx, file)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ids.HasAnyID() {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
want := func(k MarkerKind) bool {
|
||||
if len(opts.Segments) == 0 {
|
||||
return true
|
||||
}
|
||||
for _, kind := range opts.Segments {
|
||||
if kind == k {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var outcomes []ContributionOutcome
|
||||
for _, sub := range s.submitters() {
|
||||
providerID := sub.ID()
|
||||
if opts.Provider != "" && opts.Provider != providerID {
|
||||
continue
|
||||
}
|
||||
cfg, ok := s.config.Get(providerID)
|
||||
if !ok || !cfg.ContributeEnabled {
|
||||
continue
|
||||
}
|
||||
for _, seg := range fileSegments(file) {
|
||||
if !want(seg.kind) {
|
||||
continue
|
||||
}
|
||||
if outcome, attempted := s.contributeSegment(ctx, sub, providerID, cfg, file, ids, seg, opts.Auto); attempted {
|
||||
outcomes = append(outcomes, outcome)
|
||||
if outcome.Status == OutcomeStatusRateLimited {
|
||||
return outcomes, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return outcomes, nil
|
||||
}
|
||||
|
||||
func (s *ContributionService) contributeSegment(
|
||||
ctx context.Context,
|
||||
sub Submitter,
|
||||
providerID string,
|
||||
cfg ProviderConfig,
|
||||
file *models.MediaFile,
|
||||
ids ExternalIDs,
|
||||
seg segmentData,
|
||||
auto bool,
|
||||
) (ContributionOutcome, bool) {
|
||||
if seg.start == nil || seg.end == nil {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
source := ""
|
||||
if seg.source != nil {
|
||||
source = *seg.source
|
||||
}
|
||||
// Only ever contribute our own data; never round-trip online markers.
|
||||
if source != models.MarkerSourceScanner && source != models.MarkerSourceManual {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
if auto {
|
||||
if !cfg.ContributeAutoLocal || source != models.MarkerSourceScanner || seg.kind != MarkerKindIntro {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
confidence := 0.0
|
||||
if seg.confidence != nil {
|
||||
confidence = *seg.confidence
|
||||
}
|
||||
if confidence < cfg.ContributeMinConfidence {
|
||||
return ContributionOutcome{}, false
|
||||
}
|
||||
}
|
||||
if missing := missingRequiredExternalIDs(sub, ids); len(missing) > 0 {
|
||||
return ContributionOutcome{
|
||||
Provider: providerID,
|
||||
Segment: seg.kind,
|
||||
Status: OutcomeStatusSkipped,
|
||||
Reason: strings.Join(missing, ",") + " id required",
|
||||
}, true
|
||||
}
|
||||
|
||||
startMs := int64(*seg.start * 1000)
|
||||
endMs := int64(*seg.end * 1000)
|
||||
durMs := int64(file.Duration) * 1000
|
||||
hash := ContentHash(seg.name, &startMs, &endMs, &durMs, contributionTargetParts(ids)...)
|
||||
|
||||
already, err := s.store.AlreadySubmitted(ctx, file.ID, providerID, seg.name, hash)
|
||||
if err != nil {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusError, Reason: err.Error()}, true
|
||||
}
|
||||
if already {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusSkipped, Reason: "already submitted"}, true
|
||||
}
|
||||
|
||||
startDur := time.Duration(*seg.start * float64(time.Second))
|
||||
endDur := time.Duration(*seg.end * float64(time.Second))
|
||||
req := SubmissionRequest{
|
||||
Kind: ids.Kind,
|
||||
ExternalIDs: ids.AsRequestMap(),
|
||||
SeasonNumber: ids.SeasonNumber,
|
||||
EpisodeNumber: ids.EpisodeNumber,
|
||||
Segment: seg.kind,
|
||||
Start: &startDur,
|
||||
End: &endDur,
|
||||
Duration: time.Duration(file.Duration) * time.Second,
|
||||
}
|
||||
|
||||
row := ContributionRow{
|
||||
MediaFileID: file.ID,
|
||||
Provider: providerID,
|
||||
SegmentKind: seg.name,
|
||||
Source: source,
|
||||
SubmittedStartMs: &startMs,
|
||||
SubmittedEndMs: &endMs,
|
||||
VideoDurationMs: &durMs,
|
||||
ContentHash: hash,
|
||||
}
|
||||
|
||||
result, err := sub.SubmitMarker(ctx, req)
|
||||
if err != nil {
|
||||
msg := err.Error()
|
||||
row.Status = OutcomeStatusError
|
||||
row.Error = &msg
|
||||
if recErr := s.store.Record(ctx, row); recErr != nil {
|
||||
s.logger.Warn("record contribution error failed", "file_id", file.ID, "provider", providerID, "segment", seg.name, "error", recErr)
|
||||
}
|
||||
if after, ok := RetryAfter(err); ok {
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusRateLimited, Reason: msg, RetryAfter: after}, true
|
||||
}
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: OutcomeStatusError, Reason: msg}, true
|
||||
}
|
||||
|
||||
row.Status = result.Status
|
||||
if row.Status == "" {
|
||||
row.Status = SubmissionStatusPending
|
||||
}
|
||||
if result.ID != "" {
|
||||
id := result.ID
|
||||
row.SubmissionID = &id
|
||||
}
|
||||
if err := s.store.Record(ctx, row); err != nil {
|
||||
s.logger.Warn("record contribution failed", "file_id", file.ID, "provider", providerID, "segment", seg.name, "error", err)
|
||||
}
|
||||
return ContributionOutcome{Provider: providerID, Segment: seg.kind, Status: row.Status, SubmissionID: result.ID}, true
|
||||
}
|
||||
|
||||
func contributionTargetParts(ids ExternalIDs) []string {
|
||||
return []string{
|
||||
itemTypeName(ids.Kind),
|
||||
ids.TmdbID,
|
||||
ids.ImdbID,
|
||||
ids.TvdbID,
|
||||
intPart(ids.SeasonNumber),
|
||||
intPart(ids.EpisodeNumber),
|
||||
}
|
||||
}
|
||||
|
||||
func intPart(v int) string {
|
||||
if v == 0 {
|
||||
return ""
|
||||
}
|
||||
return strconv.Itoa(v)
|
||||
}
|
||||
|
||||
func missingRequiredExternalIDs(sub Submitter, ids ExternalIDs) []string {
|
||||
reqProvider, ok := sub.(SubmissionRequirementProvider)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
reqs := reqProvider.SubmissionRequirements()
|
||||
if len(reqs.RequiredExternalIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
values := ids.AsRequestMap()
|
||||
var missing []string
|
||||
for _, key := range reqs.RequiredExternalIDs {
|
||||
if strings.TrimSpace(values[key]) == "" {
|
||||
missing = append(missing, key)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakeSubmitter struct {
|
||||
id string
|
||||
submitted []SubmissionRequest
|
||||
result SubmissionResult
|
||||
err error
|
||||
required []string
|
||||
}
|
||||
|
||||
func (f *fakeSubmitter) ID() string { return f.id }
|
||||
func (f *fakeSubmitter) FetchMarkers(context.Context, Request) (Result, error) { return Result{}, nil }
|
||||
func (f *fakeSubmitter) SubmitMarker(_ context.Context, req SubmissionRequest) (SubmissionResult, error) {
|
||||
f.submitted = append(f.submitted, req)
|
||||
if f.err != nil {
|
||||
return SubmissionResult{}, f.err
|
||||
}
|
||||
if f.result.Status == "" {
|
||||
return SubmissionResult{ID: "id1", Status: SubmissionStatusPending}, nil
|
||||
}
|
||||
return f.result, nil
|
||||
}
|
||||
func (f *fakeSubmitter) FetchUserStats(context.Context) (UserStats, error) { return UserStats{}, nil }
|
||||
func (f *fakeSubmitter) SubmissionRequirements() SubmissionRequirements {
|
||||
return SubmissionRequirements{RequiredExternalIDs: f.required}
|
||||
}
|
||||
|
||||
type fakeResolver struct{ ids ExternalIDs }
|
||||
|
||||
func (f fakeResolver) ResolveForFile(context.Context, *models.MediaFile) (ExternalIDs, error) {
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
type fakeConfig map[string]ProviderConfig
|
||||
|
||||
func (f fakeConfig) Get(p string) (ProviderConfig, bool) { c, ok := f[p]; return c, ok }
|
||||
|
||||
type fakeRecorder struct {
|
||||
already bool
|
||||
recorded []ContributionRow
|
||||
}
|
||||
|
||||
func (f *fakeRecorder) AlreadySubmitted(context.Context, int, string, string, string) (bool, error) {
|
||||
return f.already, nil
|
||||
}
|
||||
func (f *fakeRecorder) Record(_ context.Context, row ContributionRow) error {
|
||||
f.recorded = append(f.recorded, row)
|
||||
return nil
|
||||
}
|
||||
|
||||
func floatPtr(v float64) *float64 { return &v }
|
||||
func strPtr(v string) *string { return &v }
|
||||
|
||||
func newContribFile() *models.MediaFile {
|
||||
return &models.MediaFile{ID: 7, EpisodeID: "ep1", SeasonNumber: 1, EpisodeNumber: 3, Duration: 1800}
|
||||
}
|
||||
|
||||
func newContribService(sub *fakeSubmitter, cfg fakeConfig, rec *fakeRecorder) *ContributionService {
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TmdbID: "1234", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
return NewContributionService(reg, resolver, cfg, rec, nil)
|
||||
}
|
||||
|
||||
func TestContributeSkipsOnlineSourced(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceOnline) // came FROM introdb
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, &fakeRecorder{})
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 0 {
|
||||
t.Errorf("online-sourced marker must not be submitted, got %d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 0 {
|
||||
t.Errorf("expected no outcomes, got %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSubmitsManualOnDemand(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.CreditsStart, file.CreditsEnd = floatPtr(1500), floatPtr(1800)
|
||||
file.CreditsMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 || sub.submitted[0].Segment != MarkerKindCredits {
|
||||
t.Fatalf("expected 1 credits submission, got %+v", sub.submitted)
|
||||
}
|
||||
if len(rec.recorded) != 1 || rec.recorded[0].Status != SubmissionStatusPending {
|
||||
t.Errorf("expected recorded pending, got %+v", rec.recorded)
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != SubmissionStatusPending {
|
||||
t.Errorf("outcomes = %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeAutoGatesOnThresholdAndKind(t *testing.T) {
|
||||
cfg := fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true, ContributeAutoLocal: true, ContributeMinConfidence: 0.9}}
|
||||
|
||||
subLow := &fakeSubmitter{id: "introdb"}
|
||||
fileLow := newContribFile()
|
||||
fileLow.IntroStart, fileLow.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
fileLow.IntroMarkersSource = strPtr(models.MarkerSourceScanner)
|
||||
fileLow.IntroMarkersConfidence = floatPtr(0.8)
|
||||
if _, err := newContribService(subLow, cfg, &fakeRecorder{}).ContributeFile(context.Background(), fileLow, ContributeOptions{Auto: true}); err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(subLow.submitted) != 0 {
|
||||
t.Errorf("below-threshold scanner intro must be skipped, got %d", len(subLow.submitted))
|
||||
}
|
||||
|
||||
subHigh := &fakeSubmitter{id: "introdb"}
|
||||
fileHigh := newContribFile()
|
||||
fileHigh.IntroStart, fileHigh.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
fileHigh.IntroMarkersSource = strPtr(models.MarkerSourceScanner)
|
||||
fileHigh.IntroMarkersConfidence = floatPtr(0.95)
|
||||
if _, err := newContribService(subHigh, cfg, &fakeRecorder{}).ContributeFile(context.Background(), fileHigh, ContributeOptions{Auto: true}); err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(subHigh.submitted) != 1 || subHigh.submitted[0].Segment != MarkerKindIntro {
|
||||
t.Errorf("above-threshold scanner intro should submit, got %+v", subHigh.submitted)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSkipsDuplicate(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, &fakeRecorder{already: true})
|
||||
|
||||
outcomes, _ := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if len(sub.submitted) != 0 {
|
||||
t.Errorf("duplicate must not submit, got %d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusSkipped {
|
||||
t.Errorf("expected skipped outcome, got %+v", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeSkipsWhenProviderRequiredIDMissing(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb", required: []string{ExternalIDKeyTMDB}}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TvdbID: "777", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
svc := NewContributionService(reg, resolver, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec, nil)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 0 || len(rec.recorded) != 0 {
|
||||
t.Fatalf("tmdb-missing marker should be skipped before submit/record, submitted=%d recorded=%d", len(sub.submitted), len(rec.recorded))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusSkipped || outcomes[0].Reason != "tmdb id required" {
|
||||
t.Fatalf("outcomes = %+v, want skipped tmdb id required", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeAllowsTVDBOnlyWhenProviderDoesNotRequireTMDB(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "plugin:1:markers"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
|
||||
reg := NewRegistry(nil)
|
||||
_ = reg.Register(sub)
|
||||
resolver := fakeResolver{ids: ExternalIDs{Kind: ItemKindEpisode, TvdbID: "777", SeasonNumber: 1, EpisodeNumber: 3}}
|
||||
svc := NewContributionService(reg, resolver, fakeConfig{"plugin:1:markers": {Provider: "plugin:1:markers", ContributeEnabled: true}}, rec, nil)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 || sub.submitted[0].ExternalIDs[ExternalIDKeyTVDB] != "777" {
|
||||
t.Fatalf("tvdb-only marker should submit to provider without tmdb requirement, submitted=%+v", sub.submitted)
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != SubmissionStatusPending {
|
||||
t.Fatalf("outcomes = %+v, want pending", outcomes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeStopsOnRetryAfterError(t *testing.T) {
|
||||
sub := &fakeSubmitter{
|
||||
id: "introdb",
|
||||
err: &RetryAfterError{Provider: "introdb", RetryAfter: 45 * time.Second, Message: "usage limited"},
|
||||
}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
file.CreditsStart, file.CreditsEnd = floatPtr(1700), floatPtr(1800)
|
||||
file.CreditsMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
rec := &fakeRecorder{}
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb", ContributeEnabled: true}}, rec)
|
||||
|
||||
outcomes, err := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("ContributeFile: %v", err)
|
||||
}
|
||||
if len(sub.submitted) != 1 {
|
||||
t.Fatalf("expected contribution loop to stop after first rate limit, submitted=%d", len(sub.submitted))
|
||||
}
|
||||
if len(outcomes) != 1 || outcomes[0].Status != OutcomeStatusRateLimited || outcomes[0].RetryAfter != 45*time.Second {
|
||||
t.Fatalf("outcomes = %+v, want one rate_limited retry-after outcome", outcomes)
|
||||
}
|
||||
if len(rec.recorded) != 1 || rec.recorded[0].Status != OutcomeStatusError {
|
||||
t.Fatalf("recorded = %+v, want one error audit row", rec.recorded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeDisabledProviderNoop(t *testing.T) {
|
||||
sub := &fakeSubmitter{id: "introdb"}
|
||||
file := newContribFile()
|
||||
file.IntroStart, file.IntroEnd = floatPtr(0), floatPtr(60)
|
||||
file.IntroMarkersSource = strPtr(models.MarkerSourceManual)
|
||||
// contribute_enabled defaults false
|
||||
svc := newContribService(sub, fakeConfig{"introdb": {Provider: "introdb"}}, &fakeRecorder{})
|
||||
|
||||
outcomes, _ := svc.ContributeFile(context.Background(), file, ContributeOptions{})
|
||||
if len(sub.submitted) != 0 || len(outcomes) != 0 {
|
||||
t.Errorf("disabled provider must be a no-op, submitted=%d outcomes=%d", len(sub.submitted), len(outcomes))
|
||||
}
|
||||
}
|
||||
|
||||
func TestContentHashStableAndSensitive(t *testing.T) {
|
||||
s, e, d := int64(0), int64(60000), int64(1800000)
|
||||
target := contributionTargetParts(ExternalIDs{Kind: ItemKindEpisode, TmdbID: "1234", SeasonNumber: 1, EpisodeNumber: 3})
|
||||
h1 := ContentHash("intro", &s, &e, &d, target...)
|
||||
if h1 != ContentHash("intro", &s, &e, &d, target...) {
|
||||
t.Error("hash not stable for identical input")
|
||||
}
|
||||
e2 := int64(61000)
|
||||
if ContentHash("intro", &s, &e2, &d, target...) == h1 {
|
||||
t.Error("changed end should change hash")
|
||||
}
|
||||
rematched := contributionTargetParts(ExternalIDs{Kind: ItemKindEpisode, TmdbID: "9999", SeasonNumber: 1, EpisodeNumber: 3})
|
||||
if ContentHash("intro", &s, &e, &d, rematched...) == h1 {
|
||||
t.Error("changed resolved target should change hash")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ContributionRow is one submission audit record from marker_contributions.
|
||||
type ContributionRow struct {
|
||||
ID string
|
||||
MediaFileID int
|
||||
Provider string
|
||||
SegmentKind string
|
||||
Source string
|
||||
SubmittedStartMs *int64
|
||||
SubmittedEndMs *int64
|
||||
VideoDurationMs *int64
|
||||
ContentHash string
|
||||
SubmissionID *string
|
||||
Status string
|
||||
HTTPStatus *int
|
||||
Error *string
|
||||
SubmittedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
}
|
||||
|
||||
// ContributionStore persists marker contribution attempts for idempotency and
|
||||
// audit.
|
||||
type ContributionStore struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
// NewContributionStore constructs a store backed by the supplied pool.
|
||||
func NewContributionStore(pool *pgxpool.Pool) *ContributionStore {
|
||||
return &ContributionStore{pool: pool}
|
||||
}
|
||||
|
||||
// ContentHash is a stable hash over the contributed value and resolved target.
|
||||
// Identical submissions hash identically (never resubmitted); a marker
|
||||
// correction or rematch to a different provider identity hashes differently.
|
||||
func ContentHash(segmentKind string, startMs, endMs, durationMs *int64, targetParts ...string) string {
|
||||
h := sha256.New()
|
||||
fmt.Fprintf(h, "%s|%s|%s|%s", segmentKind, ptrIntStr(startMs), ptrIntStr(endMs), ptrIntStr(durationMs))
|
||||
for _, part := range targetParts {
|
||||
fmt.Fprintf(h, "|%s", part)
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil))[:32]
|
||||
}
|
||||
|
||||
func ptrIntStr(v *int64) string {
|
||||
if v == nil {
|
||||
return "null"
|
||||
}
|
||||
return strconv.FormatInt(*v, 10)
|
||||
}
|
||||
|
||||
// AlreadySubmitted reports whether a non-error contribution with this value-hash
|
||||
// already exists for the file+provider+segment. Errors are excluded so failed
|
||||
// attempts can be retried.
|
||||
func (s *ContributionStore) AlreadySubmitted(ctx context.Context, fileID int, provider, segmentKind, contentHash string) (bool, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return false, nil
|
||||
}
|
||||
var exists bool
|
||||
if err := s.pool.QueryRow(ctx, `
|
||||
SELECT EXISTS (
|
||||
SELECT 1 FROM marker_contributions
|
||||
WHERE media_file_id = $1 AND provider = $2 AND segment_kind = $3
|
||||
AND content_hash = $4 AND status <> 'error'
|
||||
)`, fileID, provider, segmentKind, contentHash).Scan(&exists); err != nil {
|
||||
return false, fmt.Errorf("check marker contribution: %w", err)
|
||||
}
|
||||
return exists, nil
|
||||
}
|
||||
|
||||
// Record upserts a contribution row keyed by (file, provider, segment, hash).
|
||||
func (s *ContributionStore) Record(ctx context.Context, row ContributionRow) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("contribution store unavailable")
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_contributions (
|
||||
media_file_id, provider, segment_kind, source,
|
||||
submitted_start_ms, submitted_end_ms, video_duration_ms,
|
||||
content_hash, submission_id, status, http_status, error, updated_at
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, now())
|
||||
ON CONFLICT (media_file_id, provider, segment_kind, content_hash) DO UPDATE SET
|
||||
source = EXCLUDED.source,
|
||||
submitted_start_ms = EXCLUDED.submitted_start_ms,
|
||||
submitted_end_ms = EXCLUDED.submitted_end_ms,
|
||||
video_duration_ms = EXCLUDED.video_duration_ms,
|
||||
submission_id = EXCLUDED.submission_id,
|
||||
status = EXCLUDED.status,
|
||||
http_status = EXCLUDED.http_status,
|
||||
error = EXCLUDED.error,
|
||||
updated_at = now()`,
|
||||
row.MediaFileID, row.Provider, row.SegmentKind, row.Source,
|
||||
row.SubmittedStartMs, row.SubmittedEndMs, row.VideoDurationMs,
|
||||
row.ContentHash, row.SubmissionID, row.Status, row.HTTPStatus, row.Error,
|
||||
); err != nil {
|
||||
return fmt.Errorf("record marker contribution: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CandidateLocalIntroFiles returns ids of episode files carrying a local
|
||||
// (scanner) intro marker at or above minConfidence — the auto-contribution
|
||||
// candidates. Iterated by keyset (id > afterID) for stable paging.
|
||||
func (s *ContributionStore) CandidateLocalIntroFiles(ctx context.Context, minConfidence float64, afterID, limit int) ([]int, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id FROM media_files
|
||||
WHERE episode_id IS NOT NULL
|
||||
AND intro_markers_source = $1
|
||||
AND intro_start IS NOT NULL AND intro_end IS NOT NULL
|
||||
AND COALESCE(intro_markers_confidence, 0) >= $2
|
||||
AND id > $3
|
||||
ORDER BY id
|
||||
LIMIT $4`, models.MarkerSourceScanner, minConfidence, afterID, limit)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query contribution candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var ids []int
|
||||
for rows.Next() {
|
||||
var id int
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, fmt.Errorf("scan candidate id: %w", err)
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
return ids, rows.Err()
|
||||
}
|
||||
|
||||
// ListByFile returns the contribution history for a file, newest first.
|
||||
func (s *ContributionStore) ListByFile(ctx context.Context, fileID int) ([]ContributionRow, error) {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT id, media_file_id, provider, segment_kind, source,
|
||||
submitted_start_ms, submitted_end_ms, video_duration_ms,
|
||||
content_hash, submission_id, status, http_status, error,
|
||||
submitted_at, updated_at
|
||||
FROM marker_contributions WHERE media_file_id = $1
|
||||
ORDER BY updated_at DESC`, fileID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list marker contributions: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []ContributionRow
|
||||
for rows.Next() {
|
||||
var r ContributionRow
|
||||
if err := rows.Scan(
|
||||
&r.ID, &r.MediaFileID, &r.Provider, &r.SegmentKind, &r.Source,
|
||||
&r.SubmittedStartMs, &r.SubmittedEndMs, &r.VideoDurationMs,
|
||||
&r.ContentHash, &r.SubmissionID, &r.Status, &r.HTTPStatus, &r.Error,
|
||||
&r.SubmittedAt, &r.UpdatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan marker contribution: %w", err)
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -1,225 +0,0 @@
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRetries = 3
|
||||
maxResponseBody = 1 << 20 // 1 MB
|
||||
defaultTimeout = 15 * time.Second
|
||||
defaultCacheTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
// Client is an HTTP client for the TheIntroDB /v3/media endpoint. Each
|
||||
// instance has its own rate limiter and response cache; concurrent fetches
|
||||
// for the same lookup key collapse to a single HTTP round trip via the cache.
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
mu sync.RWMutex
|
||||
apiKey string
|
||||
baseURL string
|
||||
limiter *rate.Limiter
|
||||
cache *cache.TTLCache[*mediaResponse]
|
||||
cacheTTL time.Duration
|
||||
}
|
||||
|
||||
// NewClient builds a Client with the canonical rate limit and cache TTL.
|
||||
// The apiKey may be empty — TheIntroDB serves read traffic without a key,
|
||||
// the key only gates access to the caller's own pending submissions.
|
||||
func NewClient(apiKey string) *Client {
|
||||
return &Client{
|
||||
httpClient: &http.Client{Timeout: defaultTimeout},
|
||||
apiKey: strings.TrimSpace(apiKey),
|
||||
baseURL: DefaultBaseURL,
|
||||
// TheIntroDB documents 30 requests / 10 seconds per IP. We stay
|
||||
// conservatively below that: 2 req/s sustained, burst 5.
|
||||
limiter: rate.NewLimiter(2, 5),
|
||||
cache: cache.NewTTLCache[*mediaResponse](),
|
||||
cacheTTL: defaultCacheTTL,
|
||||
}
|
||||
}
|
||||
|
||||
// SetBaseURL overrides the API base URL (used by tests).
|
||||
func (c *Client) SetBaseURL(u string) {
|
||||
c.mu.Lock()
|
||||
c.baseURL = u
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetAPIKey rotates the bearer token in-place. Safe to call concurrently
|
||||
// with in-flight requests; subsequent requests use the new key.
|
||||
func (c *Client) SetAPIKey(apiKey string) {
|
||||
c.mu.Lock()
|
||||
c.apiKey = strings.TrimSpace(apiKey)
|
||||
c.mu.Unlock()
|
||||
}
|
||||
|
||||
// Close releases the background sweeper goroutine inside the response cache.
|
||||
func (c *Client) Close() {
|
||||
if c.cache != nil {
|
||||
c.cache.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// FetchEpisode looks up segment timestamps for a TV episode.
|
||||
// At least one of tmdbID or imdbID must be non-empty.
|
||||
func (c *Client) FetchEpisode(ctx context.Context, tmdbID, imdbID string, season, episode int, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id or imdb_id required")
|
||||
}
|
||||
if season <= 0 || episode <= 0 {
|
||||
return nil, fmt.Errorf("introdb: episode lookup requires season and episode > 0 (got %d/%d)", season, episode)
|
||||
}
|
||||
q := url.Values{}
|
||||
if tmdbID != "" {
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
} else {
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
q.Set("season", strconv.Itoa(season))
|
||||
q.Set("episode", strconv.Itoa(episode))
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyEpisode(tmdbID, imdbID, season, episode, durationMS))
|
||||
}
|
||||
|
||||
// FetchMovie looks up segment timestamps for a movie.
|
||||
// At least one of tmdbID or imdbID must be non-empty.
|
||||
func (c *Client) FetchMovie(ctx context.Context, tmdbID, imdbID string, durationMS int64) (*mediaResponse, error) {
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return nil, fmt.Errorf("introdb: tmdb_id or imdb_id required")
|
||||
}
|
||||
q := url.Values{}
|
||||
if tmdbID != "" {
|
||||
q.Set("tmdb_id", tmdbID)
|
||||
} else {
|
||||
q.Set("imdb_id", imdbID)
|
||||
}
|
||||
if durationMS > 0 {
|
||||
q.Set("duration_ms", strconv.FormatInt(durationMS, 10))
|
||||
}
|
||||
return c.fetch(ctx, q, cacheKeyMovie(tmdbID, imdbID, durationMS))
|
||||
}
|
||||
|
||||
func (c *Client) fetch(ctx context.Context, q url.Values, key string) (*mediaResponse, error) {
|
||||
if cached, ok := c.cache.Get(key); ok {
|
||||
return cached, nil
|
||||
}
|
||||
|
||||
if err := c.limiter.Wait(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
c.mu.RLock()
|
||||
baseURL := c.baseURL
|
||||
apiKey := c.apiKey
|
||||
c.mu.RUnlock()
|
||||
|
||||
reqURL := baseURL + "/media?" + q.Encode()
|
||||
|
||||
for attempt := 0; attempt <= maxRetries; attempt++ {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("introdb: create request: %w", err)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("User-Agent", "Silo-Server/markers")
|
||||
if apiKey != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+apiKey)
|
||||
}
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("introdb: request failed: %w", err)
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotFound {
|
||||
resp.Body.Close()
|
||||
// Cache negatives too so the next playback start doesn't trigger
|
||||
// another fetch for known-empty content.
|
||||
c.cache.Set(key, nil, c.cacheTTL)
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusTooManyRequests {
|
||||
resp.Body.Close()
|
||||
if attempt < maxRetries {
|
||||
backoff := retryAfterOrDefault(resp, attempt)
|
||||
select {
|
||||
case <-time.After(backoff):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: rate limited after %d retries", maxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 500 {
|
||||
resp.Body.Close()
|
||||
if attempt < maxRetries {
|
||||
backoff := time.Duration(1<<attempt) * time.Second
|
||||
select {
|
||||
case <-time.After(backoff):
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
continue
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: server error %d after %d retries", resp.StatusCode, maxRetries)
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseBody))
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("introdb: HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
|
||||
}
|
||||
|
||||
var out mediaResponse
|
||||
decodeErr := json.NewDecoder(io.LimitReader(resp.Body, maxResponseBody)).Decode(&out)
|
||||
resp.Body.Close()
|
||||
if decodeErr != nil {
|
||||
return nil, fmt.Errorf("introdb: decode response: %w", decodeErr)
|
||||
}
|
||||
c.cache.Set(key, &out, c.cacheTTL)
|
||||
return &out, nil
|
||||
}
|
||||
return nil, fmt.Errorf("introdb: max retries exceeded")
|
||||
}
|
||||
|
||||
func retryAfterOrDefault(resp *http.Response, attempt int) time.Duration {
|
||||
if val := resp.Header.Get("Retry-After"); val != "" {
|
||||
if secs, err := strconv.Atoi(val); err == nil && secs > 0 {
|
||||
return time.Duration(secs) * time.Second
|
||||
}
|
||||
}
|
||||
return time.Duration(1<<attempt) * time.Second
|
||||
}
|
||||
|
||||
func cacheKeyEpisode(tmdbID, imdbID string, season, episode int, durationMS int64) string {
|
||||
if tmdbID != "" {
|
||||
return fmt.Sprintf("tmdb:%s:s%de%d:d%d", tmdbID, season, episode, durationMS)
|
||||
}
|
||||
return fmt.Sprintf("imdb:%s:s%de%d:d%d", imdbID, season, episode, durationMS)
|
||||
}
|
||||
|
||||
func cacheKeyMovie(tmdbID, imdbID string, durationMS int64) string {
|
||||
if tmdbID != "" {
|
||||
return fmt.Sprintf("tmdb:movie:%s:d%d", tmdbID, durationMS)
|
||||
}
|
||||
return fmt.Sprintf("imdb:movie:%s:d%d", imdbID, durationMS)
|
||||
}
|
||||
@@ -1,116 +0,0 @@
|
||||
package introdb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// Provider implements markers.Provider against the TheIntroDB API.
|
||||
// Movies are looked up by TMDB or IMDB ID; episodes additionally need a
|
||||
// season and episode number. The provider returns at most one Marker per
|
||||
// segment kind — when TheIntroDB has multiple candidate ranges (multiple
|
||||
// release versions), we pick the first usable one.
|
||||
type Provider struct {
|
||||
client *Client
|
||||
}
|
||||
|
||||
// NewProvider constructs a Provider backed by the supplied client. Pass an
|
||||
// already-configured *Client (from NewClient) so callers control the API
|
||||
// key, base URL, and lifecycle.
|
||||
func NewProvider(client *Client) *Provider {
|
||||
return &Provider{client: client}
|
||||
}
|
||||
|
||||
// ID satisfies markers.Provider; it returns the canonical provider tag
|
||||
// stored in *_markers_provider columns and emitted in telemetry.
|
||||
func (p *Provider) ID() string { return ProviderID }
|
||||
|
||||
// FetchMarkers issues a single GET /v3/media call and converts the
|
||||
// response into a markers.Result. A nil Provider (or one with no usable
|
||||
// IDs) returns an empty result rather than an error so callers can
|
||||
// chain providers via the Registry.
|
||||
func (p *Provider) FetchMarkers(ctx context.Context, req markers.Request) (markers.Result, error) {
|
||||
if p == nil || p.client == nil {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
tmdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyTMDB])
|
||||
imdbID := strings.TrimSpace(req.ExternalIDs[markers.ExternalIDKeyIMDB])
|
||||
if tmdbID == "" && imdbID == "" {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
durationMS := int64(req.Duration / time.Millisecond)
|
||||
|
||||
var resp *mediaResponse
|
||||
var err error
|
||||
switch req.Kind {
|
||||
case markers.ItemKindEpisode:
|
||||
if req.SeasonNumber <= 0 || req.EpisodeNumber <= 0 {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
resp, err = p.client.FetchEpisode(ctx, tmdbID, imdbID, req.SeasonNumber, req.EpisodeNumber, durationMS)
|
||||
case markers.ItemKindMovie:
|
||||
resp, err = p.client.FetchMovie(ctx, tmdbID, imdbID, durationMS)
|
||||
default:
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return markers.Result{}, err
|
||||
}
|
||||
if resp == nil {
|
||||
return markers.Result{}, nil
|
||||
}
|
||||
|
||||
result := markers.Result{
|
||||
ProviderID: ProviderID,
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Algorithm: Algorithm,
|
||||
}
|
||||
if m, ok := pickMarker(resp.Intro, markers.MarkerKindIntro, req.Duration, true); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Credits, markers.MarkerKindCredits, req.Duration, false); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Recap, markers.MarkerKindRecap, req.Duration, true); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
if m, ok := pickMarker(resp.Preview, markers.MarkerKindPreview, req.Duration, false); ok {
|
||||
result.Markers = append(result.Markers, m)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// pickMarker selects the first usable segment from a TheIntroDB response
|
||||
// array. `requireEnd` is true for segments where the end timestamp is the
|
||||
// load-bearing field (intro, recap) — they're allowed to start at 0 if
|
||||
// `start_ms` is omitted. For trailing segments (credits, preview) the
|
||||
// start is required but the end defaults to the file duration.
|
||||
func pickMarker(stamps []segmentTimestamps, kind markers.MarkerKind, totalDuration time.Duration, requireEnd bool) (markers.Marker, bool) {
|
||||
for _, s := range stamps {
|
||||
start := time.Duration(0)
|
||||
end := totalDuration
|
||||
if s.StartMs != nil {
|
||||
start = time.Duration(*s.StartMs) * time.Millisecond
|
||||
}
|
||||
if s.EndMs != nil {
|
||||
end = time.Duration(*s.EndMs) * time.Millisecond
|
||||
}
|
||||
if requireEnd && s.EndMs == nil {
|
||||
continue
|
||||
}
|
||||
if !requireEnd && s.StartMs == nil {
|
||||
continue
|
||||
}
|
||||
if end <= start {
|
||||
continue
|
||||
}
|
||||
return markers.Marker{Kind: kind, Start: start, End: end, Confidence: 0.9}, true
|
||||
}
|
||||
return markers.Marker{}, false
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
// Package introdb implements a markers.Provider against the public
|
||||
// TheIntroDB API (https://theintrodb.org). The provider is read-only:
|
||||
// it fetches intro/recap/credits/preview timestamps for episodes and
|
||||
// movies. Submissions are intentionally not supported.
|
||||
package introdb
|
||||
|
||||
// ProviderID is the canonical identifier stored in
|
||||
// media_files.*_markers_provider for markers sourced from TheIntroDB.
|
||||
const ProviderID = "introdb"
|
||||
|
||||
// Algorithm is the algorithm tag written alongside markers. The version
|
||||
// suffix lets us invalidate or refresh markers if the upstream contract
|
||||
// changes.
|
||||
const Algorithm = "introdb:v3"
|
||||
|
||||
// DefaultBaseURL is the production TheIntroDB v3 endpoint. Overridable
|
||||
// in tests via Client.SetBaseURL.
|
||||
const DefaultBaseURL = "https://api.theintrodb.org/v3"
|
||||
|
||||
// mediaResponse mirrors the JSON shape returned by GET /v3/media.
|
||||
// Each segment kind is an array of zero or more entries; absent fields
|
||||
// are decoded as empty slices via Go's zero-value semantics.
|
||||
type mediaResponse struct {
|
||||
TmdbID int `json:"tmdb_id"`
|
||||
Type string `json:"type"`
|
||||
Season *int `json:"season,omitempty"`
|
||||
Episode *int `json:"episode,omitempty"`
|
||||
Intro []segmentTimestamps `json:"intro,omitempty"`
|
||||
Recap []segmentTimestamps `json:"recap,omitempty"`
|
||||
Credits []segmentTimestamps `json:"credits,omitempty"`
|
||||
Preview []segmentTimestamps `json:"preview,omitempty"`
|
||||
}
|
||||
|
||||
// segmentTimestamps is the per-occurrence shape returned by TheIntroDB.
|
||||
// Either bound may be nil — for intro/recap, start may be omitted (segment
|
||||
// begins at file start); for credits/preview, end may be omitted (segment
|
||||
// runs to file end).
|
||||
type segmentTimestamps struct {
|
||||
StartMs *int64 `json:"start_ms,omitempty"`
|
||||
EndMs *int64 `json:"end_ms,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,429 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"google.golang.org/genproto/googleapis/rpc/errdetails"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
"google.golang.org/protobuf/types/known/structpb"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/pluginhost"
|
||||
)
|
||||
|
||||
const pluginMarkerAlgorithm = "plugin:marker_provider.v1"
|
||||
|
||||
type pluginMarkerClient interface {
|
||||
FetchMarkers(ctx context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error)
|
||||
SubmitMarker(ctx context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error)
|
||||
GetMarkerProviderStats(ctx context.Context, req *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error)
|
||||
}
|
||||
|
||||
type pluginMarkerResolver interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error)
|
||||
}
|
||||
|
||||
type pluginMarkerClientFactory func(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error)
|
||||
|
||||
type PluginResolverAdapter struct {
|
||||
inner interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
}
|
||||
}
|
||||
|
||||
func NewPluginResolverAdapter(svc interface {
|
||||
MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
}) *PluginResolverAdapter {
|
||||
if svc == nil {
|
||||
return nil
|
||||
}
|
||||
return &PluginResolverAdapter{inner: svc}
|
||||
}
|
||||
|
||||
func (a *PluginResolverAdapter) MarkerProviderClient(ctx context.Context, installationID int, capabilityID string) (pluginMarkerClient, error) {
|
||||
return a.inner.MarkerProviderClient(ctx, installationID, capabilityID)
|
||||
}
|
||||
|
||||
type PluginProviderOptions struct {
|
||||
InstallationID int
|
||||
CapabilityID string
|
||||
DisplayName string
|
||||
PluginID string
|
||||
RequiredExternalIDs []string
|
||||
}
|
||||
|
||||
// PluginProvider adapts a marker_provider.v1 plugin capability to Silo's
|
||||
// marker Provider/Submitter interfaces.
|
||||
type PluginProvider struct {
|
||||
installationID int
|
||||
capabilityID string
|
||||
displayName string
|
||||
pluginID string
|
||||
requiredExternalIDs []string
|
||||
clientFactory pluginMarkerClientFactory
|
||||
}
|
||||
|
||||
var _ Submitter = (*PluginProvider)(nil)
|
||||
|
||||
func NewPluginProvider(opts PluginProviderOptions, resolver pluginMarkerResolver) (*PluginProvider, error) {
|
||||
if resolver == nil {
|
||||
return nil, fmt.Errorf("plugin marker resolver is required")
|
||||
}
|
||||
return NewPluginProviderWithClientFactory(opts, resolver.MarkerProviderClient)
|
||||
}
|
||||
|
||||
func NewPluginProviderWithClientFactory(opts PluginProviderOptions, clientFactory pluginMarkerClientFactory) (*PluginProvider, error) {
|
||||
if opts.InstallationID <= 0 {
|
||||
return nil, fmt.Errorf("plugin marker installation id is required")
|
||||
}
|
||||
if strings.TrimSpace(opts.CapabilityID) == "" {
|
||||
return nil, fmt.Errorf("plugin marker capability id is required")
|
||||
}
|
||||
if clientFactory == nil {
|
||||
return nil, fmt.Errorf("plugin marker client factory is required")
|
||||
}
|
||||
displayName := strings.TrimSpace(opts.DisplayName)
|
||||
if displayName == "" {
|
||||
displayName = opts.CapabilityID
|
||||
}
|
||||
requiredIDs := normalizeExternalIDKeys(opts.RequiredExternalIDs)
|
||||
return &PluginProvider{
|
||||
installationID: opts.InstallationID,
|
||||
capabilityID: strings.TrimSpace(opts.CapabilityID),
|
||||
displayName: displayName,
|
||||
pluginID: strings.TrimSpace(opts.PluginID),
|
||||
requiredExternalIDs: requiredIDs,
|
||||
clientFactory: clientFactory,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func PluginProviderID(installationID int, capabilityID string) string {
|
||||
return fmt.Sprintf("plugin:%d:%s", installationID, strings.TrimSpace(capabilityID))
|
||||
}
|
||||
|
||||
func (p *PluginProvider) ID() string {
|
||||
if p == nil {
|
||||
return ""
|
||||
}
|
||||
return PluginProviderID(p.installationID, p.capabilityID)
|
||||
}
|
||||
|
||||
func (p *PluginProvider) ProviderDescription() ProviderDescriptor {
|
||||
if p == nil {
|
||||
return ProviderDescriptor{}
|
||||
}
|
||||
return ProviderDescriptor{
|
||||
ID: p.ID(),
|
||||
DisplayName: p.displayName,
|
||||
SourceType: ProviderSourcePlugin,
|
||||
PluginID: p.pluginID,
|
||||
PluginInstallationID: p.installationID,
|
||||
CapabilityID: p.capabilityID,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PluginProvider) SubmissionRequirements() SubmissionRequirements {
|
||||
if p == nil || len(p.requiredExternalIDs) == 0 {
|
||||
return SubmissionRequirements{}
|
||||
}
|
||||
return SubmissionRequirements{RequiredExternalIDs: append([]string(nil), p.requiredExternalIDs...)}
|
||||
}
|
||||
|
||||
func (p *PluginProvider) FetchMarkers(ctx context.Context, req Request) (Result, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
response, err := client.FetchMarkers(ctx, &pluginv1.FetchMarkersRequest{
|
||||
ItemType: itemTypeName(req.Kind),
|
||||
ExternalIds: externalIDsProto(req.ExternalIDs),
|
||||
SeasonNumber: int32(req.SeasonNumber),
|
||||
EpisodeNumber: int32(req.EpisodeNumber),
|
||||
DurationSeconds: req.Duration.Seconds(),
|
||||
})
|
||||
if err != nil {
|
||||
return Result{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return Result{}, nil
|
||||
}
|
||||
|
||||
result := Result{
|
||||
SourceClass: models.MarkerSourcePlugin,
|
||||
ProviderID: p.ID(),
|
||||
Algorithm: pluginMarkerAlgorithm,
|
||||
}
|
||||
for _, segment := range response.GetMarkers() {
|
||||
marker, ok := markerFromPluginSegment(segment, req.Duration)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
marker.SourceClass = models.MarkerSourcePlugin
|
||||
marker.ProviderID = p.ID()
|
||||
if marker.Algorithm == "" {
|
||||
marker.Algorithm = pluginMarkerAlgorithm
|
||||
}
|
||||
result.Markers = append(result.Markers, marker)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (p *PluginProvider) SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return SubmissionResult{}, fmt.Errorf("plugin marker provider not configured")
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return SubmissionResult{}, err
|
||||
}
|
||||
body := &pluginv1.SubmitMarkerRequest{
|
||||
ItemType: itemTypeName(req.Kind),
|
||||
ExternalIds: externalIDsProto(req.ExternalIDs),
|
||||
SeasonNumber: int32(req.SeasonNumber),
|
||||
EpisodeNumber: int32(req.EpisodeNumber),
|
||||
Segment: markerKindName(req.Segment),
|
||||
DurationSeconds: req.Duration.Seconds(),
|
||||
}
|
||||
if req.Start != nil {
|
||||
start := req.Start.Seconds()
|
||||
body.StartSeconds = &start
|
||||
}
|
||||
if req.End != nil {
|
||||
end := req.End.Seconds()
|
||||
body.EndSeconds = &end
|
||||
}
|
||||
response, err := client.SubmitMarker(ctx, body)
|
||||
if err != nil {
|
||||
return SubmissionResult{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return SubmissionResult{Status: SubmissionStatusPending}, nil
|
||||
}
|
||||
status := strings.TrimSpace(response.GetStatus())
|
||||
if status == "" {
|
||||
status = SubmissionStatusPending
|
||||
}
|
||||
return SubmissionResult{ID: response.GetSubmissionId(), Status: status, Weight: response.GetWeight()}, nil
|
||||
}
|
||||
|
||||
func (p *PluginProvider) FetchUserStats(ctx context.Context) (UserStats, error) {
|
||||
if p == nil || p.clientFactory == nil {
|
||||
return UserStats{}, fmt.Errorf("plugin marker provider not configured")
|
||||
}
|
||||
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
|
||||
if err != nil {
|
||||
return UserStats{}, err
|
||||
}
|
||||
response, err := client.GetMarkerProviderStats(ctx, &pluginv1.GetMarkerProviderStatsRequest{})
|
||||
if err != nil {
|
||||
return UserStats{}, pluginProviderError(p.ID(), err)
|
||||
}
|
||||
if response == nil {
|
||||
return UserStats{}, nil
|
||||
}
|
||||
return UserStats{
|
||||
Total: int(response.GetTotal()),
|
||||
Accepted: int(response.GetAccepted()),
|
||||
Pending: int(response.GetPending()),
|
||||
Rejected: int(response.GetRejected()),
|
||||
AcceptanceRate: response.GetAcceptanceRate(),
|
||||
CurrentStreak: int(response.GetCurrentStreak()),
|
||||
BestStreak: int(response.GetBestStreak()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func markerFromPluginSegment(segment *pluginv1.MarkerSegment, duration time.Duration) (Marker, bool) {
|
||||
if segment == nil {
|
||||
return Marker{}, false
|
||||
}
|
||||
kind, ok := markerKindFromName(segment.GetSegment())
|
||||
if !ok {
|
||||
return Marker{}, false
|
||||
}
|
||||
start := time.Duration(0)
|
||||
end := duration
|
||||
if segment.StartSeconds != nil {
|
||||
start = secondsDuration(segment.GetStartSeconds())
|
||||
}
|
||||
if segment.EndSeconds != nil {
|
||||
end = secondsDuration(segment.GetEndSeconds())
|
||||
}
|
||||
if start < 0 {
|
||||
return Marker{}, false
|
||||
}
|
||||
if duration > 0 && (start >= duration || end > duration) {
|
||||
return Marker{}, false
|
||||
}
|
||||
if end <= start {
|
||||
return Marker{}, false
|
||||
}
|
||||
return Marker{
|
||||
Kind: kind,
|
||||
Start: start,
|
||||
End: end,
|
||||
Confidence: segment.GetConfidence(),
|
||||
SubmissionCount: int(segment.GetSubmissionCount()),
|
||||
Algorithm: strings.TrimSpace(segment.GetAlgorithm()),
|
||||
}, true
|
||||
}
|
||||
|
||||
func externalIDsProto(ids map[string]string) *pluginv1.MarkerExternalIDs {
|
||||
if len(ids) == 0 {
|
||||
return &pluginv1.MarkerExternalIDs{}
|
||||
}
|
||||
providerIDs := make(map[string]any, len(ids))
|
||||
for key, value := range ids {
|
||||
if strings.TrimSpace(value) != "" {
|
||||
providerIDs[key] = value
|
||||
}
|
||||
}
|
||||
structIDs, _ := structpb.NewStruct(providerIDs)
|
||||
return &pluginv1.MarkerExternalIDs{
|
||||
TmdbId: strings.TrimSpace(ids[ExternalIDKeyTMDB]),
|
||||
ImdbId: strings.TrimSpace(ids[ExternalIDKeyIMDB]),
|
||||
TvdbId: strings.TrimSpace(ids[ExternalIDKeyTVDB]),
|
||||
ProviderIds: structIDs,
|
||||
}
|
||||
}
|
||||
|
||||
func itemTypeName(kind ItemKind) string {
|
||||
switch kind {
|
||||
case ItemKindEpisode:
|
||||
return "episode"
|
||||
case ItemKindMovie:
|
||||
return "movie"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func markerKindName(kind MarkerKind) string {
|
||||
switch kind {
|
||||
case MarkerKindIntro:
|
||||
return "intro"
|
||||
case MarkerKindCredits:
|
||||
return "credits"
|
||||
case MarkerKindRecap:
|
||||
return "recap"
|
||||
case MarkerKindPreview:
|
||||
return "preview"
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func markerKindFromName(name string) (MarkerKind, bool) {
|
||||
switch strings.ToLower(strings.TrimSpace(name)) {
|
||||
case "intro":
|
||||
return MarkerKindIntro, true
|
||||
case "credits":
|
||||
return MarkerKindCredits, true
|
||||
case "recap":
|
||||
return MarkerKindRecap, true
|
||||
case "preview":
|
||||
return MarkerKindPreview, true
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
|
||||
func secondsDuration(seconds float64) time.Duration {
|
||||
return time.Duration(seconds * float64(time.Second))
|
||||
}
|
||||
|
||||
func normalizeExternalIDKeys(keys []string) []string {
|
||||
seen := map[string]struct{}{}
|
||||
out := make([]string, 0, len(keys))
|
||||
for _, key := range keys {
|
||||
key = strings.ToLower(strings.TrimSpace(key))
|
||||
if key == "" {
|
||||
continue
|
||||
}
|
||||
switch key {
|
||||
case ExternalIDKeyTMDB, ExternalIDKeyIMDB, ExternalIDKeyTVDB:
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[key]; ok {
|
||||
continue
|
||||
}
|
||||
seen[key] = struct{}{}
|
||||
out = append(out, key)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pluginProviderError(providerID string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
st, ok := status.FromError(err)
|
||||
if !ok || st.Code() != codes.ResourceExhausted {
|
||||
return err
|
||||
}
|
||||
retryAfter := time.Duration(0)
|
||||
for _, detail := range st.Details() {
|
||||
if retry, ok := detail.(*errdetails.RetryInfo); ok && retry.GetRetryDelay() != nil {
|
||||
retryAfter = retry.GetRetryDelay().AsDuration()
|
||||
break
|
||||
}
|
||||
}
|
||||
return &RetryAfterError{Provider: providerID, RetryAfter: retryAfter, Message: err.Error()}
|
||||
}
|
||||
|
||||
func PluginRequiredExternalIDsFromMetadata(metadata map[string]any) []string {
|
||||
raw, ok := metadata["required_external_ids"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case []string:
|
||||
return normalizeExternalIDKeys(typed)
|
||||
case []any:
|
||||
keys := make([]string, 0, len(typed))
|
||||
for _, value := range typed {
|
||||
if s, ok := value.(string); ok {
|
||||
keys = append(keys, s)
|
||||
}
|
||||
}
|
||||
return normalizeExternalIDKeys(keys)
|
||||
case string:
|
||||
if strings.TrimSpace(typed) == "" {
|
||||
return nil
|
||||
}
|
||||
return normalizeExternalIDKeys(strings.Split(typed, ","))
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func PluginDefaultFetchPriorityFromMetadata(metadata map[string]any) (int, bool) {
|
||||
raw, ok := metadata["default_fetch_priority"]
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
switch typed := raw.(type) {
|
||||
case int:
|
||||
return typed, true
|
||||
case int32:
|
||||
return int(typed), true
|
||||
case int64:
|
||||
return int(typed), true
|
||||
case float64:
|
||||
return int(typed), true
|
||||
case string:
|
||||
parsed, err := strconv.Atoi(strings.TrimSpace(typed))
|
||||
return parsed, err == nil
|
||||
default:
|
||||
return 0, false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakePluginMarkerClient struct {
|
||||
fetchResp *pluginv1.FetchMarkersResponse
|
||||
fetchReq *pluginv1.FetchMarkersRequest
|
||||
submitReq *pluginv1.SubmitMarkerRequest
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) FetchMarkers(_ context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error) {
|
||||
f.fetchReq = req
|
||||
return f.fetchResp, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) SubmitMarker(_ context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error) {
|
||||
f.submitReq = req
|
||||
return &pluginv1.SubmitMarkerResponse{SubmissionId: "sub1", Status: SubmissionStatusPending, Weight: 2}, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginMarkerClient) GetMarkerProviderStats(context.Context, *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error) {
|
||||
return &pluginv1.MarkerProviderStatsResponse{Total: 3, Accepted: 2, Pending: 1, AcceptanceRate: 0.66}, nil
|
||||
}
|
||||
|
||||
func TestPluginProviderFetchMapsAllSegments(t *testing.T) {
|
||||
start10, end60 := 10.0, 60.0
|
||||
creditsStart := 1700.0
|
||||
previewStart := 1750.0
|
||||
client := &fakePluginMarkerClient{fetchResp: &pluginv1.FetchMarkersResponse{Markers: []*pluginv1.MarkerSegment{
|
||||
{Segment: "intro", StartSeconds: &start10, EndSeconds: &end60, Confidence: 0.8, SubmissionCount: 2, Algorithm: "intro:v1"},
|
||||
{Segment: "credits", StartSeconds: &creditsStart, Confidence: 0.9, SubmissionCount: 3},
|
||||
{Segment: "recap", EndSeconds: &start10, Confidence: 0.7},
|
||||
{Segment: "preview", StartSeconds: &previewStart, Confidence: 0.6},
|
||||
}}}
|
||||
provider, err := NewPluginProviderWithClientFactory(PluginProviderOptions{
|
||||
InstallationID: 12,
|
||||
CapabilityID: "markers",
|
||||
DisplayName: "Markers",
|
||||
PluginID: "silo.markers",
|
||||
}, func(context.Context, int, string) (pluginMarkerClient, error) {
|
||||
return client, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPluginProviderWithClientFactory: %v", err)
|
||||
}
|
||||
|
||||
res, err := provider.FetchMarkers(context.Background(), Request{
|
||||
Kind: ItemKindEpisode,
|
||||
ExternalIDs: map[string]string{ExternalIDKeyTVDB: "777"},
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 2,
|
||||
Duration: 1800 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMarkers: %v", err)
|
||||
}
|
||||
if client.fetchReq.GetItemType() != "episode" || client.fetchReq.GetExternalIds().GetTvdbId() != "777" {
|
||||
t.Fatalf("fetch request = %+v", client.fetchReq)
|
||||
}
|
||||
if res.SourceClass != models.MarkerSourcePlugin || res.ProviderID != "plugin:12:markers" {
|
||||
t.Fatalf("result provenance = source %q provider %q", res.SourceClass, res.ProviderID)
|
||||
}
|
||||
byKind := map[MarkerKind]Marker{}
|
||||
for _, marker := range res.Markers {
|
||||
byKind[marker.Kind] = marker
|
||||
if marker.SourceClass != models.MarkerSourcePlugin || marker.ProviderID != "plugin:12:markers" {
|
||||
t.Fatalf("marker provenance = %+v", marker)
|
||||
}
|
||||
}
|
||||
if len(byKind) != 4 {
|
||||
t.Fatalf("mapped %d markers, want 4: %+v", len(byKind), res.Markers)
|
||||
}
|
||||
if got := byKind[MarkerKindCredits]; got.End != 1800*time.Second {
|
||||
t.Fatalf("credits end = %s, want duration default", got.End)
|
||||
}
|
||||
if got := byKind[MarkerKindRecap]; got.Start != 0 {
|
||||
t.Fatalf("recap start = %s, want zero default", got.Start)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginProviderRejectsOutOfBoundsSegments(t *testing.T) {
|
||||
negativeStart := -1.0
|
||||
start10, end61 := 10.0, 61.0
|
||||
validStart := 50.0
|
||||
duration := time.Minute
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
segment *pluginv1.MarkerSegment
|
||||
wantOK bool
|
||||
wantEnd time.Duration
|
||||
}{
|
||||
{
|
||||
name: "negative start",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "intro", StartSeconds: &negativeStart},
|
||||
},
|
||||
{
|
||||
name: "end past duration",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "intro", StartSeconds: &start10, EndSeconds: &end61},
|
||||
},
|
||||
{
|
||||
name: "default end uses duration",
|
||||
segment: &pluginv1.MarkerSegment{Segment: "credits", StartSeconds: &validStart},
|
||||
wantOK: true,
|
||||
wantEnd: duration,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
marker, ok := markerFromPluginSegment(tt.segment, duration)
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("ok = %v, want %v", ok, tt.wantOK)
|
||||
}
|
||||
if tt.wantOK && marker.End != tt.wantEnd {
|
||||
t.Fatalf("end = %s, want %s", marker.End, tt.wantEnd)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPluginProviderSubmitMapsRequest(t *testing.T) {
|
||||
client := &fakePluginMarkerClient{}
|
||||
provider, err := NewPluginProviderWithClientFactory(PluginProviderOptions{
|
||||
InstallationID: 12,
|
||||
CapabilityID: "markers",
|
||||
}, func(context.Context, int, string) (pluginMarkerClient, error) {
|
||||
return client, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("NewPluginProviderWithClientFactory: %v", err)
|
||||
}
|
||||
start, end := 5*time.Second, 30*time.Second
|
||||
result, err := provider.SubmitMarker(context.Background(), SubmissionRequest{
|
||||
Kind: ItemKindMovie,
|
||||
ExternalIDs: map[string]string{ExternalIDKeyIMDB: "tt1"},
|
||||
Segment: MarkerKindIntro,
|
||||
Start: &start,
|
||||
End: &end,
|
||||
Duration: 90 * time.Minute,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SubmitMarker: %v", err)
|
||||
}
|
||||
if result.ID != "sub1" || result.Status != SubmissionStatusPending || result.Weight != 2 {
|
||||
t.Fatalf("submit result = %+v", result)
|
||||
}
|
||||
if client.submitReq.GetItemType() != "movie" || client.submitReq.GetExternalIds().GetImdbId() != "tt1" {
|
||||
t.Fatalf("submit request identity = %+v", client.submitReq)
|
||||
}
|
||||
if client.submitReq.GetSegment() != "intro" || client.submitReq.GetStartSeconds() != 5 || client.submitReq.GetEndSeconds() != 30 {
|
||||
t.Fatalf("submit request segment = %+v", client.submitReq)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package markers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ProviderConfig is the per-provider behavior row from marker_provider_config.
|
||||
// fetch_* control multi-source read dispatch; contribute_* gate submission and
|
||||
// default off so contribution is opt-in per provider.
|
||||
type ProviderConfig struct {
|
||||
Provider string
|
||||
FetchEnabled bool
|
||||
FetchPriority int
|
||||
ContributeEnabled bool
|
||||
ContributeAutoLocal bool
|
||||
ContributeMinConfidence float64
|
||||
}
|
||||
|
||||
const providerConfigColumns = `provider, fetch_enabled, fetch_priority, contribute_enabled, contribute_auto_local, contribute_min_confidence`
|
||||
|
||||
// ProviderConfigStore is a cached read/write facade over marker_provider_config.
|
||||
// Reads serve from an in-memory snapshot; call Reload at startup and after a
|
||||
// settings-changed event. Update writes through and refreshes the snapshot.
|
||||
type ProviderConfigStore struct {
|
||||
pool *pgxpool.Pool
|
||||
mu sync.RWMutex
|
||||
cache map[string]ProviderConfig
|
||||
}
|
||||
|
||||
// NewProviderConfigStore constructs a store backed by the supplied pool.
|
||||
func NewProviderConfigStore(pool *pgxpool.Pool) *ProviderConfigStore {
|
||||
return &ProviderConfigStore{pool: pool, cache: map[string]ProviderConfig{}}
|
||||
}
|
||||
|
||||
// Reload replaces the in-memory snapshot from the database.
|
||||
func (s *ProviderConfigStore) Reload(ctx context.Context) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return nil
|
||||
}
|
||||
rows, err := s.pool.Query(ctx, `SELECT `+providerConfigColumns+` FROM marker_provider_config`)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load marker provider config: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
next := make(map[string]ProviderConfig)
|
||||
for rows.Next() {
|
||||
var c ProviderConfig
|
||||
if err := rows.Scan(
|
||||
&c.Provider,
|
||||
&c.FetchEnabled,
|
||||
&c.FetchPriority,
|
||||
&c.ContributeEnabled,
|
||||
&c.ContributeAutoLocal,
|
||||
&c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("scan marker provider config: %w", err)
|
||||
}
|
||||
next[c.Provider] = c
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return fmt.Errorf("iterate marker provider config: %w", err)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
s.cache = next
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the config for a provider id from the snapshot.
|
||||
func (s *ProviderConfigStore) Get(provider string) (ProviderConfig, bool) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
c, ok := s.cache[provider]
|
||||
return c, ok
|
||||
}
|
||||
|
||||
// List returns all configs sorted by (fetch_priority asc, provider asc).
|
||||
func (s *ProviderConfigStore) List() []ProviderConfig {
|
||||
s.mu.RLock()
|
||||
out := make([]ProviderConfig, 0, len(s.cache))
|
||||
for _, c := range s.cache {
|
||||
out = append(out, c)
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].FetchPriority != out[j].FetchPriority {
|
||||
return out[i].FetchPriority < out[j].FetchPriority
|
||||
}
|
||||
return out[i].Provider < out[j].Provider
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// EnabledForFetch returns the fetch-enabled providers in priority order.
|
||||
func (s *ProviderConfigStore) EnabledForFetch() []ProviderConfig {
|
||||
all := s.List()
|
||||
out := make([]ProviderConfig, 0, len(all))
|
||||
for _, c := range all {
|
||||
if c.FetchEnabled {
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Update upserts a provider config row and refreshes the snapshot.
|
||||
func (s *ProviderConfigStore) Update(ctx context.Context, c ProviderConfig) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("marker provider config store unavailable")
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_provider_config (
|
||||
provider, fetch_enabled, fetch_priority,
|
||||
contribute_enabled, contribute_auto_local, contribute_min_confidence, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())
|
||||
ON CONFLICT (provider) DO UPDATE SET
|
||||
fetch_enabled = EXCLUDED.fetch_enabled,
|
||||
fetch_priority = EXCLUDED.fetch_priority,
|
||||
contribute_enabled = EXCLUDED.contribute_enabled,
|
||||
contribute_auto_local = EXCLUDED.contribute_auto_local,
|
||||
contribute_min_confidence = EXCLUDED.contribute_min_confidence,
|
||||
updated_at = now()`,
|
||||
c.Provider, c.FetchEnabled, c.FetchPriority,
|
||||
c.ContributeEnabled, c.ContributeAutoLocal, c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("update marker provider config: %w", err)
|
||||
}
|
||||
return s.Reload(ctx)
|
||||
}
|
||||
|
||||
// Ensure inserts a default provider config row if one does not already exist.
|
||||
// Existing rows are left untouched so admin choices survive plugin restarts and
|
||||
// upgrades.
|
||||
func (s *ProviderConfigStore) Ensure(ctx context.Context, c ProviderConfig) error {
|
||||
if s == nil || s.pool == nil {
|
||||
return fmt.Errorf("marker provider config store unavailable")
|
||||
}
|
||||
if _, ok := s.Get(c.Provider); ok {
|
||||
return nil
|
||||
}
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
INSERT INTO marker_provider_config (
|
||||
provider, fetch_enabled, fetch_priority,
|
||||
contribute_enabled, contribute_auto_local, contribute_min_confidence, updated_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, now())
|
||||
ON CONFLICT (provider) DO NOTHING`,
|
||||
c.Provider, c.FetchEnabled, c.FetchPriority,
|
||||
c.ContributeEnabled, c.ContributeAutoLocal, c.ContributeMinConfidence,
|
||||
); err != nil {
|
||||
return fmt.Errorf("ensure marker provider config: %w", err)
|
||||
}
|
||||
return s.Reload(ctx)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package markers
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestProviderConfigEnabledForFetchSortsAndFilters(t *testing.T) {
|
||||
s := &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"alpha": {Provider: "alpha", FetchEnabled: true, FetchPriority: 200},
|
||||
"bravo": {Provider: "bravo", FetchEnabled: true, FetchPriority: 100},
|
||||
"charlie": {Provider: "charlie", FetchEnabled: false, FetchPriority: 50},
|
||||
}}
|
||||
|
||||
got := s.EnabledForFetch()
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("EnabledForFetch returned %d providers, want 2 (charlie disabled)", len(got))
|
||||
}
|
||||
if got[0].Provider != "bravo" || got[1].Provider != "alpha" {
|
||||
t.Errorf("order = [%s %s], want [bravo alpha] by fetch_priority", got[0].Provider, got[1].Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderConfigGet(t *testing.T) {
|
||||
s := &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"introdb": {Provider: "introdb", FetchEnabled: true, ContributeEnabled: false},
|
||||
}}
|
||||
c, ok := s.Get("introdb")
|
||||
if !ok {
|
||||
t.Fatal("expected introdb config")
|
||||
}
|
||||
if c.ContributeEnabled {
|
||||
t.Error("contribute should default off")
|
||||
}
|
||||
if _, ok := s.Get("missing"); ok {
|
||||
t.Error("unexpected config for unknown provider")
|
||||
}
|
||||
}
|
||||
+299
-7
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
@@ -32,6 +33,36 @@ type Provider interface {
|
||||
FetchMarkers(ctx context.Context, req Request) (Result, error)
|
||||
}
|
||||
|
||||
const (
|
||||
ProviderSourceBuiltIn = "built_in"
|
||||
ProviderSourcePlugin = "plugin"
|
||||
)
|
||||
|
||||
// ProviderDescriptor is optional provider metadata surfaced by admin APIs.
|
||||
type ProviderDescriptor struct {
|
||||
ID string
|
||||
DisplayName string
|
||||
SourceType string
|
||||
PluginID string
|
||||
PluginInstallationID int
|
||||
CapabilityID string
|
||||
}
|
||||
|
||||
type DescribedProvider interface {
|
||||
ProviderDescription() ProviderDescriptor
|
||||
}
|
||||
|
||||
// SubmissionRequirements describes IDs the generic contribution path can
|
||||
// validate before calling a provider. Providers may still perform richer
|
||||
// validation inside SubmitMarker.
|
||||
type SubmissionRequirements struct {
|
||||
RequiredExternalIDs []string
|
||||
}
|
||||
|
||||
type SubmissionRequirementProvider interface {
|
||||
SubmissionRequirements() SubmissionRequirements
|
||||
}
|
||||
|
||||
type ItemKind int
|
||||
|
||||
const (
|
||||
@@ -72,15 +103,104 @@ type Result struct {
|
||||
}
|
||||
|
||||
type Marker struct {
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
Kind MarkerKind
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Confidence float64
|
||||
SubmissionCount int
|
||||
SourceClass string
|
||||
// ProviderID and Algorithm identify the source of this individual marker.
|
||||
// They are usually empty for a single-provider Result (the Result-level
|
||||
// SourceClass/ProviderID/Algorithm apply); FetchMerged sets them per marker
|
||||
// so a merged result records correct per-segment provenance.
|
||||
ProviderID string
|
||||
Algorithm string
|
||||
}
|
||||
|
||||
// SubmissionRequest is a single-segment contribution to an online marker
|
||||
// source. Start is nil when the segment begins at the start of the file
|
||||
// (intro/recap); End is nil when it runs to the end (credits/preview).
|
||||
type SubmissionRequest struct {
|
||||
Kind ItemKind
|
||||
ExternalIDs map[string]string
|
||||
SeasonNumber int
|
||||
EpisodeNumber int
|
||||
Segment MarkerKind
|
||||
Start *time.Duration
|
||||
End *time.Duration
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// SubmissionStatus values returned by a Submitter.
|
||||
const (
|
||||
SubmissionStatusPending = "pending"
|
||||
SubmissionStatusAccepted = "accepted"
|
||||
SubmissionStatusRejected = "rejected"
|
||||
)
|
||||
|
||||
// SubmissionResult is the provider's acknowledgement of a submitted segment.
|
||||
type SubmissionResult struct {
|
||||
ID string
|
||||
Status string
|
||||
Weight float64
|
||||
}
|
||||
|
||||
// UserStats is a contribution-account summary used to validate a key and show
|
||||
// contribution totals in the admin UI.
|
||||
type UserStats struct {
|
||||
Total int
|
||||
Accepted int
|
||||
Pending int
|
||||
Rejected int
|
||||
AcceptanceRate float64
|
||||
CurrentStreak int
|
||||
BestStreak int
|
||||
}
|
||||
|
||||
// RetryAfterError marks provider errors that should pause contribution work
|
||||
// until RetryAfter has elapsed, such as TheIntroDB usage-limit responses.
|
||||
type RetryAfterError struct {
|
||||
Provider string
|
||||
RetryAfter time.Duration
|
||||
Message string
|
||||
}
|
||||
|
||||
func (e *RetryAfterError) Error() string {
|
||||
if e == nil {
|
||||
return ""
|
||||
}
|
||||
if e.Message != "" {
|
||||
return e.Message
|
||||
}
|
||||
if e.Provider != "" {
|
||||
return fmt.Sprintf("%s: retry after %s", e.Provider, e.RetryAfter)
|
||||
}
|
||||
return fmt.Sprintf("retry after %s", e.RetryAfter)
|
||||
}
|
||||
|
||||
// RetryAfter extracts a provider backoff duration from err.
|
||||
func RetryAfter(err error) (time.Duration, bool) {
|
||||
var retryErr *RetryAfterError
|
||||
if errors.As(err, &retryErr) && retryErr != nil {
|
||||
return retryErr.RetryAfter, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Submitter is an optional capability implemented by providers that accept
|
||||
// marker contributions. The contribution service type-asserts a Provider to
|
||||
// Submitter; non-contributing providers simply don't implement it.
|
||||
type Submitter interface {
|
||||
Provider
|
||||
SubmitMarker(ctx context.Context, req SubmissionRequest) (SubmissionResult, error)
|
||||
FetchUserStats(ctx context.Context) (UserStats, error)
|
||||
}
|
||||
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
providers []Provider
|
||||
logger *slog.Logger
|
||||
config *ProviderConfigStore
|
||||
}
|
||||
|
||||
func NewRegistry(logger *slog.Logger) *Registry {
|
||||
@@ -98,6 +218,8 @@ func (r *Registry) Register(provider Provider) error {
|
||||
if id == "" {
|
||||
return fmt.Errorf("marker provider ID is required")
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for _, existing := range r.providers {
|
||||
if existing.ID() == id {
|
||||
return fmt.Errorf("marker provider %q already registered", id)
|
||||
@@ -108,7 +230,12 @@ func (r *Registry) Register(provider Provider) error {
|
||||
}
|
||||
|
||||
func (r *Registry) Providers() []Provider {
|
||||
if r == nil || len(r.providers) == 0 {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
if len(r.providers) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := make([]Provider, len(r.providers))
|
||||
@@ -116,13 +243,43 @@ func (r *Registry) Providers() []Provider {
|
||||
return out
|
||||
}
|
||||
|
||||
// SetProviders atomically replaces the registered provider set. It is used when
|
||||
// plugin lifecycle changes require rebuilding the marker-provider list.
|
||||
func (r *Registry) SetProviders(providers []Provider) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
|
||||
seen := make(map[string]struct{}, len(providers))
|
||||
next := make([]Provider, 0, len(providers))
|
||||
for _, provider := range providers {
|
||||
if provider == nil {
|
||||
return fmt.Errorf("marker provider is nil")
|
||||
}
|
||||
id := strings.TrimSpace(provider.ID())
|
||||
if id == "" {
|
||||
return fmt.Errorf("marker provider ID is required")
|
||||
}
|
||||
if _, ok := seen[id]; ok {
|
||||
return fmt.Errorf("marker provider %q already registered", id)
|
||||
}
|
||||
seen[id] = struct{}{}
|
||||
next = append(next, provider)
|
||||
}
|
||||
r.providers = next
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *Registry) FetchFirstHit(ctx context.Context, req Request) (Result, bool, error) {
|
||||
if r == nil || len(r.providers) == 0 {
|
||||
providers := r.Providers()
|
||||
if len(providers) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, provider := range r.providers {
|
||||
for _, provider := range providers {
|
||||
result, err := provider.FetchMarkers(ctx, req)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
@@ -144,6 +301,141 @@ func (r *Registry) FetchFirstHit(ctx context.Context, req Request) (Result, bool
|
||||
return Result{}, false, lastErr
|
||||
}
|
||||
|
||||
// UseConfigStore attaches a per-provider config store so FetchMerged consults
|
||||
// fetch_enabled / fetch_priority. Without one, all registered providers
|
||||
// participate in registration order.
|
||||
func (r *Registry) UseConfigStore(store *ProviderConfigStore) {
|
||||
if r != nil {
|
||||
r.config = store
|
||||
}
|
||||
}
|
||||
|
||||
// FetchMerged queries every fetch-enabled provider concurrently and keeps, per
|
||||
// segment kind, the best candidate — ranked by provider fetch priority (lower
|
||||
// preferred), then submission count, then confidence, then provider id for
|
||||
// deterministic output. Submission count and confidence are provider-local
|
||||
// quality signals; explicit admin priority is the cross-provider winner rule.
|
||||
// The winning markers are stamped with source/provider/algorithm so the write
|
||||
// path records correct per-segment provenance. With a single enabled provider
|
||||
// this returns the same result as FetchFirstHit.
|
||||
func (r *Registry) FetchMerged(ctx context.Context, req Request) (Result, bool, error) {
|
||||
if r == nil || len(r.Providers()) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
entries := r.fetchEntries()
|
||||
if len(entries) == 0 {
|
||||
return Result{}, false, nil
|
||||
}
|
||||
|
||||
type fetched struct {
|
||||
entry fetchEntry
|
||||
result Result
|
||||
err error
|
||||
}
|
||||
out := make([]fetched, len(entries))
|
||||
var wg sync.WaitGroup
|
||||
for i, e := range entries {
|
||||
wg.Add(1)
|
||||
go func(i int, e fetchEntry) {
|
||||
defer wg.Done()
|
||||
res, err := e.provider.FetchMarkers(ctx, req)
|
||||
out[i] = fetched{entry: e, result: res, err: err}
|
||||
}(i, e)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
best := make(map[MarkerKind]mergeCandidate)
|
||||
var lastErr error
|
||||
for _, f := range out {
|
||||
if f.err != nil {
|
||||
lastErr = f.err
|
||||
r.logProviderError(f.entry.provider.ID(), req, f.err)
|
||||
continue
|
||||
}
|
||||
for _, m := range f.result.Markers {
|
||||
m.SourceClass = firstNonEmpty(m.SourceClass, f.result.SourceClass, models.MarkerSourceOnline)
|
||||
m.ProviderID = firstNonEmpty(m.ProviderID, f.result.ProviderID, f.entry.provider.ID())
|
||||
m.Algorithm = firstNonEmpty(m.Algorithm, f.result.Algorithm)
|
||||
cand := mergeCandidate{marker: m, priority: f.entry.priority}
|
||||
if cur, ok := best[m.Kind]; !ok || cand.better(cur) {
|
||||
best[m.Kind] = cand
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(best) == 0 {
|
||||
return Result{}, false, lastErr
|
||||
}
|
||||
|
||||
merged := Result{SourceClass: models.MarkerSourceOnline}
|
||||
for _, kind := range []MarkerKind{MarkerKindIntro, MarkerKindCredits, MarkerKindRecap, MarkerKindPreview} {
|
||||
if cand, ok := best[kind]; ok {
|
||||
merged.Markers = append(merged.Markers, cand.marker)
|
||||
}
|
||||
}
|
||||
return merged, true, nil
|
||||
}
|
||||
|
||||
type fetchEntry struct {
|
||||
provider Provider
|
||||
priority int
|
||||
}
|
||||
|
||||
// fetchEntries returns the providers to query with their priorities. When a
|
||||
// config store is set only fetch-enabled providers participate, ordered by
|
||||
// fetch_priority; otherwise all providers participate in registration order.
|
||||
func (r *Registry) fetchEntries() []fetchEntry {
|
||||
providers := r.Providers()
|
||||
if r.config == nil {
|
||||
entries := make([]fetchEntry, 0, len(providers))
|
||||
for i, p := range providers {
|
||||
entries = append(entries, fetchEntry{provider: p, priority: i})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
priority := make(map[string]int)
|
||||
for _, c := range r.config.EnabledForFetch() {
|
||||
priority[c.Provider] = c.FetchPriority
|
||||
}
|
||||
entries := make([]fetchEntry, 0, len(providers))
|
||||
for _, p := range providers {
|
||||
if prio, ok := priority[p.ID()]; ok {
|
||||
entries = append(entries, fetchEntry{provider: p, priority: prio})
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
type mergeCandidate struct {
|
||||
marker Marker
|
||||
priority int
|
||||
}
|
||||
|
||||
// better reports whether a should win over b for the same segment kind: more
|
||||
// preferred fetch priority, then more submissions, then higher confidence, then
|
||||
// deterministic provider id ordering.
|
||||
func (a mergeCandidate) better(b mergeCandidate) bool {
|
||||
if a.priority != b.priority {
|
||||
return a.priority < b.priority
|
||||
}
|
||||
if a.marker.SubmissionCount != b.marker.SubmissionCount {
|
||||
return a.marker.SubmissionCount > b.marker.SubmissionCount
|
||||
}
|
||||
if a.marker.Confidence != b.marker.Confidence {
|
||||
return a.marker.Confidence > b.marker.Confidence
|
||||
}
|
||||
return a.marker.ProviderID < b.marker.ProviderID
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, v := range values {
|
||||
if strings.TrimSpace(v) != "" {
|
||||
return v
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (r *Registry) logProviderError(providerID string, req Request, err error) {
|
||||
logger := slog.Default()
|
||||
if r != nil && r.logger != nil {
|
||||
|
||||
@@ -84,6 +84,148 @@ func TestRegistryFetchFirstHitReturnsLastProviderErrorAsMiss(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedKeepsBestPerSegmentByProviderPriority(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Algorithm: "a:v1", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.7, SubmissionCount: 1},
|
||||
{Kind: MarkerKindCredits, Start: 100 * time.Second, End: 110 * time.Second, Confidence: 0.5, SubmissionCount: 1},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Algorithm: "b:v1", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.9, SubmissionCount: 500},
|
||||
{Kind: MarkerKindCredits, Start: 105 * time.Second, End: 120 * time.Second, Confidence: 0.6, SubmissionCount: 400},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
if err := registry.Register(a); err != nil {
|
||||
t.Fatalf("register a: %v", err)
|
||||
}
|
||||
if err := registry.Register(b); err != nil {
|
||||
t.Fatalf("register b: %v", err)
|
||||
}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("expected hit")
|
||||
}
|
||||
|
||||
byKind := map[MarkerKind]Marker{}
|
||||
for _, m := range res.Markers {
|
||||
byKind[m.Kind] = m
|
||||
}
|
||||
if intro := byKind[MarkerKindIntro]; intro.ProviderID != "a" || intro.End != 30*time.Second {
|
||||
t.Errorf("intro = %+v, want priority provider a end 30s", intro)
|
||||
}
|
||||
if credits := byKind[MarkerKindCredits]; credits.ProviderID != "a" || credits.End != 110*time.Second {
|
||||
t.Errorf("credits = %+v, want priority provider a end 110s", credits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedUsesConfiguredProviderPriority(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.9, SubmissionCount: 500},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.6, SubmissionCount: 1},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(a)
|
||||
_ = registry.Register(b)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"a": {Provider: "a", FetchEnabled: true, FetchPriority: 20},
|
||||
"b": {Provider: "b", FetchEnabled: true, FetchPriority: 10},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "b" || got.End != 25*time.Second {
|
||||
t.Fatalf("winner = %+v, want lower fetch_priority provider b", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedFallsBackWhenHigherPriorityProviderErrors(t *testing.T) {
|
||||
higher := &fakeProvider{id: "higher", err: errors.New("temporary provider failure")}
|
||||
lower := &fakeProvider{id: "lower", result: Result{ProviderID: "lower", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.6},
|
||||
}}}
|
||||
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(higher)
|
||||
_ = registry.Register(lower)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"higher": {Provider: "higher", FetchEnabled: true, FetchPriority: 10},
|
||||
"lower": {Provider: "lower", FetchEnabled: true, FetchPriority: 20},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged returned error despite fallback marker: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "lower" {
|
||||
t.Fatalf("winner provider = %q, want lower", got.ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedTieBreaksSamePriorityByQuality(t *testing.T) {
|
||||
a := &fakeProvider{id: "a", result: Result{ProviderID: "a", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.7, SubmissionCount: 1},
|
||||
}}}
|
||||
b := &fakeProvider{id: "b", result: Result{ProviderID: "b", Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 25 * time.Second, Confidence: 0.9, SubmissionCount: 2},
|
||||
}}}
|
||||
registry := NewRegistry(nil)
|
||||
_ = registry.Register(a)
|
||||
_ = registry.Register(b)
|
||||
registry.config = &ProviderConfigStore{cache: map[string]ProviderConfig{
|
||||
"a": {Provider: "a", FetchEnabled: true, FetchPriority: 10},
|
||||
"b": {Provider: "b", FetchEnabled: true, FetchPriority: 10},
|
||||
}}
|
||||
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 {
|
||||
t.Fatalf("merged = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if got := res.Markers[0]; got.ProviderID != "b" {
|
||||
t.Fatalf("winner provider = %q, want b by same-priority submission count", got.ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryFetchMergedSingleProviderParity(t *testing.T) {
|
||||
only := &fakeProvider{id: "introdb", result: Result{
|
||||
ProviderID: "introdb",
|
||||
SourceClass: "online",
|
||||
Algorithm: "introdb:v3",
|
||||
Markers: []Marker{{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.9}},
|
||||
}}
|
||||
registry := NewRegistry(nil)
|
||||
if err := registry.Register(only); err != nil {
|
||||
t.Fatalf("register: %v", err)
|
||||
}
|
||||
res, ok, err := registry.FetchMerged(context.Background(), Request{Kind: ItemKindEpisode})
|
||||
if err != nil {
|
||||
t.Fatalf("FetchMerged: %v", err)
|
||||
}
|
||||
if !ok || len(res.Markers) != 1 || res.Markers[0].Kind != MarkerKindIntro {
|
||||
t.Fatalf("single-provider merge = %+v, ok=%v", res.Markers, ok)
|
||||
}
|
||||
if res.Markers[0].ProviderID != "introdb" {
|
||||
t.Errorf("provider = %q, want introdb (stamped from result)", res.Markers[0].ProviderID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSetting(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
||||
+138
-49
@@ -11,6 +11,12 @@ import (
|
||||
// wins; an equal-priority source wins only if its confidence is strictly
|
||||
// higher than what's already stored. Unknown/empty existing source is treated
|
||||
// as priority zero so any defined source can replace it.
|
||||
//
|
||||
// A manual write is the exception: an explicit human edit is authoritative and
|
||||
// always applies at equal-or-higher priority (last-writer-wins). Without this,
|
||||
// correcting a previously hand-set marker — whose confidence is fixed at 1.0 —
|
||||
// would be silently rejected by the strictly-higher-confidence rule, leaving
|
||||
// users unable to fix their own (or each other's) edits.
|
||||
func CanWriteMarker(existingSource *string, existingConfidence *float64, newSource string, newConfidence *float64) bool {
|
||||
currentSource := ""
|
||||
if existingSource != nil {
|
||||
@@ -24,59 +30,97 @@ func CanWriteMarker(existingSource *string, existingConfidence *float64, newSour
|
||||
if newPriority < existingPriority {
|
||||
return false
|
||||
}
|
||||
if newSource == models.MarkerSourceManual {
|
||||
return true
|
||||
}
|
||||
if existingConfidence != nil && newConfidence != nil {
|
||||
return *newConfidence > *existingConfidence
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// SegmentPayload is the storage-agnostic per-segment marker write: its bounds
|
||||
// plus the provenance (provider/confidence/algorithm) of the marker that
|
||||
// produced it. A merged multi-provider result carries a different provider per
|
||||
// segment; a single-source result repeats the same provider across segments.
|
||||
type SegmentPayload struct {
|
||||
Start, End *float64
|
||||
Source string
|
||||
Provider *string
|
||||
Confidence *float64
|
||||
Algorithm string
|
||||
}
|
||||
|
||||
// Present reports whether the segment carries a bound to write.
|
||||
func (s SegmentPayload) Present() bool { return s.Start != nil || s.End != nil }
|
||||
|
||||
// MarkerUpdatePayload is the storage-agnostic shape produced from a provider
|
||||
// Result. Repositories convert it into their concrete write column set.
|
||||
// Pointer fields are nil when the result didn't carry that segment kind.
|
||||
// Result. Repositories convert it into their concrete write column set. Source
|
||||
// is the shared source class (online/scanner/manual/...); each SegmentPayload
|
||||
// carries its own provider/confidence/algorithm so a merged multi-provider
|
||||
// result records correct per-segment provenance.
|
||||
type MarkerUpdatePayload struct {
|
||||
IntroStart, IntroEnd *float64
|
||||
CreditsStart, CreditsEnd *float64
|
||||
RecapStart, RecapEnd *float64
|
||||
PreviewStart, PreviewEnd *float64
|
||||
Source string
|
||||
Provider *string
|
||||
Confidence *float64
|
||||
Algorithm string
|
||||
Intro SegmentPayload
|
||||
Credits SegmentPayload
|
||||
Recap SegmentPayload
|
||||
Preview SegmentPayload
|
||||
Source string
|
||||
}
|
||||
|
||||
// HasAnySegment reports whether the payload carries at least one segment
|
||||
// range. Callers can short-circuit empty writes without touching the DB.
|
||||
func (p MarkerUpdatePayload) HasAnySegment() bool {
|
||||
return p.IntroStart != nil || p.IntroEnd != nil ||
|
||||
p.CreditsStart != nil || p.CreditsEnd != nil ||
|
||||
p.RecapStart != nil || p.RecapEnd != nil ||
|
||||
p.PreviewStart != nil || p.PreviewEnd != nil
|
||||
return p.Intro.Present() || p.Credits.Present() || p.Recap.Present() || p.Preview.Present()
|
||||
}
|
||||
|
||||
// BuildUpdatePayload converts a provider Result into the storage payload.
|
||||
// All four segment kinds in the result map to the corresponding *Start/*End
|
||||
// pointer pair; absent kinds remain nil. The Result's Algorithm field is
|
||||
// authoritative; the falls back to `external:<source>` if absent so writes
|
||||
// always carry an algorithm tag for provenance.
|
||||
//
|
||||
// Confidence: the highest per-marker confidence in the result is promoted to
|
||||
// the shared write-time confidence. Per-segment confidence values are not
|
||||
// preserved individually — the write path uses a single confidence per
|
||||
// upsert, and providers today return uniform confidence across all segments
|
||||
// of a single fetch.
|
||||
func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
payload := MarkerUpdatePayload{
|
||||
Source: result.SourceClass,
|
||||
Algorithm: result.Algorithm,
|
||||
// SummaryConfidence returns the highest per-segment confidence present, for the
|
||||
// legacy shared markers_confidence column. Returns nil when no segment carries
|
||||
// a confidence.
|
||||
func (p MarkerUpdatePayload) SummaryConfidence() *float64 {
|
||||
var max float64
|
||||
found := false
|
||||
for _, s := range []SegmentPayload{p.Intro, p.Credits, p.Recap, p.Preview} {
|
||||
if s.Confidence != nil && (!found || *s.Confidence > max) {
|
||||
max = *s.Confidence
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if payload.Algorithm == "" && payload.Source != "" {
|
||||
payload.Algorithm = "external:" + payload.Source
|
||||
}
|
||||
if provider := strings.TrimSpace(result.ProviderID); provider != "" {
|
||||
payload.Provider = &provider
|
||||
if !found {
|
||||
return nil
|
||||
}
|
||||
return &max
|
||||
}
|
||||
|
||||
var maxConfidence float64
|
||||
// SummarySource returns the highest-priority source class present in the
|
||||
// per-segment payloads, using confidence as the tie-breaker. This keeps the
|
||||
// legacy shared markers_source column coherent even when individual segment
|
||||
// provenance differs.
|
||||
func (p MarkerUpdatePayload) SummarySource() string {
|
||||
source := strings.TrimSpace(p.Source)
|
||||
confidence := p.SummaryConfidence()
|
||||
found := source != ""
|
||||
for _, s := range []SegmentPayload{p.Intro, p.Credits, p.Recap, p.Preview} {
|
||||
if !s.Present() || strings.TrimSpace(s.Source) == "" {
|
||||
continue
|
||||
}
|
||||
if !found ||
|
||||
models.MarkerSourcePriority(s.Source) > models.MarkerSourcePriority(source) ||
|
||||
(models.MarkerSourcePriority(s.Source) == models.MarkerSourcePriority(source) && confidenceGreater(s.Confidence, confidence)) {
|
||||
source = s.Source
|
||||
confidence = s.Confidence
|
||||
found = true
|
||||
}
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
// BuildUpdatePayload converts a provider Result into the storage payload. Each
|
||||
// marker maps to its segment with its own provenance: ProviderID/Algorithm fall
|
||||
// back to the Result-level values (used for single-provider results), and the
|
||||
// algorithm finally falls back to external:<source> so every write carries an
|
||||
// algorithm tag.
|
||||
func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
payload := MarkerUpdatePayload{Source: result.SourceClass}
|
||||
for _, m := range result.Markers {
|
||||
start := m.Start.Seconds()
|
||||
end := m.End.Seconds()
|
||||
@@ -84,26 +128,71 @@ func BuildUpdatePayload(result Result) MarkerUpdatePayload {
|
||||
continue
|
||||
}
|
||||
startPtr, endPtr := start, end
|
||||
source := markerSource(m, result)
|
||||
seg := SegmentPayload{
|
||||
Start: &startPtr,
|
||||
End: &endPtr,
|
||||
Source: source,
|
||||
Provider: markerProvider(m, result),
|
||||
Algorithm: markerAlgorithm(m, result.Algorithm, source),
|
||||
}
|
||||
if m.Confidence > 0 {
|
||||
conf := m.Confidence
|
||||
seg.Confidence = &conf
|
||||
}
|
||||
switch m.Kind {
|
||||
case MarkerKindIntro:
|
||||
payload.IntroStart = &startPtr
|
||||
payload.IntroEnd = &endPtr
|
||||
payload.Intro = seg
|
||||
case MarkerKindCredits:
|
||||
payload.CreditsStart = &startPtr
|
||||
payload.CreditsEnd = &endPtr
|
||||
payload.Credits = seg
|
||||
case MarkerKindRecap:
|
||||
payload.RecapStart = &startPtr
|
||||
payload.RecapEnd = &endPtr
|
||||
payload.Recap = seg
|
||||
case MarkerKindPreview:
|
||||
payload.PreviewStart = &startPtr
|
||||
payload.PreviewEnd = &endPtr
|
||||
payload.Preview = seg
|
||||
}
|
||||
if m.Confidence > maxConfidence {
|
||||
maxConfidence = m.Confidence
|
||||
}
|
||||
}
|
||||
if maxConfidence > 0 {
|
||||
payload.Confidence = &maxConfidence
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func markerSource(m Marker, result Result) string {
|
||||
source := strings.TrimSpace(m.SourceClass)
|
||||
if source == "" {
|
||||
source = strings.TrimSpace(result.SourceClass)
|
||||
}
|
||||
return source
|
||||
}
|
||||
|
||||
// markerProvider returns the per-marker provider, falling back to the
|
||||
// Result-level provider (single-provider results), or nil when neither is set.
|
||||
func markerProvider(m Marker, result Result) *string {
|
||||
provider := strings.TrimSpace(m.ProviderID)
|
||||
if provider == "" {
|
||||
provider = strings.TrimSpace(result.ProviderID)
|
||||
}
|
||||
if provider == "" {
|
||||
return nil
|
||||
}
|
||||
return &provider
|
||||
}
|
||||
|
||||
// markerAlgorithm returns the per-marker algorithm, falling back to the
|
||||
// already-resolved Result-level algorithm.
|
||||
func markerAlgorithm(m Marker, resultAlgorithm, source string) string {
|
||||
if a := strings.TrimSpace(m.Algorithm); a != "" {
|
||||
return a
|
||||
}
|
||||
if strings.TrimSpace(resultAlgorithm) == "" && strings.TrimSpace(source) != "" {
|
||||
return "external:" + source
|
||||
}
|
||||
return resultAlgorithm
|
||||
}
|
||||
|
||||
func confidenceGreater(a, b *float64) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
return *a > *b
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestBuildUpdatePayloadAggregatesConfidence(t *testing.T) {
|
||||
func TestBuildUpdatePayloadPreservesPerSegmentConfidence(t *testing.T) {
|
||||
result := Result{
|
||||
ProviderID: "introdb",
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
@@ -21,35 +21,71 @@ func TestBuildUpdatePayloadAggregatesConfidence(t *testing.T) {
|
||||
|
||||
payload := BuildUpdatePayload(result)
|
||||
|
||||
if payload.Confidence == nil {
|
||||
t.Fatal("confidence should be populated")
|
||||
if payload.Intro.Confidence == nil || *payload.Intro.Confidence != 0.7 {
|
||||
t.Errorf("intro confidence = %v, want 0.7", payload.Intro.Confidence)
|
||||
}
|
||||
if *payload.Confidence != 0.9 {
|
||||
t.Errorf("confidence = %v, want 0.9 (max across markers)", *payload.Confidence)
|
||||
if payload.Credits.Confidence == nil || *payload.Credits.Confidence != 0.9 {
|
||||
t.Errorf("credits confidence = %v, want 0.9", payload.Credits.Confidence)
|
||||
}
|
||||
if payload.Algorithm != "introdb:v3" {
|
||||
t.Errorf("algorithm = %q, want introdb:v3", payload.Algorithm)
|
||||
if payload.Recap.Confidence == nil || *payload.Recap.Confidence != 0.5 {
|
||||
t.Errorf("recap confidence = %v, want 0.5", payload.Recap.Confidence)
|
||||
}
|
||||
if payload.IntroStart == nil || *payload.IntroStart != 10 {
|
||||
t.Errorf("intro start = %v, want 10", payload.IntroStart)
|
||||
if sc := payload.SummaryConfidence(); sc == nil || *sc != 0.9 {
|
||||
t.Errorf("summary confidence = %v, want 0.9 (max)", sc)
|
||||
}
|
||||
if payload.RecapStart == nil || *payload.RecapStart != 0 {
|
||||
t.Errorf("recap start = %v, want 0", payload.RecapStart)
|
||||
if payload.Intro.Algorithm != "introdb:v3" {
|
||||
t.Errorf("intro algorithm = %q, want introdb:v3", payload.Intro.Algorithm)
|
||||
}
|
||||
if payload.PreviewStart != nil {
|
||||
t.Errorf("preview start = %v, want nil (no preview marker)", payload.PreviewStart)
|
||||
if payload.Intro.Start == nil || *payload.Intro.Start != 10 {
|
||||
t.Errorf("intro start = %v, want 10", payload.Intro.Start)
|
||||
}
|
||||
if payload.Recap.Start == nil || *payload.Recap.Start != 0 {
|
||||
t.Errorf("recap start = %v, want 0", payload.Recap.Start)
|
||||
}
|
||||
if payload.Preview.Present() {
|
||||
t.Errorf("preview should be absent (no preview marker)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildUpdatePayloadFallsBackAlgorithm(t *testing.T) {
|
||||
func TestBuildUpdatePayloadPerMarkerProvider(t *testing.T) {
|
||||
// A merged result: each marker carries its own provider/algorithm.
|
||||
result := Result{
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Markers: []Marker{
|
||||
{Kind: MarkerKindIntro, Start: 0, End: 30 * time.Second, Confidence: 0.8, ProviderID: "introdb", Algorithm: "introdb:v3"},
|
||||
{Kind: MarkerKindCredits, Start: 100 * time.Second, End: 120 * time.Second, Confidence: 0.7, SourceClass: models.MarkerSourcePlugin, ProviderID: "plugin:1:markers", Algorithm: "other:v1"},
|
||||
},
|
||||
}
|
||||
payload := BuildUpdatePayload(result)
|
||||
if payload.Intro.Provider == nil || *payload.Intro.Provider != "introdb" {
|
||||
t.Errorf("intro provider = %v, want introdb", payload.Intro.Provider)
|
||||
}
|
||||
if payload.Credits.Provider == nil || *payload.Credits.Provider != "plugin:1:markers" {
|
||||
t.Errorf("credits provider = %v, want plugin:1:markers", payload.Credits.Provider)
|
||||
}
|
||||
if payload.Credits.Source != models.MarkerSourcePlugin {
|
||||
t.Errorf("credits source = %q, want plugin", payload.Credits.Source)
|
||||
}
|
||||
if payload.Credits.Algorithm != "other:v1" {
|
||||
t.Errorf("credits algorithm = %q, want other:v1", payload.Credits.Algorithm)
|
||||
}
|
||||
if payload.Intro.Source != models.MarkerSourceOnline {
|
||||
t.Errorf("intro source = %q, want online fallback", payload.Intro.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildUpdatePayloadFallsBackAlgorithmAndProvider(t *testing.T) {
|
||||
result := Result{
|
||||
ProviderID: "custom",
|
||||
SourceClass: models.MarkerSourceOnline,
|
||||
Markers: []Marker{{Kind: MarkerKindIntro, Start: 0, End: 10 * time.Second}},
|
||||
}
|
||||
payload := BuildUpdatePayload(result)
|
||||
if payload.Algorithm != "external:online" {
|
||||
t.Errorf("algorithm = %q, want external:online fallback", payload.Algorithm)
|
||||
if payload.Intro.Algorithm != "external:online" {
|
||||
t.Errorf("intro algorithm = %q, want external:online fallback", payload.Intro.Algorithm)
|
||||
}
|
||||
if payload.Intro.Provider == nil || *payload.Intro.Provider != "custom" {
|
||||
t.Errorf("intro provider = %v, want custom (result-level fallback)", payload.Intro.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,3 +114,24 @@ func TestCanWriteMarkerHigherPriorityWinsRegardless(t *testing.T) {
|
||||
t.Error("lower priority should not overwrite higher")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanWriteMarkerManualAlwaysWinsLastWriter(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
conf := 1.0
|
||||
|
||||
// A manual edit must overwrite an existing manual marker even though both
|
||||
// share priority 4 and confidence 1.0 — otherwise corrections silently fail.
|
||||
if !CanWriteMarker(&manual, &conf, models.MarkerSourceManual, &conf) {
|
||||
t.Error("manual edit should overwrite an existing manual marker (last-writer-wins)")
|
||||
}
|
||||
// Manual still wins over lower-priority sources.
|
||||
online := models.MarkerSourceOnline
|
||||
highConf := 0.99
|
||||
if !CanWriteMarker(&online, &highConf, models.MarkerSourceManual, &conf) {
|
||||
t.Error("manual edit should overwrite a high-confidence online marker")
|
||||
}
|
||||
// A non-manual source still cannot displace a manual marker.
|
||||
if CanWriteMarker(&manual, &conf, models.MarkerSourceOnline, &highConf) {
|
||||
t.Error("online source should never overwrite a manual marker")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,10 @@ import (
|
||||
|
||||
const EpisodeRecentStillWindow = 45 * 24 * time.Hour
|
||||
|
||||
var episodePlaceholderTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd|episode\s+\d+)$`)
|
||||
var (
|
||||
episodePlaceholderTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd|episode\s+\d+)$`)
|
||||
episodeProvisionalTitlePattern = regexp.MustCompile(`(?i)^(tba|tbd)$`)
|
||||
)
|
||||
|
||||
func IsEpisodePlaceholderTitle(title string) bool {
|
||||
title = strings.TrimSpace(title)
|
||||
@@ -20,6 +23,23 @@ func IsEpisodePlaceholderTitle(title string) bool {
|
||||
return episodePlaceholderTitlePattern.MatchString(title)
|
||||
}
|
||||
|
||||
func isEpisodeProvisionalTitle(title string) bool {
|
||||
title = strings.TrimSpace(title)
|
||||
if title == "" {
|
||||
return false
|
||||
}
|
||||
return episodeProvisionalTitlePattern.MatchString(title)
|
||||
}
|
||||
|
||||
func episodeHasProviderMatch(ep *models.Episode) bool {
|
||||
if ep == nil || strings.EqualFold(strings.TrimSpace(ep.MetadataSource), "scanner_fallback") {
|
||||
return false
|
||||
}
|
||||
return strings.TrimSpace(ep.TmdbID) != "" ||
|
||||
strings.TrimSpace(ep.TvdbID) != "" ||
|
||||
strings.TrimSpace(ep.ImdbID) != ""
|
||||
}
|
||||
|
||||
func EpisodeHasIncompleteMetadata(ep *models.Episode, now time.Time) bool {
|
||||
if ep == nil {
|
||||
return false
|
||||
@@ -43,3 +63,29 @@ func EpisodeHasIncompleteMetadata(ep *models.Episode, now time.Time) bool {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// EpisodeHasActionableMetadataDebt returns true when another provider refresh
|
||||
// can reasonably be expected to improve the episode metadata.
|
||||
func EpisodeHasActionableMetadataDebt(ep *models.Episode, now time.Time) bool {
|
||||
if ep == nil {
|
||||
return false
|
||||
}
|
||||
if strings.TrimSpace(ep.Title) == "" {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(strings.TrimSpace(ep.MetadataSource), "scanner_fallback") {
|
||||
return true
|
||||
}
|
||||
if !episodeHasProviderMatch(ep) {
|
||||
return true
|
||||
}
|
||||
if isEpisodeProvisionalTitle(ep.Title) {
|
||||
return true
|
||||
}
|
||||
if ep.AirDate != nil &&
|
||||
!ep.AirDate.Before(now.Add(-EpisodeRecentStillWindow)) &&
|
||||
strings.TrimSpace(ep.StillPath) == "" {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -46,4 +46,110 @@ func TestEpisodeHasIncompleteMetadata(t *testing.T) {
|
||||
if !EpisodeHasIncompleteMetadata(incomplete, now) {
|
||||
t.Fatal("expected fallback placeholder episode to be treated as incomplete")
|
||||
}
|
||||
|
||||
providerNumbered := &models.Episode{
|
||||
Title: "Episode 1",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
}
|
||||
if !EpisodeHasIncompleteMetadata(providerNumbered, now) {
|
||||
t.Fatal("expected provider numbered title to remain quality-incomplete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpisodeHasActionableMetadataDebt(t *testing.T) {
|
||||
now := time.Date(2026, 4, 14, 12, 0, 0, 0, time.UTC)
|
||||
recent := now.Add(-7 * 24 * time.Hour)
|
||||
old := now.Add(-90 * 24 * time.Hour)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
ep *models.Episode
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "provider numbered title with overview",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "provider numbered title with empty overview",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
TvdbID: "9607609",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "provider provisional title",
|
||||
ep: &models.Episode{
|
||||
Title: "TBD",
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "scanner fallback numbered title",
|
||||
ep: &models.Episode{
|
||||
Title: "Episode 1",
|
||||
MetadataSource: "scanner_fallback",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "missing title",
|
||||
ep: &models.Episode{
|
||||
TmdbID: "3812334",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "missing provider id",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "recent provider episode missing still",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
AirDate: &recent,
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "old provider episode missing still",
|
||||
ep: &models.Episode{
|
||||
Title: "Pilot",
|
||||
Overview: "Provider overview",
|
||||
TmdbID: "3812334",
|
||||
AirDate: &old,
|
||||
MetadataSource: "provider",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := EpisodeHasActionableMetadataDebt(tc.ep, now); got != tc.want {
|
||||
t.Fatalf("EpisodeHasActionableMetadataDebt() = %v, want %v", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
// isPgConstraintViolation reports whether err (or any error it wraps) is a
|
||||
// Postgres error with the given SQLSTATE code raised by the named constraint.
|
||||
// Centralizes the errors.As(&pgconn.PgError) + Code/ConstraintName boilerplate
|
||||
// that several constraint-specific predicates in this package share.
|
||||
func isPgConstraintViolation(err error, code, constraint string) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) {
|
||||
return false
|
||||
}
|
||||
return pgErr.Code == code && pgErr.ConstraintName == constraint
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestHandleChildProvider404KeepsProviderID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ids := map[string]string{
|
||||
"tmdb": "32843",
|
||||
"tvdb": "164951",
|
||||
}
|
||||
|
||||
if !handleChildProvider404("tmdb", ids, errors.New("tmdb: HTTP 404: not found"), "season", 0) {
|
||||
t.Fatal("handleChildProvider404() = false, want true")
|
||||
}
|
||||
if ids["tmdb"] != "32843" {
|
||||
t.Fatalf("tmdb id = %q, want preserved", ids["tmdb"])
|
||||
}
|
||||
if ids["tvdb"] != "164951" {
|
||||
t.Fatalf("tvdb id = %q, want preserved", ids["tvdb"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleProvider404DropsProviderID(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
ids := map[string]string{
|
||||
"tmdb": "32843",
|
||||
"tvdb": "164951",
|
||||
}
|
||||
|
||||
if !handleProvider404(nil, ids, "tmdb", errors.New("tmdb: HTTP 404: not found")) {
|
||||
t.Fatal("handleProvider404() = false, want true")
|
||||
}
|
||||
if _, ok := ids["tmdb"]; ok {
|
||||
t.Fatalf("tmdb id was not dropped: %v", ids)
|
||||
}
|
||||
if ids["tvdb"] != "164951" {
|
||||
t.Fatalf("tvdb id = %q, want preserved", ids["tvdb"])
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -577,25 +579,59 @@ func betterCanonicalCandidate(left, right canonicalCandidate) bool {
|
||||
return left.ContentID < right.ContentID
|
||||
}
|
||||
|
||||
func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID, canonicalID string) error {
|
||||
if sourceID == "" || canonicalID == "" || sourceID == canonicalID {
|
||||
return nil
|
||||
// mediaItemMergeStep is one statement in the ordered media-item merge sequence
|
||||
// run by canonicalizeMediaItemReferencesTx. Every step references $1 (the
|
||||
// source content ID); steps that move data onto the canonical row also
|
||||
// reference $2 (the canonical content ID).
|
||||
type mediaItemMergeStep struct {
|
||||
name string
|
||||
sql string
|
||||
}
|
||||
|
||||
// mergeStepPlaceholderRe matches positional query placeholders ($1, $2, …).
|
||||
var mergeStepPlaceholderRe = regexp.MustCompile(`\$(\d+)`)
|
||||
|
||||
// maxPlaceholder returns the highest positional placeholder number referenced
|
||||
// in sql (0 if none). It reads the full placeholder number, so "$20" is 20 and
|
||||
// is never confused with "$2" — unlike a substring check.
|
||||
func maxPlaceholder(sql string) int {
|
||||
highest := 0
|
||||
for _, m := range mergeStepPlaceholderRe.FindAllStringSubmatch(sql, -1) {
|
||||
if n, err := strconv.Atoi(m[1]); err == nil && n > highest {
|
||||
highest = n
|
||||
}
|
||||
}
|
||||
if err := ensureSeriesCanMove(ctx, tx, sourceID, canonicalID); err != nil {
|
||||
return err
|
||||
return highest
|
||||
}
|
||||
|
||||
// mergeStepArgs returns the positional arguments for a merge step, sized to the
|
||||
// placeholders the SQL actually binds. Passing too many args (e.g. an unused
|
||||
// $2) makes pgx reject the Exec with "mismatched param and argument count"
|
||||
// under the default QueryExecModeCacheStatement, aborting the whole merge
|
||||
// transaction. Every merge step binds $1 (source) and optionally $2
|
||||
// (canonical); TestMergeStepPlaceholdersAreBounded enforces that contract, so
|
||||
// the default branch is unreachable for the defined steps and fails loud if a
|
||||
// future step violates it.
|
||||
func mergeStepArgs(sql, sourceID, canonicalID string) []any {
|
||||
switch n := maxPlaceholder(sql); n {
|
||||
case 1:
|
||||
return []any{sourceID}
|
||||
case 2:
|
||||
return []any{sourceID, canonicalID}
|
||||
default:
|
||||
panic(fmt.Sprintf("merge step binds unsupported placeholder count %d: %q", n, sql))
|
||||
}
|
||||
steps := []struct {
|
||||
name string
|
||||
sql string
|
||||
}{
|
||||
{"merge provider ids", `
|
||||
}
|
||||
|
||||
var mediaItemMergeSteps = []mediaItemMergeStep{
|
||||
{"merge provider ids", `
|
||||
INSERT INTO media_item_provider_ids (content_id, item_type, provider, provider_id, created_at, updated_at)
|
||||
SELECT $2, item_type, provider, provider_id, created_at, NOW()
|
||||
FROM media_item_provider_ids
|
||||
WHERE content_id = $1
|
||||
ON CONFLICT DO NOTHING`},
|
||||
{"drop source provider ids", `DELETE FROM media_item_provider_ids WHERE content_id = $1`},
|
||||
{"merge legacy provider columns", `
|
||||
{"drop source provider ids", `DELETE FROM media_item_provider_ids WHERE content_id = $1`},
|
||||
{"merge legacy provider columns", `
|
||||
UPDATE media_items dest
|
||||
SET imdb_id = COALESCE(NULLIF(dest.imdb_id, ''), src.imdb_id),
|
||||
tmdb_id = COALESCE(NULLIF(dest.tmdb_id, ''), src.tmdb_id),
|
||||
@@ -603,10 +639,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
updated_at = NOW()
|
||||
FROM media_items src
|
||||
WHERE dest.content_id = $2 AND src.content_id = $1`},
|
||||
{"move media files", `UPDATE media_files SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"move seasons", `UPDATE seasons SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"move episodes", `UPDATE episodes SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"merge library memberships", `
|
||||
{"move media files", `UPDATE media_files SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"move seasons", `UPDATE seasons SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"move episodes", `UPDATE episodes SET series_id = $2, updated_at = NOW() WHERE series_id = $1`},
|
||||
{"merge library memberships", `
|
||||
INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at)
|
||||
SELECT $2, media_folder_id, MIN(first_seen_at)
|
||||
FROM media_item_libraries
|
||||
@@ -614,10 +650,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY media_folder_id
|
||||
ON CONFLICT (content_id, media_folder_id) DO UPDATE
|
||||
SET first_seen_at = LEAST(media_item_libraries.first_seen_at, EXCLUDED.first_seen_at)`},
|
||||
{"delete source library memberships", `DELETE FROM media_item_libraries WHERE content_id = $1`},
|
||||
{"move root claims", `UPDATE media_item_roots SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"move group claims", `UPDATE media_item_groups SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"merge collection items", `
|
||||
{"delete source library memberships", `DELETE FROM media_item_libraries WHERE content_id = $1`},
|
||||
{"move root claims", `UPDATE media_item_roots SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"move group claims", `UPDATE media_item_groups SET content_id = $2, last_seen_at = NOW() WHERE content_id = $1`},
|
||||
{"merge collection items", `
|
||||
INSERT INTO library_collection_items (collection_id, media_item_id, position, source_rank, created_at, updated_at)
|
||||
SELECT collection_id, $2, MIN(position), MIN(source_rank), MIN(created_at), NOW()
|
||||
FROM library_collection_items
|
||||
@@ -627,8 +663,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
SET position = LEAST(library_collection_items.position, EXCLUDED.position),
|
||||
source_rank = LEAST(library_collection_items.source_rank, EXCLUDED.source_rank),
|
||||
updated_at = NOW()`},
|
||||
{"delete source collection items", `DELETE FROM library_collection_items WHERE media_item_id = $1`},
|
||||
{"merge personal collection items", `
|
||||
{"delete source collection items", `DELETE FROM library_collection_items WHERE media_item_id = $1`},
|
||||
{"merge personal collection items", `
|
||||
INSERT INTO user_personal_collection_items (user_id, collection_id, media_item_id, position, added_at)
|
||||
SELECT user_id, collection_id, $2, MIN(position), MIN(added_at)
|
||||
FROM user_personal_collection_items
|
||||
@@ -637,8 +673,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, collection_id, media_item_id) DO UPDATE
|
||||
SET position = LEAST(user_personal_collection_items.position, EXCLUDED.position),
|
||||
added_at = LEAST(user_personal_collection_items.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source personal collection items", `DELETE FROM user_personal_collection_items WHERE media_item_id = $1`},
|
||||
{"merge favorites", `
|
||||
{"delete source personal collection items", `DELETE FROM user_personal_collection_items WHERE media_item_id = $1`},
|
||||
{"merge favorites", `
|
||||
INSERT INTO user_favorites (user_id, profile_id, media_item_id, added_at)
|
||||
SELECT user_id, profile_id, $2, MIN(added_at)
|
||||
FROM user_favorites
|
||||
@@ -646,8 +682,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY user_id, profile_id
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET added_at = LEAST(user_favorites.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source favorites", `DELETE FROM user_favorites WHERE media_item_id = $1`},
|
||||
{"merge watchlist", `
|
||||
{"delete source favorites", `DELETE FROM user_favorites WHERE media_item_id = $1`},
|
||||
{"merge watchlist", `
|
||||
INSERT INTO user_watchlist (user_id, profile_id, media_item_id, added_at)
|
||||
SELECT user_id, profile_id, $2, MIN(added_at)
|
||||
FROM user_watchlist
|
||||
@@ -655,8 +691,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
GROUP BY user_id, profile_id
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET added_at = LEAST(user_watchlist.added_at, EXCLUDED.added_at)`},
|
||||
{"delete source watchlist", `DELETE FROM user_watchlist WHERE media_item_id = $1`},
|
||||
{"merge ratings", `
|
||||
{"delete source watchlist", `DELETE FROM user_watchlist WHERE media_item_id = $1`},
|
||||
{"merge ratings", `
|
||||
INSERT INTO user_ratings (user_id, profile_id, media_item_id, rating, rated_at)
|
||||
SELECT DISTINCT ON (user_id, profile_id) user_id, profile_id, $2, rating, rated_at
|
||||
FROM user_ratings
|
||||
@@ -665,8 +701,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET rating = CASE WHEN EXCLUDED.rated_at >= user_ratings.rated_at THEN EXCLUDED.rating ELSE user_ratings.rating END,
|
||||
rated_at = GREATEST(user_ratings.rated_at, EXCLUDED.rated_at)`},
|
||||
{"delete source ratings", `DELETE FROM user_ratings WHERE media_item_id = $1`},
|
||||
{"merge progress", `
|
||||
{"delete source ratings", `DELETE FROM user_ratings WHERE media_item_id = $1`},
|
||||
{"merge progress", `
|
||||
INSERT INTO user_watch_progress (
|
||||
user_id, profile_id, media_item_id, position_seconds, duration_seconds, completed,
|
||||
updated_at, last_file_id, last_resolution, last_hdr, last_codec_video, last_edition_key
|
||||
@@ -685,9 +721,9 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
last_hdr = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_hdr ELSE user_watch_progress.last_hdr END,
|
||||
last_codec_video = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_codec_video ELSE user_watch_progress.last_codec_video END,
|
||||
last_edition_key = CASE WHEN EXCLUDED.updated_at >= user_watch_progress.updated_at THEN EXCLUDED.last_edition_key ELSE user_watch_progress.last_edition_key END`},
|
||||
{"delete source progress", `DELETE FROM user_watch_progress WHERE media_item_id = $1`},
|
||||
{"move history", `UPDATE user_watch_history SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"merge hidden history", `
|
||||
{"delete source progress", `DELETE FROM user_watch_progress WHERE media_item_id = $1`},
|
||||
{"move history", `UPDATE user_watch_history SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"merge hidden history", `
|
||||
INSERT INTO user_history_hidden_items (user_id, profile_id, media_item_id, hidden_before, updated_at)
|
||||
SELECT user_id, profile_id, $2, MAX(hidden_before), MAX(updated_at)
|
||||
FROM user_history_hidden_items
|
||||
@@ -696,8 +732,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
ON CONFLICT (user_id, profile_id, media_item_id) DO UPDATE
|
||||
SET hidden_before = GREATEST(user_history_hidden_items.hidden_before, EXCLUDED.hidden_before),
|
||||
updated_at = GREATEST(user_history_hidden_items.updated_at, EXCLUDED.updated_at)`},
|
||||
{"delete source hidden history", `DELETE FROM user_history_hidden_items WHERE media_item_id = $1`},
|
||||
{"delete duplicate home item dismissals", `
|
||||
{"delete source hidden history", `DELETE FROM user_history_hidden_items WHERE media_item_id = $1`},
|
||||
{"delete duplicate home item dismissals", `
|
||||
DELETE FROM user_home_item_dismissals src
|
||||
USING user_home_item_dismissals dest
|
||||
WHERE src.media_item_id = $1
|
||||
@@ -705,33 +741,33 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id
|
||||
AND dest.surface = src.surface`},
|
||||
{"move home item dismissals", `UPDATE user_home_item_dismissals SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"remap home item dismissals series", `UPDATE user_home_item_dismissals SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate audio preferences", `
|
||||
{"move home item dismissals", `UPDATE user_home_item_dismissals SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"remap home item dismissals series", `UPDATE user_home_item_dismissals SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate audio preferences", `
|
||||
DELETE FROM user_audio_preferences src
|
||||
USING user_audio_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move audio preferences", `UPDATE user_audio_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate subtitle preferences", `
|
||||
{"move audio preferences", `UPDATE user_audio_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate subtitle preferences", `
|
||||
DELETE FROM user_subtitle_preferences src
|
||||
USING user_subtitle_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move subtitle preferences", `UPDATE user_subtitle_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate series playback preferences", `
|
||||
{"move subtitle preferences", `UPDATE user_subtitle_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"delete duplicate series playback preferences", `
|
||||
DELETE FROM user_series_playback_preferences src
|
||||
USING user_series_playback_preferences dest
|
||||
WHERE src.series_id = $1
|
||||
AND dest.series_id = $2
|
||||
AND dest.user_id = src.user_id
|
||||
AND dest.profile_id = src.profile_id`},
|
||||
{"move series playback preferences", `UPDATE user_series_playback_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"merge plex sync item bindings timestamps", `
|
||||
{"move series playback preferences", `UPDATE user_series_playback_preferences SET series_id = $2 WHERE series_id = $1`},
|
||||
{"merge plex sync item bindings timestamps", `
|
||||
UPDATE plex_sync_item_bindings dest
|
||||
SET last_seen_at = GREATEST(dest.last_seen_at, src.last_seen_at),
|
||||
updated_at = NOW()
|
||||
@@ -740,35 +776,35 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.connection_id = src.connection_id
|
||||
AND (dest.media_item_id = $2 OR dest.plex_rating_key = src.plex_rating_key)
|
||||
AND dest.media_item_id <> src.media_item_id`},
|
||||
{"delete duplicate plex sync item bindings", `
|
||||
{"delete duplicate plex sync item bindings", `
|
||||
DELETE FROM plex_sync_item_bindings src
|
||||
USING plex_sync_item_bindings dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.connection_id = src.connection_id
|
||||
AND (dest.media_item_id = $2 OR dest.plex_rating_key = src.plex_rating_key)
|
||||
AND dest.media_item_id <> src.media_item_id`},
|
||||
{"move remaining plex sync item bindings", `UPDATE plex_sync_item_bindings SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"delete duplicate plex sync item state", `
|
||||
{"move remaining plex sync item bindings", `UPDATE plex_sync_item_bindings SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"delete duplicate plex sync item state", `
|
||||
DELETE FROM plex_sync_item_state src
|
||||
USING plex_sync_item_state dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND dest.mapping_id = src.mapping_id`},
|
||||
{"move plex sync item state", `UPDATE plex_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move webhook sync item state", `UPDATE webhook_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch together rooms selected", `UPDATE watch_together_rooms SET selected_content_id = $2 WHERE selected_content_id = $1`},
|
||||
{"move watch together suggestions", `UPDATE watch_together_suggestions SET content_id = $2 WHERE content_id = $1`},
|
||||
{"move admin playback history", `UPDATE playback_history_admin SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move user downloads", `UPDATE user_downloads SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move downloads", `UPDATE downloads SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"merge embeddings", `
|
||||
{"move plex sync item state", `UPDATE plex_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move webhook sync item state", `UPDATE webhook_sync_item_state SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch together rooms selected", `UPDATE watch_together_rooms SET selected_content_id = $2 WHERE selected_content_id = $1`},
|
||||
{"move watch together suggestions", `UPDATE watch_together_suggestions SET content_id = $2 WHERE content_id = $1`},
|
||||
{"move admin playback history", `UPDATE playback_history_admin SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move user downloads", `UPDATE user_downloads SET media_item_id = $2 WHERE media_item_id = $1`},
|
||||
{"move downloads", `UPDATE downloads SET content_id = $2, updated_at = NOW() WHERE content_id = $1`},
|
||||
{"merge embeddings", `
|
||||
INSERT INTO media_item_embeddings (media_item_id, embedding, model, canonical_text, created_at, updated_at)
|
||||
SELECT $2, embedding, model, canonical_text, created_at, updated_at
|
||||
FROM media_item_embeddings
|
||||
WHERE media_item_id = $1
|
||||
ON CONFLICT (media_item_id) DO NOTHING`},
|
||||
{"delete source embeddings", `DELETE FROM media_item_embeddings WHERE media_item_id = $1`},
|
||||
{"merge item localizations", `
|
||||
{"delete source embeddings", `DELETE FROM media_item_embeddings WHERE media_item_id = $1`},
|
||||
{"merge item localizations", `
|
||||
INSERT INTO media_item_localizations (
|
||||
content_id, language, title, sort_title, overview, tagline, poster_path, poster_thumbhash,
|
||||
backdrop_path, backdrop_thumbhash, logo_path, created_at, updated_at
|
||||
@@ -778,8 +814,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
FROM media_item_localizations
|
||||
WHERE content_id = $1
|
||||
ON CONFLICT (content_id, language) DO NOTHING`},
|
||||
{"delete source localizations", `DELETE FROM media_item_localizations WHERE content_id = $1`},
|
||||
{"dedupe people", `
|
||||
{"delete source localizations", `DELETE FROM media_item_localizations WHERE content_id = $1`},
|
||||
{"dedupe people", `
|
||||
DELETE FROM item_people src
|
||||
USING item_people dest
|
||||
WHERE src.content_id = $1
|
||||
@@ -787,8 +823,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND dest.person_id = src.person_id
|
||||
AND dest.kind = src.kind
|
||||
AND dest.character = src.character`},
|
||||
{"move people", `UPDATE item_people SET content_id = $2 WHERE content_id = $1`},
|
||||
{"merge refresh debt", `
|
||||
{"move people", `UPDATE item_people SET content_id = $2 WHERE content_id = $1`},
|
||||
{"merge refresh debt", `
|
||||
INSERT INTO metadata_refresh_debt (
|
||||
target_type, content_id, priority, reason_mask, next_refresh_at,
|
||||
claimed_at, lease_expires_at, last_attempt_at, last_success_at,
|
||||
@@ -806,8 +842,8 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
attempt_count = GREATEST(metadata_refresh_debt.attempt_count, EXCLUDED.attempt_count),
|
||||
last_error = COALESCE(NULLIF(metadata_refresh_debt.last_error, ''), EXCLUDED.last_error),
|
||||
updated_at = NOW()`},
|
||||
{"delete source refresh debt", `DELETE FROM metadata_refresh_debt WHERE target_type = 'item' AND content_id = $1`},
|
||||
{"merge stale media ids", `
|
||||
{"delete source refresh debt", `DELETE FROM metadata_refresh_debt WHERE target_type = 'item' AND content_id = $1`},
|
||||
{"merge stale media ids", `
|
||||
INSERT INTO stale_media_ids (content_id, provider, provider_id, first_seen_at, last_seen_at)
|
||||
SELECT $2, provider, provider_id, first_seen_at, last_seen_at
|
||||
FROM stale_media_ids
|
||||
@@ -816,10 +852,10 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
SET provider_id = COALESCE(NULLIF(stale_media_ids.provider_id, ''), EXCLUDED.provider_id),
|
||||
first_seen_at = LEAST(stale_media_ids.first_seen_at, EXCLUDED.first_seen_at),
|
||||
last_seen_at = GREATEST(stale_media_ids.last_seen_at, EXCLUDED.last_seen_at)`},
|
||||
{"delete source stale media ids", `DELETE FROM stale_media_ids WHERE content_id = $1`},
|
||||
{"move watch provider history exports", `UPDATE watch_provider_history_exports SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch provider scrobble sessions", `UPDATE watch_provider_scrobble_sessions SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"merge watch provider favorites by provider key", `
|
||||
{"delete source stale media ids", `DELETE FROM stale_media_ids WHERE content_id = $1`},
|
||||
{"move watch provider history exports", `UPDATE watch_provider_history_exports SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move watch provider scrobble sessions", `UPDATE watch_provider_scrobble_sessions SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"merge watch provider favorites by provider key", `
|
||||
UPDATE watch_provider_favorite_items dest
|
||||
SET remote_present = dest.remote_present OR src.remote_present,
|
||||
local_present = dest.local_present OR src.local_present,
|
||||
@@ -834,7 +870,7 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
AND src.connection_id = dest.connection_id
|
||||
AND src.provider_item_key <> ''
|
||||
AND src.provider_item_key = dest.provider_item_key`},
|
||||
{"merge watch provider favorites by connection", `
|
||||
{"merge watch provider favorites by connection", `
|
||||
UPDATE watch_provider_favorite_items dest
|
||||
SET provider_item_key = CASE
|
||||
WHEN dest.provider_item_key = '' THEN src.provider_item_key
|
||||
@@ -854,17 +890,25 @@ func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID,
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND src.connection_id = dest.connection_id`},
|
||||
{"delete duplicate watch provider favorites", `
|
||||
{"delete duplicate watch provider favorites", `
|
||||
DELETE FROM watch_provider_favorite_items src
|
||||
USING watch_provider_favorite_items dest
|
||||
WHERE src.media_item_id = $1
|
||||
AND dest.media_item_id = $2
|
||||
AND src.connection_id = dest.connection_id
|
||||
AND (src.provider_item_key = dest.provider_item_key OR dest.media_item_id = $2)`},
|
||||
{"move remaining watch provider favorites", `UPDATE watch_provider_favorite_items SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
{"move remaining watch provider favorites", `UPDATE watch_provider_favorite_items SET media_item_id = $2, updated_at = NOW() WHERE media_item_id = $1`},
|
||||
}
|
||||
|
||||
func canonicalizeMediaItemReferencesTx(ctx context.Context, tx pgx.Tx, sourceID, canonicalID string) error {
|
||||
if sourceID == "" || canonicalID == "" || sourceID == canonicalID {
|
||||
return nil
|
||||
}
|
||||
for _, step := range steps {
|
||||
if _, err := tx.Exec(ctx, step.sql, sourceID, canonicalID); err != nil {
|
||||
if err := ensureSeriesCanMove(ctx, tx, sourceID, canonicalID); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, step := range mediaItemMergeSteps {
|
||||
if _, err := tx.Exec(ctx, step.sql, mergeStepArgs(step.sql, sourceID, canonicalID)...); err != nil {
|
||||
return fmt.Errorf("%s: %w", step.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
@@ -42,6 +43,73 @@ func TestProviderEntriesFromDriftUsesDurableProvidersOnly(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaxPlaceholder(t *testing.T) {
|
||||
// maxPlaceholder must read the FULL placeholder number, not a substring:
|
||||
// "$20" is placeholder 20, never "$2". This is the property the old
|
||||
// strings.Contains(sql, "$2") heuristic got wrong.
|
||||
cases := []struct {
|
||||
sql string
|
||||
want int
|
||||
}{
|
||||
{`DELETE FROM media_item_provider_ids WHERE content_id = $1`, 1},
|
||||
{`UPDATE media_files SET content_id = $2 WHERE content_id = $1`, 2},
|
||||
{`INSERT INTO t SELECT $2 FROM s WHERE content_id = $1`, 2},
|
||||
{`SELECT $1, $2, $1`, 2}, // repeated placeholders count once
|
||||
{`SELECT $20 FROM t`, 20},
|
||||
{`SELECT $12 FROM t`, 12},
|
||||
{`DELETE FROM t WHERE x = 'literal'`, 0},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := maxPlaceholder(tc.sql); got != tc.want {
|
||||
t.Errorf("maxPlaceholder(%q) = %d, want %d", tc.sql, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStepArgsMatchesPlaceholderArity(t *testing.T) {
|
||||
const sourceID = "src"
|
||||
const canonicalID = "canon"
|
||||
|
||||
// A step that only references $1 must receive exactly one argument.
|
||||
// Passing canonicalID as an unused $2 makes pgx reject the Exec with
|
||||
// "mismatched param and argument count" under the default
|
||||
// QueryExecModeCacheStatement, aborting the whole merge transaction.
|
||||
onlySource := mergeStepArgs(`DELETE FROM media_item_provider_ids WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(onlySource) != 1 || onlySource[0] != sourceID {
|
||||
t.Fatalf("mergeStepArgs($1-only) = %#v, want [%q]", onlySource, sourceID)
|
||||
}
|
||||
|
||||
// A step that references $2 must receive both arguments in order.
|
||||
both := mergeStepArgs(`UPDATE media_files SET content_id = $2 WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(both) != 2 || both[0] != sourceID || both[1] != canonicalID {
|
||||
t.Fatalf("mergeStepArgs($1+$2) = %#v, want [%q %q]", both, sourceID, canonicalID)
|
||||
}
|
||||
|
||||
// $2 may appear before $1 (e.g. INSERT ... SELECT $2 ...). Args are still
|
||||
// positional [source, canonical] regardless of textual order.
|
||||
reversed := mergeStepArgs(`INSERT INTO t SELECT $2 FROM s WHERE content_id = $1`, sourceID, canonicalID)
|
||||
if len(reversed) != 2 || reversed[0] != sourceID || reversed[1] != canonicalID {
|
||||
t.Fatalf("mergeStepArgs(reversed) = %#v, want [%q %q]", reversed, sourceID, canonicalID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStepPlaceholdersAreBounded(t *testing.T) {
|
||||
// Every merge step must bind $1 (the source) and may bind $2 (the
|
||||
// canonical). A step binding $3+ (or none) cannot be satisfied by the two
|
||||
// IDs the loop passes and would abort the merge at runtime; this guard —
|
||||
// checking the actual placeholder bound rather than re-deriving the arg
|
||||
// count from the same predicate the helper uses — catches such a step.
|
||||
for _, step := range mediaItemMergeSteps {
|
||||
n := maxPlaceholder(step.sql)
|
||||
if n < 1 || n > 2 {
|
||||
t.Errorf("merge step %q binds %d placeholders; merge supports only $1 and $2", step.name, n)
|
||||
}
|
||||
if !strings.Contains(step.sql, "$1") {
|
||||
t.Errorf("merge step %q does not bind $1 (source)", step.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBothCandidatesHaveContentDetectsRealUserData(t *testing.T) {
|
||||
withFiles := canonicalCandidate{ContentID: "a", ActiveFiles: 2}
|
||||
withLibraries := canonicalCandidate{ContentID: "b", LibraryCount: 1}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package metadata
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRefreshFollowUpContentID(t *testing.T) {
|
||||
// After a refresh, follow-up writes (stale-ID clear/record, debt sync) must
|
||||
// target the canonical content ID the item was persisted/merged into, not
|
||||
// the requested ID — which may have been deleted when the item was
|
||||
// canonicalized into an existing one during mergeAndPersist.
|
||||
if got := refreshFollowUpContentID("req", nil); got != "req" {
|
||||
t.Fatalf("nil result: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: ""}); got != "req" {
|
||||
t.Fatalf("empty result id: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: " "}); got != "req" {
|
||||
t.Fatalf("blank result id: got %q, want %q", got, "req")
|
||||
}
|
||||
if got := refreshFollowUpContentID("req", &ProcessResult{ContentID: "canonical"}); got != "canonical" {
|
||||
t.Fatalf("canonicalized result id: got %q, want %q", got, "canonical")
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/naming"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
@@ -242,6 +241,28 @@ func handleProvider404(
|
||||
return true
|
||||
}
|
||||
|
||||
func handleChildProvider404(
|
||||
provider string,
|
||||
providerIDs map[string]string,
|
||||
err error,
|
||||
attrs ...any,
|
||||
) bool {
|
||||
if !isProvider404(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
provider = strings.ToLower(strings.TrimSpace(provider))
|
||||
logAttrs := append([]any{"provider", provider}, attrs...)
|
||||
if provider != "" && providerIDs != nil {
|
||||
if providerID := strings.TrimSpace(providerIDs[provider]); providerID != "" {
|
||||
logAttrs = append(logAttrs, "provider_id", providerID)
|
||||
}
|
||||
}
|
||||
logAttrs = append(logAttrs, "error", err)
|
||||
slog.Info("metadata: provider returned 404 for unavailable child metadata", logAttrs...)
|
||||
return true
|
||||
}
|
||||
|
||||
// MetadataService is the unified pipeline orchestrator.
|
||||
type MetadataService struct {
|
||||
chainRepo *ChainRepository
|
||||
@@ -1137,7 +1158,10 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
Language: req.Language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
|
||||
// Pass nil for provider404s so this refresh can drop the
|
||||
// provider from the in-memory merge without recording a durable
|
||||
// stale item ID from the season chain.
|
||||
if handleProvider404(nil, accumulatedIDs, p.Slug(), err,
|
||||
"content_id", req.ContentID,
|
||||
"season", 0,
|
||||
) {
|
||||
@@ -1170,7 +1194,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
Language: req.Language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
|
||||
if handleChildProvider404(p.Slug(), accumulatedIDs, err,
|
||||
"content_id", req.ContentID,
|
||||
"season", season.SeasonNumber,
|
||||
) {
|
||||
@@ -1210,18 +1234,28 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
|
||||
// Refresh stale ID records on successful refresh: clear anything resolved,
|
||||
// then keep only the providers that still 404ed during this run.
|
||||
if s.staleIDRepo != nil && req.ContentID != "" {
|
||||
if delErr := s.staleIDRepo.DeleteByContentID(ctx, req.ContentID); delErr != nil {
|
||||
// Stale-ID follow-up targets the canonical content ID the item was
|
||||
// persisted/merged into. When mergeAndPersist canonicalizes this item into
|
||||
// an existing one, req.ContentID is the now-deleted source: clearing or
|
||||
// recording stale IDs against it would touch nothing (or FK-violate), so
|
||||
// the still-404ing providers must be recorded on result.ContentID instead.
|
||||
// provider404s is only allocated when req.ContentID was set (the refresh
|
||||
// targeted a known item). Guarding on it preserves the original gating so a
|
||||
// content-id-less refresh that canonicalizes into an existing item does not
|
||||
// wipe that item's stale rows without re-recording any.
|
||||
followUpContentID := refreshFollowUpContentID(req.ContentID, result)
|
||||
if s.staleIDRepo != nil && followUpContentID != "" && provider404s != nil {
|
||||
if delErr := s.staleIDRepo.DeleteByContentID(ctx, followUpContentID); delErr != nil {
|
||||
slog.Warn("metadata: failed to clear stale IDs after refresh",
|
||||
"content_id", req.ContentID, "error", delErr)
|
||||
"content_id", followUpContentID, "error", delErr)
|
||||
}
|
||||
for slug, providerID := range provider404s {
|
||||
if providerID == "" {
|
||||
continue
|
||||
}
|
||||
if upsertErr := s.staleIDRepo.Upsert(ctx, req.ContentID, slug, providerID); upsertErr != nil {
|
||||
if upsertErr := s.staleIDRepo.Upsert(ctx, followUpContentID, slug, providerID); upsertErr != nil {
|
||||
slog.Warn("metadata: failed to persist stale provider ID after partial refresh",
|
||||
"content_id", req.ContentID,
|
||||
"content_id", followUpContentID,
|
||||
"provider", slug,
|
||||
"provider_id", providerID,
|
||||
"error", upsertErr)
|
||||
@@ -1239,6 +1273,19 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// refreshFollowUpContentID returns the content ID that a completed refresh's
|
||||
// follow-up writes (stale-ID clear/record, debt sync) should target: the
|
||||
// canonical ID the item was persisted or merged into when available, falling
|
||||
// back to the requested ID. mergeAndPersist can canonicalize an item into an
|
||||
// existing one, deleting the requested ID, so follow-up writes keyed on
|
||||
// req.ContentID would miss the surviving item.
|
||||
func refreshFollowUpContentID(reqContentID string, result *ProcessResult) string {
|
||||
if result != nil && strings.TrimSpace(result.ContentID) != "" {
|
||||
return result.ContentID
|
||||
}
|
||||
return reqContentID
|
||||
}
|
||||
|
||||
// mergeAndPersist handles the final merge into existing item and DB persistence.
|
||||
func (s *MetadataService) mergeAndPersist(
|
||||
ctx context.Context,
|
||||
@@ -1917,7 +1964,7 @@ func (s *MetadataService) syncRefreshDebtForSeason(ctx context.Context, seasonID
|
||||
reasonMask := int64(0)
|
||||
now := time.Now().UTC()
|
||||
for _, episode := range episodes {
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
reasonMask = RefreshDebtReasonEpisodeIncomplete
|
||||
break
|
||||
}
|
||||
@@ -1952,7 +1999,7 @@ func (s *MetadataService) syncRefreshDebtForEpisode(ctx context.Context, episode
|
||||
}
|
||||
reasonMask := int64(0)
|
||||
now := time.Now().UTC()
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
reasonMask = RefreshDebtReasonEpisodeIncomplete
|
||||
}
|
||||
if reasonMask == 0 {
|
||||
@@ -2336,7 +2383,7 @@ func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, provide
|
||||
Language: language,
|
||||
})
|
||||
if err != nil {
|
||||
if handleProvider404(nil, providerIDs, p.Slug(), err, "season", seasonNumber) {
|
||||
if handleChildProvider404(p.Slug(), providerIDs, err, "season", seasonNumber) {
|
||||
continue
|
||||
}
|
||||
slog.Warn("metadata: target episode provider error",
|
||||
@@ -3468,7 +3515,7 @@ func (s *MetadataService) refreshSeriesEpisodeMetadataState(ctx context.Context,
|
||||
|
||||
incomplete := false
|
||||
for _, episode := range episodes {
|
||||
if EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
incomplete = true
|
||||
}
|
||||
if err := s.syncVisibleEpisodeRefreshDebt(ctx, episode, now); err != nil {
|
||||
@@ -3487,7 +3534,7 @@ func (s *MetadataService) syncVisibleEpisodeRefreshDebt(ctx context.Context, epi
|
||||
if s == nil || s.refreshDebtRepo == nil || episode == nil || strings.TrimSpace(episode.ContentID) == "" {
|
||||
return nil
|
||||
}
|
||||
if !EpisodeHasIncompleteMetadata(episode, now) {
|
||||
if !EpisodeHasActionableMetadataDebt(episode, now) {
|
||||
return s.refreshDebtRepo.DeleteTargetDebt(ctx, RefreshTargetEpisode, episode.ContentID)
|
||||
}
|
||||
reasonMask := RefreshDebtReasonEpisodeIncomplete
|
||||
@@ -4422,14 +4469,7 @@ func rebindDeletableStatuses(allowMatchedSource bool) []string {
|
||||
}
|
||||
|
||||
func isProviderIDUniqueConflict(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if !errors.As(err, &pgErr) {
|
||||
return false
|
||||
}
|
||||
return pgErr.Code == "23505" && pgErr.ConstraintName == "media_item_provider_ids_provider_provider_id_item_type_key"
|
||||
return isPgConstraintViolation(err, "23505", "media_item_provider_ids_provider_provider_id_item_type_key")
|
||||
}
|
||||
|
||||
func (s *MetadataService) repairMatchedDuplicateProviderOwnersByFolderAndPathPrefix(ctx context.Context, folderID int, pathPrefix string) (int, error) {
|
||||
|
||||
@@ -986,6 +986,98 @@ func TestSyncRefreshDebtForItemPreservesRequestedEpisodeDebt(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshSeriesEpisodeMetadataStateUsesActionableDebt(t *testing.T) {
|
||||
now := time.Date(2026, 4, 14, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
t.Run("provider numbered episode clears stale debt", func(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
episodes := newFakeEpisodeRepo()
|
||||
debts := newFakeRefreshDebtRepo()
|
||||
h.service.episodeRepo = episodes
|
||||
h.service.refreshDebtRepo = debts
|
||||
h.itemRepo.items["series-provider"] = &models.MediaItem{
|
||||
ContentID: "series-provider",
|
||||
Type: "series",
|
||||
EpisodeMetadataIncomplete: true,
|
||||
}
|
||||
if err := episodes.Upsert(ctx, &models.Episode{
|
||||
ContentID: "episode-provider",
|
||||
SeriesID: "series-provider",
|
||||
SeasonID: "season-provider",
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Title: "Episode 1",
|
||||
TvdbID: "9607609",
|
||||
MetadataSource: "provider",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed provider episode: %v", err)
|
||||
}
|
||||
debts.debts[fakeRefreshDebtKey(RefreshTargetEpisode, "episode-provider")] = &models.MetadataRefreshDebt{
|
||||
TargetType: RefreshTargetEpisode,
|
||||
ContentID: "episode-provider",
|
||||
ReasonMask: RefreshDebtReasonEpisodeIncomplete,
|
||||
}
|
||||
|
||||
h.service.refreshSeriesEpisodeMetadataState(ctx, "series-provider", now)
|
||||
|
||||
item, err := h.itemRepo.GetByID(ctx, "series-provider")
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID: %v", err)
|
||||
}
|
||||
if item.EpisodeMetadataIncomplete {
|
||||
t.Fatal("expected provider numbered episode to clear series incomplete flag")
|
||||
}
|
||||
if item.EpisodeMetadataLastCheckedAt == nil || !item.EpisodeMetadataLastCheckedAt.Equal(now) {
|
||||
t.Fatalf("last checked at = %v, want %v", item.EpisodeMetadataLastCheckedAt, now)
|
||||
}
|
||||
if _, err := debts.GetTarget(ctx, RefreshTargetEpisode, "episode-provider"); !errors.Is(err, ErrRefreshDebtNotFound) {
|
||||
t.Fatalf("provider episode debt after refresh = %v, want ErrRefreshDebtNotFound", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("scanner fallback episode creates debt", func(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
episodes := newFakeEpisodeRepo()
|
||||
debts := newFakeRefreshDebtRepo()
|
||||
h.service.episodeRepo = episodes
|
||||
h.service.refreshDebtRepo = debts
|
||||
h.itemRepo.items["series-fallback"] = &models.MediaItem{
|
||||
ContentID: "series-fallback",
|
||||
Type: "series",
|
||||
}
|
||||
if err := episodes.Upsert(ctx, &models.Episode{
|
||||
ContentID: "episode-fallback",
|
||||
SeriesID: "series-fallback",
|
||||
SeasonID: "season-fallback",
|
||||
SeasonNumber: 1,
|
||||
EpisodeNumber: 1,
|
||||
Title: "Episode 1",
|
||||
MetadataSource: "scanner_fallback",
|
||||
}); err != nil {
|
||||
t.Fatalf("seed fallback episode: %v", err)
|
||||
}
|
||||
|
||||
h.service.refreshSeriesEpisodeMetadataState(ctx, "series-fallback", now)
|
||||
|
||||
item, err := h.itemRepo.GetByID(ctx, "series-fallback")
|
||||
if err != nil {
|
||||
t.Fatalf("GetByID: %v", err)
|
||||
}
|
||||
if !item.EpisodeMetadataIncomplete {
|
||||
t.Fatal("expected fallback episode to keep series incomplete flag")
|
||||
}
|
||||
debt, err := debts.GetTarget(ctx, RefreshTargetEpisode, "episode-fallback")
|
||||
if err != nil {
|
||||
t.Fatalf("GetTarget fallback debt: %v", err)
|
||||
}
|
||||
if !hasRefreshDebtReason(debt.ReasonMask, RefreshDebtReasonEpisodeIncomplete) {
|
||||
t.Fatalf("fallback debt reason mask = %d, want episode incomplete", debt.ReasonMask)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRequestStaleMetadataRefreshStartsOnDemandRefreshOnce(t *testing.T) {
|
||||
h := newTestHarness()
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -3,6 +3,7 @@ package metadata
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -55,10 +56,39 @@ func (r *StaleMediaIDRepository) Upsert(ctx context.Context, contentID, provider
|
||||
SET provider_id = EXCLUDED.provider_id,
|
||||
last_seen_at = NOW()
|
||||
`, contentID, provider, providerID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upserting stale media ID: %w", err)
|
||||
return resolveStaleUpsertError(err, contentID, provider)
|
||||
}
|
||||
|
||||
// staleMediaIDContentFKConstraint is the Postgres-auto-generated name of the
|
||||
// stale_media_ids.content_id → media_items(content_id) foreign key, declared
|
||||
// inline in migrations/sql/036_stale_media_ids.sql (hence "{table}_{column}_fkey").
|
||||
const staleMediaIDContentFKConstraint = "stale_media_ids_content_id_fkey"
|
||||
|
||||
// resolveStaleUpsertError maps an Upsert error to the value Upsert should
|
||||
// return. The referenced media item can be deleted or merged away (e.g. by
|
||||
// provider-ID canonicalization) between a metadata refresh starting and its
|
||||
// 404 landing here; the parent row is then gone and there is nothing left to
|
||||
// track, so the resulting foreign-key violation is a logged no-op. Every other
|
||||
// error is wrapped and propagated.
|
||||
func resolveStaleUpsertError(err error, contentID, provider string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
if isMissingContentForeignKeyViolation(err) {
|
||||
slog.Info("metadata: skipping stale media ID for missing content item",
|
||||
"content_id", contentID,
|
||||
"provider", provider,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("upserting stale media ID: %w", err)
|
||||
}
|
||||
|
||||
// isMissingContentForeignKeyViolation reports whether err is a Postgres
|
||||
// foreign-key violation against stale_media_ids.content_id — i.e. the media
|
||||
// item the stale ID would reference no longer exists.
|
||||
func isMissingContentForeignKeyViolation(err error) bool {
|
||||
return isPgConstraintViolation(err, "23503", staleMediaIDContentFKConstraint)
|
||||
}
|
||||
|
||||
// GetByContentID loads all stale external ID records for a single item.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
package metadata
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
)
|
||||
|
||||
func TestIsMissingContentForeignKeyViolation(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"generic", errors.New("boom"), false},
|
||||
{
|
||||
"unique violation",
|
||||
&pgconn.PgError{Code: "23505", ConstraintName: "stale_media_ids_pkey"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"fk violation on a different constraint",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "some_other_fkey"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"fk violation on the content_id fkey",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"},
|
||||
true,
|
||||
},
|
||||
{
|
||||
"wrapped fk violation on the content_id fkey",
|
||||
fmt.Errorf("upserting stale media ID: %w",
|
||||
&pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"}),
|
||||
true,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isMissingContentForeignKeyViolation(tc.err); got != tc.want {
|
||||
t.Fatalf("isMissingContentForeignKeyViolation(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsPgConstraintViolation(t *testing.T) {
|
||||
const code = "23503"
|
||||
const constraint = "some_fkey"
|
||||
cases := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"nil", nil, false},
|
||||
{"generic", errors.New("boom"), false},
|
||||
{"wrong code", &pgconn.PgError{Code: "23505", ConstraintName: constraint}, false},
|
||||
{"wrong constraint", &pgconn.PgError{Code: code, ConstraintName: "other"}, false},
|
||||
{"match", &pgconn.PgError{Code: code, ConstraintName: constraint}, true},
|
||||
{"wrapped match", fmt.Errorf("x: %w", &pgconn.PgError{Code: code, ConstraintName: constraint}), true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isPgConstraintViolation(tc.err, code, constraint); got != tc.want {
|
||||
t.Fatalf("isPgConstraintViolation(%v) = %v, want %v", tc.err, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveStaleUpsertError(t *testing.T) {
|
||||
// The benign missing-parent race is swallowed (nil); every other error is
|
||||
// wrapped and propagated so a real failure still surfaces.
|
||||
if err := resolveStaleUpsertError(nil, "c", "tmdb"); err != nil {
|
||||
t.Fatalf("resolveStaleUpsertError(nil) = %v, want nil", err)
|
||||
}
|
||||
|
||||
fkErr := &pgconn.PgError{Code: "23503", ConstraintName: "stale_media_ids_content_id_fkey"}
|
||||
if err := resolveStaleUpsertError(fkErr, "c", "tmdb"); err != nil {
|
||||
t.Fatalf("resolveStaleUpsertError(missing-content FK) = %v, want nil", err)
|
||||
}
|
||||
|
||||
other := &pgconn.PgError{Code: "23505", ConstraintName: "stale_media_ids_pkey"}
|
||||
if err := resolveStaleUpsertError(other, "c", "tmdb"); err == nil {
|
||||
t.Fatal("resolveStaleUpsertError(other pg error) = nil, want wrapped error")
|
||||
}
|
||||
|
||||
generic := errors.New("connection reset")
|
||||
if err := resolveStaleUpsertError(generic, "c", "tmdb"); err == nil ||
|
||||
!errors.Is(err, generic) {
|
||||
t.Fatalf("resolveStaleUpsertError(generic) = %v, want wrapped generic", err)
|
||||
}
|
||||
}
|
||||
@@ -43,9 +43,6 @@ func (n *MarkerUpdateNotifier) MarkersUpdated(_ context.Context, file *models.Me
|
||||
credits := rangePayload(file.CreditsStart, file.CreditsEnd)
|
||||
recap := rangePayload(file.RecapStart, file.RecapEnd)
|
||||
preview := rangePayload(file.PreviewStart, file.PreviewEnd)
|
||||
if intro == nil && credits == nil && recap == nil && preview == nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, session := range n.sessions.GetSessionsByMediaFileID(file.ID) {
|
||||
if session == nil || session.ID == "" || !session.HasRealtimeConnection {
|
||||
|
||||
@@ -82,3 +82,32 @@ func TestMarkerUpdateNotifierTargetsMatchingSessions(t *testing.T) {
|
||||
t.Fatalf("payload.Credits = %#v, want credits range", payload.Credits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerUpdateNotifierSendsAllClearedMarkers(t *testing.T) {
|
||||
sessions := NewSessionManager(0, 0)
|
||||
session, _ := sessions.StartSession(1, "profile-a", 100, PlayDirect, false)
|
||||
_ = sessions.SetRealtimeConnection(session.ID, true)
|
||||
|
||||
hub := NewRealtimeHub()
|
||||
conn := &dispatchTestConn{}
|
||||
reg := hub.Register(session.ID, conn)
|
||||
defer hub.Unregister(reg)
|
||||
|
||||
notifier := NewMarkerUpdateNotifier(sessions, hub)
|
||||
notifier.MarkersUpdated(context.Background(), &models.MediaFile{ID: 100})
|
||||
|
||||
if len(conn.messages) != 1 {
|
||||
t.Fatalf("messages = %d, want 1 all-cleared marker update", len(conn.messages))
|
||||
}
|
||||
event, ok := conn.messages[0].(EventEnvelope)
|
||||
if !ok {
|
||||
t.Fatalf("message type = %T, want EventEnvelope", conn.messages[0])
|
||||
}
|
||||
var payload MarkersUpdatedPayload
|
||||
if err := json.Unmarshal(event.Payload, &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal(payload): %v", err)
|
||||
}
|
||||
if payload.Intro != nil || payload.Credits != nil || payload.Recap != nil || payload.Preview != nil {
|
||||
t.Fatalf("payload markers = %#v, want all nil", payload)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,10 +93,11 @@ type SegmentProgress struct {
|
||||
// SegmentRecoveryDecision tells the segment handler whether to briefly wait
|
||||
// for ffmpeg or seek-restart immediately.
|
||||
type SegmentRecoveryDecision struct {
|
||||
Wait bool
|
||||
WaitTimeout time.Duration
|
||||
Reason string
|
||||
Progress SegmentProgress
|
||||
Wait bool
|
||||
WaitTimeout time.Duration
|
||||
RestartOnTimeout bool
|
||||
Reason string
|
||||
Progress SegmentProgress
|
||||
}
|
||||
|
||||
// defaultSegmentDuration is the segment length when not specified. Short
|
||||
@@ -109,6 +110,7 @@ const maxPersistedFFmpegChars = 2000
|
||||
|
||||
const (
|
||||
maxSequentialMissingSegments = 2
|
||||
activeSegmentWait = 12 * time.Second
|
||||
segmentWaitGrace = 1500 * time.Millisecond
|
||||
maxSegmentWait = 6 * time.Second
|
||||
minSegmentWait = 3 * time.Second
|
||||
@@ -1096,8 +1098,9 @@ func (s *TranscodeSession) SegmentProgress(time.Time) SegmentProgress {
|
||||
func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) SegmentRecoveryDecision {
|
||||
progress := s.SegmentProgress(now)
|
||||
decision := SegmentRecoveryDecision{
|
||||
WaitTimeout: segmentWaitTimeout(progress.SegmentDuration),
|
||||
Progress: progress,
|
||||
WaitTimeout: segmentWaitTimeout(progress.SegmentDuration),
|
||||
RestartOnTimeout: true,
|
||||
Progress: progress,
|
||||
}
|
||||
|
||||
switch {
|
||||
@@ -1112,6 +1115,8 @@ func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) Se
|
||||
case !progress.HasManifest:
|
||||
if segNum <= progress.StartSegmentNumber+1 {
|
||||
decision.Wait = true
|
||||
decision.WaitTimeout = activeSegmentWait
|
||||
decision.RestartOnTimeout = false
|
||||
decision.Reason = "startup_manifest_not_ready"
|
||||
} else {
|
||||
decision.Reason = "startup_request_beyond_window"
|
||||
@@ -1122,6 +1127,8 @@ func (s *TranscodeSession) SegmentRecoveryDecision(segNum int, now time.Time) Se
|
||||
decision.Reason = "produced_output_stale"
|
||||
default:
|
||||
decision.Wait = true
|
||||
decision.WaitTimeout = activeSegmentWait
|
||||
decision.RestartOnTimeout = false
|
||||
decision.Reason = "near_produced_head"
|
||||
}
|
||||
|
||||
|
||||
@@ -486,12 +486,44 @@ func TestSegmentRecoveryDecisionWaitsForFreshNextSegment(t *testing.T) {
|
||||
if !decision.Wait {
|
||||
t.Fatalf("Wait = false, want true (reason=%s)", decision.Reason)
|
||||
}
|
||||
if decision.WaitTimeout != 3500*time.Millisecond {
|
||||
t.Fatalf("WaitTimeout = %s, want 3.5s", decision.WaitTimeout)
|
||||
if decision.WaitTimeout != 12*time.Second {
|
||||
t.Fatalf("WaitTimeout = %s, want 12s", decision.WaitTimeout)
|
||||
}
|
||||
if decision.Reason != "near_produced_head" {
|
||||
t.Fatalf("Reason = %q, want near_produced_head", decision.Reason)
|
||||
}
|
||||
if decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRecoveryDecisionUsesLongerWaitForStartupSegment(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
now := time.Now()
|
||||
|
||||
session := &TranscodeSession{
|
||||
outputDir: tempDir,
|
||||
running: true,
|
||||
opts: TranscodeOpts{
|
||||
TargetCodecVideo: "h264",
|
||||
SegmentDuration: 2,
|
||||
StartSegmentNumber: 0,
|
||||
},
|
||||
}
|
||||
|
||||
decision := session.SegmentRecoveryDecision(0, now)
|
||||
if !decision.Wait {
|
||||
t.Fatalf("Wait = false, want true (reason=%s)", decision.Reason)
|
||||
}
|
||||
if decision.WaitTimeout != 12*time.Second {
|
||||
t.Fatalf("WaitTimeout = %s, want 12s", decision.WaitTimeout)
|
||||
}
|
||||
if decision.Reason != "startup_manifest_not_ready" {
|
||||
t.Fatalf("Reason = %q, want startup_manifest_not_ready", decision.Reason)
|
||||
}
|
||||
if decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSegmentRecoveryDecisionRestartsWhenProducedOutputIsStale(t *testing.T) {
|
||||
@@ -516,6 +548,9 @@ func TestSegmentRecoveryDecisionRestartsWhenProducedOutputIsStale(t *testing.T)
|
||||
if decision.Wait {
|
||||
t.Fatal("Wait = true, want false")
|
||||
}
|
||||
if !decision.RestartOnTimeout {
|
||||
t.Fatal("RestartOnTimeout = false, want true")
|
||||
}
|
||||
if decision.Reason != "produced_output_stale" {
|
||||
t.Fatalf("Reason = %q, want produced_output_stale", decision.Reason)
|
||||
}
|
||||
|
||||
@@ -34,6 +34,11 @@ type MetadataProviderClient struct {
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type MarkerProviderClient struct {
|
||||
client pluginv1.MarkerProviderClient
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
type MediaAnalyzerClient struct {
|
||||
client pluginv1.MediaAnalyzerClient
|
||||
timeout time.Duration
|
||||
@@ -103,6 +108,16 @@ func (c *Client) MetadataProvider(capabilityID string) (*MetadataProviderClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) MarkerProvider(capabilityID string) (*MarkerProviderClient, error) {
|
||||
if err := c.requireCapability("marker_provider.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &MarkerProviderClient{
|
||||
client: c.rpc.MarkerProvider(),
|
||||
timeout: DefaultMarkerProviderTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *Client) MediaAnalyzer(capabilityID string) (*MediaAnalyzerClient, error) {
|
||||
if err := c.requireCapability("media_analyzer.v1", capabilityID); err != nil {
|
||||
return nil, err
|
||||
@@ -232,6 +247,24 @@ func (c *MetadataProviderClient) ResolveImageURLs(ctx context.Context, req *plug
|
||||
return c.client.ResolveImageURLs(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) FetchMarkers(ctx context.Context, req *pluginv1.FetchMarkersRequest) (*pluginv1.FetchMarkersResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.FetchMarkers(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) SubmitMarker(ctx context.Context, req *pluginv1.SubmitMarkerRequest) (*pluginv1.SubmitMarkerResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.SubmitMarker(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MarkerProviderClient) GetMarkerProviderStats(ctx context.Context, req *pluginv1.GetMarkerProviderStatsRequest) (*pluginv1.MarkerProviderStatsResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
return c.client.GetMarkerProviderStats(callCtx, req)
|
||||
}
|
||||
|
||||
func (c *MediaAnalyzerClient) Analyze(ctx context.Context, req *pluginv1.AnalyzeMediaRequest) (*pluginv1.AnalyzeMediaResponse, error) {
|
||||
callCtx, cancel := ensureDeadline(ctx, c.timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -105,3 +105,42 @@ func TestClient_ScanSource_CapabilityGate(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestClient_MarkerProvider_CapabilityGate(t *testing.T) {
|
||||
t.Run("absent capability returns error", func(t *testing.T) {
|
||||
c := makeTestClient(t, nil)
|
||||
_, err := c.MarkerProvider("missing")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for missing marker_provider.v1 capability, got nil")
|
||||
}
|
||||
if !errors.Is(err, ErrCapabilityNotFound) {
|
||||
t.Errorf("expected ErrCapabilityNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("declared capability returns no error", func(t *testing.T) {
|
||||
c := makeTestClient(t, []*pluginv1.CapabilityDescriptor{
|
||||
{Type: "marker_provider.v1", Id: "markers"},
|
||||
})
|
||||
got, err := c.MarkerProvider("markers")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if got == nil {
|
||||
t.Fatal("expected non-nil MarkerProviderClient")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wrong id returns error", func(t *testing.T) {
|
||||
c := makeTestClient(t, []*pluginv1.CapabilityDescriptor{
|
||||
{Type: "marker_provider.v1", Id: "markers"},
|
||||
})
|
||||
_, err := c.MarkerProvider("other")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for mismatched marker_provider.v1 capability id, got nil")
|
||||
}
|
||||
if !errors.Is(err, ErrCapabilityNotFound) {
|
||||
t.Errorf("expected ErrCapabilityNotFound, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,11 +9,12 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultHealthCheckInterval = 30 * time.Second
|
||||
DefaultHealthFailureLimit = 3
|
||||
DefaultMetadataTimeout = 30 * time.Second
|
||||
DefaultAnalyzerTimeout = 5 * time.Minute
|
||||
DefaultControlTimeout = 10 * time.Second
|
||||
DefaultHealthCheckInterval = 30 * time.Second
|
||||
DefaultHealthFailureLimit = 3
|
||||
DefaultMetadataTimeout = 30 * time.Second
|
||||
DefaultMarkerProviderTimeout = 30 * time.Second
|
||||
DefaultAnalyzerTimeout = 5 * time.Minute
|
||||
DefaultControlTimeout = 10 * time.Second
|
||||
// DefaultScanSourceTimeout covers a single PollChanges call. Most scan
|
||||
// sources return quickly, but filesystem-backed sources may need a longer
|
||||
// bounded window for an initial baseline; cooperative plugins should still
|
||||
|
||||
+62
-15
@@ -9,6 +9,8 @@ import (
|
||||
"log/slog"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"google.golang.org/protobuf/proto"
|
||||
@@ -21,6 +23,7 @@ import (
|
||||
type pluginClient interface {
|
||||
Manifest() *pluginv1.PluginManifest
|
||||
MetadataProvider(capabilityID string) (*pluginhost.MetadataProviderClient, error)
|
||||
MarkerProvider(capabilityID string) (*pluginhost.MarkerProviderClient, error)
|
||||
MediaAnalyzer(capabilityID string) (*pluginhost.MediaAnalyzerClient, error)
|
||||
ScheduledTask(capabilityID string) (*pluginhost.ScheduledTaskClient, error)
|
||||
ScanSource(capabilityID string) (*pluginhost.ScanSourceClient, error)
|
||||
@@ -52,15 +55,17 @@ type serviceConfigStore interface {
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
repositories *RepositoryStore
|
||||
installations serviceInstallationStore
|
||||
configs serviceConfigStore
|
||||
catalog *CatalogService
|
||||
installer *Installer
|
||||
archiveCache *ArchiveCache
|
||||
host Host
|
||||
testConfigSeq atomic.Int64
|
||||
dispatcher *EventDispatcher
|
||||
repositories *RepositoryStore
|
||||
installations serviceInstallationStore
|
||||
configs serviceConfigStore
|
||||
catalog *CatalogService
|
||||
installer *Installer
|
||||
archiveCache *ArchiveCache
|
||||
host Host
|
||||
testConfigSeq atomic.Int64
|
||||
dispatcher *EventDispatcher
|
||||
lifecycleMu sync.RWMutex
|
||||
lifecycleHooks []func(context.Context)
|
||||
}
|
||||
|
||||
// SetEventDispatcher wires the EventDispatcher into the Service. The
|
||||
@@ -69,12 +74,42 @@ type Service struct {
|
||||
// per-event store reads and needs no notification on install/enable/disable.
|
||||
func (s *Service) SetEventDispatcher(d *EventDispatcher) { s.dispatcher = d }
|
||||
|
||||
// OnLifecycleChange is a public hook invoked by API handlers and the installer
|
||||
// after every install / enable / disable / uninstall. It currently does
|
||||
// nothing — the dispatcher reads the live installation set on each event —
|
||||
// but the entry point is preserved so future cache-warming logic has a place
|
||||
// to attach without touching every call site again.
|
||||
func (s *Service) OnLifecycleChange(_ context.Context) {}
|
||||
// AddLifecycleHook registers a callback invoked after plugin install, enable,
|
||||
// disable, uninstall, or preload lifecycle changes.
|
||||
func (s *Service) AddLifecycleHook(hook func(context.Context)) {
|
||||
if s == nil || hook == nil {
|
||||
return
|
||||
}
|
||||
s.lifecycleMu.Lock()
|
||||
s.lifecycleHooks = append(s.lifecycleHooks, hook)
|
||||
s.lifecycleMu.Unlock()
|
||||
}
|
||||
|
||||
// OnLifecycleChange is invoked by API handlers and the installer after every
|
||||
// plugin lifecycle mutation. Hooks should be best-effort and log their own
|
||||
// errors so plugin admin operations are not failed by secondary cache refreshes.
|
||||
func (s *Service) OnLifecycleChange(ctx context.Context) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.lifecycleMu.RLock()
|
||||
hooks := append([]func(context.Context){}, s.lifecycleHooks...)
|
||||
s.lifecycleMu.RUnlock()
|
||||
for _, hook := range hooks {
|
||||
func(hook func(context.Context)) {
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
slog.Error(
|
||||
"plugin lifecycle hook panicked; continuing",
|
||||
"panic", recovered,
|
||||
"stack", string(debug.Stack()),
|
||||
)
|
||||
}
|
||||
}()
|
||||
hook(ctx)
|
||||
}(hook)
|
||||
}
|
||||
}
|
||||
|
||||
func NewService(
|
||||
repositories *RepositoryStore,
|
||||
@@ -411,6 +446,18 @@ func (s *Service) MetadataProviderClient(
|
||||
return client.MetadataProvider(capabilityID)
|
||||
}
|
||||
|
||||
func (s *Service) MarkerProviderClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
capabilityID string,
|
||||
) (*pluginhost.MarkerProviderClient, error) {
|
||||
client, err := s.ensureClient(ctx, installationID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.MarkerProvider(capabilityID)
|
||||
}
|
||||
|
||||
func (s *Service) ScheduledTaskClient(
|
||||
ctx context.Context,
|
||||
installationID int,
|
||||
|
||||
@@ -199,6 +199,10 @@ func (f *fakePluginClient) MetadataProvider(string) (*pluginhost.MetadataProvide
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginClient) MarkerProvider(string) (*pluginhost.MarkerProviderClient, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (f *fakePluginClient) MediaAnalyzer(string) (*pluginhost.MediaAnalyzerClient, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package plugins
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestServiceLifecycleChangeContinuesAfterHookPanic(t *testing.T) {
|
||||
var service Service
|
||||
called := false
|
||||
service.AddLifecycleHook(func(context.Context) {
|
||||
panic("boom")
|
||||
})
|
||||
service.AddLifecycleHook(func(context.Context) {
|
||||
called = true
|
||||
})
|
||||
|
||||
service.OnLifecycleChange(context.Background())
|
||||
|
||||
if !called {
|
||||
t.Fatal("expected later lifecycle hook to run after earlier hook panicked")
|
||||
}
|
||||
}
|
||||
@@ -222,10 +222,11 @@ func (s *Service) GetDetail(ctx context.Context, viewer Viewer, mediaType MediaT
|
||||
return nil, err
|
||||
}
|
||||
|
||||
primaryAvailable, err := s.lookupAvailable(ctx, mediaType, []int{raw.ID})
|
||||
primaryPresence, err := s.lookupAvailable(ctx, mediaType, []int{raw.ID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
primaryMatch := primaryPresence[raw.ID]
|
||||
primaryRequests, err := s.store.ListActiveByTMDB(ctx, mediaType, []int{raw.ID})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -258,8 +259,9 @@ func (s *Service) GetDetail(ctx context.Context, viewer Viewer, mediaType MediaT
|
||||
Networks: raw.Networks,
|
||||
Director: raw.Director,
|
||||
Creators: raw.Creators,
|
||||
Availability: availabilityValue(primaryAvailable[raw.ID]),
|
||||
Request: requestStateFor(viewer, policy, primaryAvailable[raw.ID], primaryRequests[raw.ID]),
|
||||
Availability: availabilityValue(primaryMatch.Available),
|
||||
LibraryContentID: primaryMatch.ContentID,
|
||||
Request: requestStateFor(viewer, policy, primaryMatch.Available, primaryRequests[raw.ID]),
|
||||
}
|
||||
if raw.TVDBID > 0 {
|
||||
tvdb := raw.TVDBID
|
||||
@@ -397,6 +399,9 @@ func (s *Service) ListMine(ctx context.Context, viewer Viewer, filter ListFilter
|
||||
if err := s.attachTargets(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reqs, nil
|
||||
}
|
||||
|
||||
@@ -411,6 +416,9 @@ func (s *Service) ListAdmin(ctx context.Context, viewer Viewer, filter ListFilte
|
||||
if err := s.attachTargets(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, reqs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reqs, nil
|
||||
}
|
||||
|
||||
@@ -430,6 +438,49 @@ func (s *Service) attachTargets(ctx context.Context, reqs ...*Request) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) attachLibraryContent(ctx context.Context, reqs ...*Request) error {
|
||||
if s == nil || s.presence == nil || len(reqs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
type requestKey struct {
|
||||
mediaType MediaType
|
||||
tmdbID int
|
||||
}
|
||||
|
||||
candidatesByType := map[MediaType][]PresenceCandidate{}
|
||||
requestsByKey := map[requestKey][]*Request{}
|
||||
seen := map[requestKey]bool{}
|
||||
for _, req := range reqs {
|
||||
if req == nil || req.TMDBID <= 0 {
|
||||
continue
|
||||
}
|
||||
key := requestKey{mediaType: req.MediaType, tmdbID: req.TMDBID}
|
||||
requestsByKey[key] = append(requestsByKey[key], req)
|
||||
if seen[key] {
|
||||
continue
|
||||
}
|
||||
seen[key] = true
|
||||
candidatesByType[req.MediaType] = append(candidatesByType[req.MediaType], requestPresenceCandidate(*req))
|
||||
}
|
||||
|
||||
for mediaType, candidates := range candidatesByType {
|
||||
matches, err := s.lookupPresence(ctx, mediaType, candidates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for tmdbID, match := range matches {
|
||||
if !match.Available || strings.TrimSpace(match.ContentID) == "" {
|
||||
continue
|
||||
}
|
||||
for _, req := range requestsByKey[requestKey{mediaType: mediaType, tmdbID: tmdbID}] {
|
||||
req.LibraryContentID = match.ContentID
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) GetRequest(ctx context.Context, viewer Viewer, id string) (*Request, error) {
|
||||
if err := s.ensureRequestsEnabled(ctx); err != nil {
|
||||
return nil, err
|
||||
@@ -444,6 +495,9 @@ func (s *Service) GetRequest(ctx context.Context, viewer Viewer, id string) (*Re
|
||||
if err := s.attachTargets(ctx, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.attachLibraryContent(ctx, req); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
@@ -905,7 +959,7 @@ func (s *Service) enrichPage(ctx context.Context, viewer Viewer, raw *tmdb.Media
|
||||
idsByType[mediaType] = append(idsByType[mediaType], item.ID)
|
||||
}
|
||||
|
||||
available := map[MediaType]map[int]bool{}
|
||||
available := map[MediaType]map[int]PresenceMatch{}
|
||||
active := map[MediaType]map[int]*Request{}
|
||||
for mediaType, ids := range idsByType {
|
||||
presence, err := s.lookupAvailable(ctx, mediaType, ids)
|
||||
@@ -931,21 +985,22 @@ func (s *Service) enrichPage(ctx context.Context, viewer Viewer, raw *tmdb.Media
|
||||
if err != nil || item.ID <= 0 {
|
||||
continue
|
||||
}
|
||||
isAvailable := available[mediaType][item.ID]
|
||||
match := available[mediaType][item.ID]
|
||||
activeRequest := active[mediaType][item.ID]
|
||||
out.Results = append(out.Results, MediaResult{
|
||||
MediaType: mediaType,
|
||||
TMDBID: item.ID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterPath: item.PosterPath,
|
||||
BackdropPath: item.BackdropPath,
|
||||
ReleaseDate: item.ReleaseDate,
|
||||
Popularity: item.Popularity,
|
||||
VoteAverage: item.VoteAverage,
|
||||
Availability: availabilityValue(isAvailable),
|
||||
Request: requestStateFor(viewer, policy, isAvailable, activeRequest),
|
||||
MediaType: mediaType,
|
||||
TMDBID: item.ID,
|
||||
Title: item.Title,
|
||||
Year: item.Year,
|
||||
Overview: item.Overview,
|
||||
PosterPath: item.PosterPath,
|
||||
BackdropPath: item.BackdropPath,
|
||||
ReleaseDate: item.ReleaseDate,
|
||||
Popularity: item.Popularity,
|
||||
VoteAverage: item.VoteAverage,
|
||||
Availability: availabilityValue(match.Available),
|
||||
LibraryContentID: match.ContentID,
|
||||
Request: requestStateFor(viewer, policy, match.Available, activeRequest),
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
@@ -958,14 +1013,6 @@ func (s *Service) lookupPresence(ctx context.Context, mediaType MediaType, candi
|
||||
return s.presence.Lookup(ctx, mediaType, candidates)
|
||||
}
|
||||
|
||||
func availabilityBoolMap(matches map[int]PresenceMatch) map[int]bool {
|
||||
out := map[int]bool{}
|
||||
for id, match := range matches {
|
||||
out[id] = match.Available
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func requestPresenceCandidate(req Request) PresenceCandidate {
|
||||
candidate := PresenceCandidate{
|
||||
TMDBID: req.TMDBID,
|
||||
@@ -1057,9 +1104,9 @@ func tmdbMediaType(mediaType MediaType) string {
|
||||
return "movie"
|
||||
}
|
||||
|
||||
func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids []int) (map[int]bool, error) {
|
||||
func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids []int) (map[int]PresenceMatch, error) {
|
||||
if s.presence == nil {
|
||||
return map[int]bool{}, nil
|
||||
return map[int]PresenceMatch{}, nil
|
||||
}
|
||||
candidates := make([]PresenceCandidate, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
@@ -1072,7 +1119,7 @@ func (s *Service) lookupAvailable(ctx context.Context, mediaType MediaType, ids
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return availabilityBoolMap(matches), nil
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
func (s *Service) enrichExternalIDs(ctx context.Context, input *CreateRequestInput) {
|
||||
|
||||
@@ -3,6 +3,7 @@ package requests
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -353,6 +354,34 @@ func TestListMineAttachesTargets(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListMineAttachesLibraryContentID(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.mine = []*Request{{
|
||||
ID: "req-1",
|
||||
Provider: "tmdb",
|
||||
MediaType: MediaTypeMovie,
|
||||
TMDBID: 42,
|
||||
Title: "Test Movie",
|
||||
Status: StatusCompleted,
|
||||
Outcome: OutcomeActive,
|
||||
RequestedByUserID: 1,
|
||||
}}
|
||||
presence := &fakePresence{available: map[MediaType]map[int]bool{
|
||||
MediaTypeMovie: {42: true},
|
||||
}}
|
||||
|
||||
got, err := NewService(store, &fakeTMDBClient{}, presence).ListMine(context.Background(), testViewer(1), ListFilter{})
|
||||
if err != nil {
|
||||
t.Fatalf("ListMine returned error: %v", err)
|
||||
}
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("ListMine returned %d requests, want 1", len(got))
|
||||
}
|
||||
if got[0].LibraryContentID != "movie-42" {
|
||||
t.Fatalf("library content id = %q, want movie-42", got[0].LibraryContentID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateRequestBlocksWhenHydratedTVDBIDIsAvailable(t *testing.T) {
|
||||
store := newFakeStore()
|
||||
store.settings.RequestsEnabled = true
|
||||
@@ -460,6 +489,9 @@ func TestSearchMarksSeriesAvailableByHydratedTVDBID(t *testing.T) {
|
||||
if got := page.Results[0].Availability; got != AvailabilityAvailable {
|
||||
t.Fatalf("availability = %q, want available", got)
|
||||
}
|
||||
if got := page.Results[0].LibraryContentID; got != "series-201992" {
|
||||
t.Fatalf("library content id = %q, want series-201992", got)
|
||||
}
|
||||
if page.Results[0].Request.Reason != "already_available" {
|
||||
t.Fatalf("request reason = %q, want already_available", page.Results[0].Request.Reason)
|
||||
}
|
||||
@@ -1594,18 +1626,30 @@ func (f *fakePresence) Lookup(_ context.Context, mediaType MediaType, candidates
|
||||
f.got = append(f.got, candidates...)
|
||||
for _, candidate := range candidates {
|
||||
if f.available != nil && f.available[mediaType][candidate.TMDBID] {
|
||||
out[candidate.TMDBID] = PresenceMatch{Available: true, MatchedProvider: "tmdb"}
|
||||
out[candidate.TMDBID] = PresenceMatch{
|
||||
Available: true,
|
||||
ContentID: fakePresenceContentID(mediaType, candidate.TMDBID),
|
||||
MatchedProvider: "tmdb",
|
||||
}
|
||||
continue
|
||||
}
|
||||
if candidate.TVDBID != nil && f.byTVDB != nil {
|
||||
if tmdbID, ok := f.byTVDB[mediaType][*candidate.TVDBID]; ok && tmdbID == candidate.TMDBID {
|
||||
out[candidate.TMDBID] = PresenceMatch{Available: true, MatchedProvider: "tvdb"}
|
||||
out[candidate.TMDBID] = PresenceMatch{
|
||||
Available: true,
|
||||
ContentID: fakePresenceContentID(mediaType, candidate.TMDBID),
|
||||
MatchedProvider: "tvdb",
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fakePresenceContentID(mediaType MediaType, tmdbID int) string {
|
||||
return fmt.Sprintf("%s-%d", mediaType, tmdbID)
|
||||
}
|
||||
|
||||
func (f *fakePresence) LookupTMDB(_ context.Context, mediaType MediaType, ids []int) (map[int]bool, error) {
|
||||
out := map[int]bool{}
|
||||
for _, id := range ids {
|
||||
|
||||
+15
-12
@@ -144,6 +144,7 @@ type Request struct {
|
||||
Targets []Target `json:"targets,omitempty"`
|
||||
ExternalID string `json:"external_id,omitempty"`
|
||||
ExternalStatus string `json:"external_status,omitempty"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
@@ -169,18 +170,19 @@ type RequestState struct {
|
||||
}
|
||||
|
||||
type MediaResult struct {
|
||||
MediaType MediaType `json:"media_type"`
|
||||
TMDBID int `json:"tmdb_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterPath string `json:"poster_path,omitempty"`
|
||||
BackdropPath string `json:"backdrop_path,omitempty"`
|
||||
ReleaseDate string `json:"release_date,omitempty"`
|
||||
Popularity float64 `json:"popularity,omitempty"`
|
||||
VoteAverage float64 `json:"vote_average,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
Request RequestState `json:"request"`
|
||||
MediaType MediaType `json:"media_type"`
|
||||
TMDBID int `json:"tmdb_id"`
|
||||
Title string `json:"title"`
|
||||
Year int `json:"year,omitempty"`
|
||||
Overview string `json:"overview,omitempty"`
|
||||
PosterPath string `json:"poster_path,omitempty"`
|
||||
BackdropPath string `json:"backdrop_path,omitempty"`
|
||||
ReleaseDate string `json:"release_date,omitempty"`
|
||||
Popularity float64 `json:"popularity,omitempty"`
|
||||
VoteAverage float64 `json:"vote_average,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
Request RequestState `json:"request"`
|
||||
}
|
||||
|
||||
type MediaPage struct {
|
||||
@@ -228,6 +230,7 @@ type MediaDetail struct {
|
||||
Creators []string `json:"creators,omitempty"`
|
||||
Recommendations []MediaResult `json:"recommendations,omitempty"`
|
||||
Availability Availability `json:"availability"`
|
||||
LibraryContentID string `json:"library_content_id,omitempty"`
|
||||
Request RequestState `json:"request"`
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ func TestSupportsAudioFile(t *testing.T) {
|
||||
{"audio.M4A", true},
|
||||
{"sample.flac", true},
|
||||
{"podcast.opus", true},
|
||||
{"track.ogg", true},
|
||||
{"poster.jpg", false},
|
||||
{"movie.mkv", false},
|
||||
{"", false},
|
||||
|
||||
@@ -126,7 +126,8 @@ func parseAudiobookFolder(ctx context.Context, ffprobePath string, folderPath st
|
||||
}
|
||||
|
||||
// Multi-file case: read header from the first file, synthesize one
|
||||
// chapter per file with title = filename stem.
|
||||
// chapter per file with title = filename stem. A 0..0 range means the
|
||||
// chapter duration is unknown; consumers should treat it as non-seekable.
|
||||
probedFirst, err := ProbeFile(ctx, ffprobePath, audioFiles[0])
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("probe first audiobook file %s: %w", audioFiles[0], err)
|
||||
|
||||
@@ -20,6 +20,14 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
type filesystemRootContentFinder interface {
|
||||
FindContentIDByRootPath(ctx context.Context, folderID int, rootPath, preferredType string) (string, error)
|
||||
}
|
||||
|
||||
type filesystemMediaItemWriter interface {
|
||||
Upsert(ctx context.Context, item *models.MediaItem) error
|
||||
}
|
||||
|
||||
// audiobookDiskFile is the on-disk projection used by audiobookFolderUnchanged.
|
||||
// Path is the absolute file path; Size and ModTime come from os.Stat.
|
||||
type audiobookDiskFile struct {
|
||||
@@ -150,7 +158,13 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF
|
||||
// for the duration of a 240k-folder scan.
|
||||
var candidates []string
|
||||
for _, root := range folder.Paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
walkErr := filepath.WalkDir(root, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if walkErr != nil {
|
||||
slog.Warn("audiobook scan: walk error", "path", path, "error", walkErr)
|
||||
return nil
|
||||
@@ -196,6 +210,9 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF
|
||||
processed int64
|
||||
failed int64
|
||||
skipped int64
|
||||
failMu sync.Mutex
|
||||
failures []error
|
||||
cancelErr error
|
||||
)
|
||||
start := time.Now()
|
||||
for i := 0; i < workers; i++ {
|
||||
@@ -207,7 +224,18 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF
|
||||
return
|
||||
}
|
||||
if err := s.reconcileAudiobookFolder(ctx, folder, path, &skipped); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
failMu.Lock()
|
||||
if cancelErr == nil {
|
||||
cancelErr = err
|
||||
}
|
||||
failMu.Unlock()
|
||||
return
|
||||
}
|
||||
atomic.AddInt64(&failed, 1)
|
||||
failMu.Lock()
|
||||
failures = append(failures, fmt.Errorf("%s: %w", path, err))
|
||||
failMu.Unlock()
|
||||
slog.Warn("audiobook scan: folder failed",
|
||||
"folder_id", folder.ID,
|
||||
"path", path,
|
||||
@@ -237,6 +265,12 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF
|
||||
}
|
||||
close(ch)
|
||||
wg.Wait()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if cancelErr != nil {
|
||||
return cancelErr
|
||||
}
|
||||
|
||||
slog.Info("audiobook scan: completed",
|
||||
"folder_id", folder.ID,
|
||||
@@ -245,6 +279,13 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF
|
||||
"skipped", atomic.LoadInt64(&skipped),
|
||||
"elapsed_sec", int(time.Since(start).Seconds()),
|
||||
)
|
||||
if processedCount := atomic.LoadInt64(&processed); processedCount > 0 {
|
||||
failedCount := atomic.LoadInt64(&failed)
|
||||
skippedCount := atomic.LoadInt64(&skipped)
|
||||
if failedCount > 0 && skippedCount == 0 && failedCount == processedCount {
|
||||
return fmt.Errorf("audiobook scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -268,7 +309,7 @@ func (s *Scanner) reconcileAudiobookFolder(ctx context.Context, folder *models.M
|
||||
return fmt.Errorf("parse audiobook folder %s: %w", folderPath, err)
|
||||
}
|
||||
|
||||
contentID, err := s.upsertAudiobookMediaItem(ctx, parsed)
|
||||
contentID, err := s.upsertAudiobookMediaItem(ctx, folder.ID, folderPath, parsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert audiobook item: %w", err)
|
||||
}
|
||||
@@ -311,17 +352,30 @@ func (s *Scanner) reconcileAudiobookFolder(ctx context.Context, folder *models.M
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertAudiobookMediaItem looks up an existing media_items row by the
|
||||
// audio file path (the most stable identity for an audiobook), updates
|
||||
// it if found, or creates a new row. Audiobook titles routinely include
|
||||
// narrator/edition suffixes that vary across copies of the same book, so
|
||||
// the previous title+year lookup risked collapsing separate narrations
|
||||
// into one row when titles were cleaned. File path side-steps that.
|
||||
func (s *Scanner) upsertAudiobookMediaItem(ctx context.Context, book *parsedAudiobook) (string, error) {
|
||||
// upsertAudiobookMediaItem reuses an item already linked to the same filesystem
|
||||
// root, then falls back to the stricter ABS duplicate rule, or creates a new row.
|
||||
// It intentionally avoids title/year-only dedupe because audiobooks often share
|
||||
// titles, unknown years, or edition-specific credits. Returns the content_id used.
|
||||
func (s *Scanner) upsertAudiobookMediaItem(ctx context.Context, folderID int, folderPath string, book *parsedAudiobook) (string, error) {
|
||||
if s.itemRepo == nil {
|
||||
return "", fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
if s.fileRepo == nil {
|
||||
return "", fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
existingID, err := s.fileRepo.FindContentIDByRootPath(ctx, folderID, folderPath, "audiobook")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find audiobook by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
cleanTitle := stripNarratorSuffix(book.Title)
|
||||
if cleanTitle == "" {
|
||||
cleanTitle = book.Title
|
||||
}
|
||||
|
||||
if existing := s.findAudiobookByFilePath(ctx, book); existing != nil {
|
||||
applyBookToMediaItem(existing, book)
|
||||
@@ -351,6 +405,39 @@ func (s *Scanner) upsertAudiobookMediaItem(ctx context.Context, book *parsedAudi
|
||||
return existing.ContentID, nil
|
||||
}
|
||||
|
||||
return createAudiobookMediaItem(ctx, s.itemRepo, book, cleanTitle)
|
||||
}
|
||||
|
||||
func resolveAudiobookMediaItem(
|
||||
ctx context.Context,
|
||||
rootFinder filesystemRootContentFinder,
|
||||
itemWriter filesystemMediaItemWriter,
|
||||
folderID int,
|
||||
folderPath string,
|
||||
book *parsedAudiobook,
|
||||
) (string, error) {
|
||||
if rootFinder == nil {
|
||||
return "", fmt.Errorf("root content finder not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return "", fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, folderPath, "audiobook")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find audiobook by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
cleanTitle := stripNarratorSuffix(book.Title)
|
||||
if cleanTitle == "" {
|
||||
cleanTitle = book.Title
|
||||
}
|
||||
return createAudiobookMediaItem(ctx, itemWriter, book, cleanTitle)
|
||||
}
|
||||
|
||||
func createAudiobookMediaItem(ctx context.Context, itemWriter filesystemMediaItemWriter, book *parsedAudiobook, cleanTitle string) (string, error) {
|
||||
id, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate content_id: %w", err)
|
||||
@@ -363,7 +450,10 @@ func (s *Scanner) upsertAudiobookMediaItem(ctx context.Context, book *parsedAudi
|
||||
Year: book.Year,
|
||||
}
|
||||
applyBookToMediaItem(item, book)
|
||||
if err := s.itemRepo.Upsert(ctx, item); err != nil {
|
||||
if item.SortTitle == "" {
|
||||
item.SortTitle = titleutil.DeriveDefaultSortTitle(item.Title)
|
||||
}
|
||||
if err := itemWriter.Upsert(ctx, item); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
@@ -575,24 +665,24 @@ func (s *Scanner) upsertAudiobookMediaFiles(
|
||||
}
|
||||
|
||||
mf := models.MediaFile{
|
||||
ContentID: contentID,
|
||||
MediaFolderID: folder.ID,
|
||||
CanonicalRootPath: folderPath,
|
||||
ObservedRootPath: folderPath,
|
||||
ContentGroupKey: contentID,
|
||||
GroupKeyVersion: 1,
|
||||
BaseTitle: book.Title,
|
||||
BaseYear: book.Year,
|
||||
BaseType: "audiobook",
|
||||
IdentityConfidence: "high",
|
||||
FilePath: af.Path,
|
||||
Chapters: chapters,
|
||||
ProbeSource: "local",
|
||||
Duration: af.Duration,
|
||||
Bitrate: af.Bitrate,
|
||||
CodecAudio: af.CodecAudio,
|
||||
Container: af.Container,
|
||||
AudioChannels: af.AudioChannels,
|
||||
ContentID: contentID,
|
||||
MediaFolderID: folder.ID,
|
||||
CanonicalRootPath: folderPath,
|
||||
ObservedRootPath: folderPath,
|
||||
ContentGroupKey: contentID,
|
||||
GroupKeyVersion: 1,
|
||||
BaseTitle: book.Title,
|
||||
BaseYear: book.Year,
|
||||
BaseType: "audiobook",
|
||||
IdentityConfidence: audiobookIdentityConfidence(book, af),
|
||||
FilePath: af.Path,
|
||||
Chapters: chapters,
|
||||
ProbeSource: "local",
|
||||
Duration: af.Duration,
|
||||
Bitrate: af.Bitrate,
|
||||
CodecAudio: af.CodecAudio,
|
||||
Container: af.Container,
|
||||
AudioChannels: af.AudioChannels,
|
||||
}
|
||||
|
||||
if _, err := s.fileRepo.Upsert(ctx, mf); err != nil {
|
||||
@@ -641,7 +731,8 @@ func audiobookPeopleCreditsEqual(existing []models.ItemPerson, desired []audiobo
|
||||
}
|
||||
|
||||
// upsertAudiobookPeople upserts author and narrator rows into item_people,
|
||||
// using the PersonRepository to find-or-create each person by name.
|
||||
// using the PersonRepository to find-or-create each person by name. When the
|
||||
// parsed credits differ, ReplacePeople makes scanner-derived credits authoritative.
|
||||
// Skips the DELETE+INSERT entirely when the existing credit set already
|
||||
// matches the desired set (case-insensitive on name, exact on kind).
|
||||
func (s *Scanner) upsertAudiobookPeople(ctx context.Context, contentID string, book *parsedAudiobook) error {
|
||||
@@ -735,6 +826,38 @@ func (s *Scanner) upsertAudiobookSeries(ctx context.Context, contentID string, b
|
||||
return nil
|
||||
}
|
||||
|
||||
func audiobookIdentityConfidence(book *parsedAudiobook, file parsedAudiobookFile) string {
|
||||
if book == nil {
|
||||
return "low"
|
||||
}
|
||||
|
||||
score := 0
|
||||
if book.Title != "" {
|
||||
score++
|
||||
}
|
||||
if book.Author != "" || book.Narrator != "" {
|
||||
score++
|
||||
}
|
||||
if book.Year > 0 {
|
||||
score++
|
||||
}
|
||||
for _, ch := range file.Chapters {
|
||||
if ch.Title != "" && ch.EndSeconds > ch.StartSeconds {
|
||||
score++
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
switch {
|
||||
case score >= 4:
|
||||
return "high"
|
||||
case score > 0:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
|
||||
// floatPtrEqual returns true when two *float64 values represent the same
|
||||
// state — both nil, or both non-nil and equal. Used to skip audiobook_series
|
||||
// re-upserts when only the series_index needs to be NULL=NULL compared.
|
||||
|
||||
@@ -2,14 +2,39 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type fakeRootContentFinder struct {
|
||||
contentID string
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeRootContentFinder) FindContentIDByRootPath(context.Context, int, string, string) (string, error) {
|
||||
f.calls++
|
||||
return f.contentID, f.err
|
||||
}
|
||||
|
||||
type fakeFilesystemItemWriter struct {
|
||||
upserts []*models.MediaItem
|
||||
}
|
||||
|
||||
func (f *fakeFilesystemItemWriter) Upsert(_ context.Context, item *models.MediaItem) error {
|
||||
cp := *item
|
||||
f.upserts = append(f.upserts, &cp)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPopulateFromTags_SeriesDoesNotFallBackToAlbum(t *testing.T) {
|
||||
// In audiobook tagging the `album` tag holds the book title, not the
|
||||
// series name. populateFromTags must NOT use it as a series fallback,
|
||||
@@ -123,6 +148,134 @@ func TestParseAudiobookFolderMultiFile(t *testing.T) {
|
||||
if f.Chapters[0].Title != wantStem {
|
||||
t.Errorf("file %d: chapter title = %q, want %q", i, f.Chapters[0].Title, wantStem)
|
||||
}
|
||||
if f.Chapters[0].StartSeconds != 0 || f.Chapters[0].EndSeconds <= f.Chapters[0].StartSeconds {
|
||||
t.Errorf("file %d: synthesized chapter range = %.3f..%.3f, want positive probed duration", i, f.Chapters[0].StartSeconds, f.Chapters[0].EndSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAudiobookIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
book := &parsedAudiobook{Title: "Tagged Book", Author: "Author", Narrator: "Narrator", Year: 2024}
|
||||
file := parsedAudiobookFile{Chapters: []ChapterInfo{{Title: "One", StartSeconds: 0, EndSeconds: 10}}}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
|
||||
book = &parsedAudiobook{Title: "Tagged Book"}
|
||||
file = parsedAudiobookFile{Chapters: []ChapterInfo{{Title: "Unknown duration"}}}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
|
||||
book = &parsedAudiobook{}
|
||||
file = parsedAudiobookFile{}
|
||||
if got := audiobookIdentityConfidence(book, file); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAudiobookFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
bookDir := filepath.Join(root, "bad-book")
|
||||
if err := os.Mkdir(bookDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir book dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(bookDir, "chapter.mp3"), []byte("not real audio"), 0o644); err != nil {
|
||||
t.Fatalf("write fake audio: %v", err)
|
||||
}
|
||||
|
||||
s := &Scanner{ffprobePath: "definitely-missing-ffprobe"}
|
||||
err := s.ScanAudiobookFolder(context.Background(), &models.MediaFolder{ID: 42, Paths: []string{root}})
|
||||
if err == nil {
|
||||
t.Fatal("ScanAudiobookFolder returned nil, want aggregate failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "folder_id=42") {
|
||||
t.Fatalf("error = %q, want folder id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanAudiobookFolderReturnsCanceledContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanAudiobookFolder(ctx, &models.MediaFolder{ID: 42, Paths: []string{root}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ScanAudiobookFolder error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemReusesRootScopedContentID(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{contentID: "book-root-id"}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Author/Same Title",
|
||||
&parsedAudiobook{Title: "Same Title", Year: 0, Author: "Author A"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAudiobookMediaItem: %v", err)
|
||||
}
|
||||
if got != "book-root-id" {
|
||||
t.Fatalf("contentID = %q, want root-scoped id", got)
|
||||
}
|
||||
if finder.calls != 1 {
|
||||
t.Fatalf("root finder calls = %d, want 1", finder.calls)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Other Author/Same Title",
|
||||
&parsedAudiobook{Title: "Same Title", Year: 0, Author: "Author B"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolveAudiobookMediaItem: %v", err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatal("contentID is empty")
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].ContentID != got || writer.upserts[0].Type != "audiobook" {
|
||||
t.Fatalf("upserted item = %+v, contentID %q", writer.upserts[0], got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveAudiobookMediaItemPropagatesRootLookupError(t *testing.T) {
|
||||
wantErr := errors.New("root lookup failed")
|
||||
finder := &fakeRootContentFinder{err: wantErr}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
_, err := resolveAudiobookMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
7,
|
||||
"/library/Author/Book",
|
||||
&parsedAudiobook{Title: "Book"},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("upserts = %d, want 0", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+586
-120
@@ -984,6 +984,7 @@ type segmentState struct {
|
||||
provider *string
|
||||
confidence *float64
|
||||
algorithm *string
|
||||
detectedAt *time.Time
|
||||
}
|
||||
|
||||
// applySegmentPatch merges the patched start/end into the segment state, then
|
||||
@@ -993,10 +994,14 @@ type segmentState struct {
|
||||
func applySegmentPatch(
|
||||
state *segmentState,
|
||||
legacySharedSource *string,
|
||||
update MarkerUpdate,
|
||||
source string,
|
||||
provider *string,
|
||||
confidence *float64,
|
||||
algorithm string,
|
||||
patchStart, patchEnd *float64,
|
||||
duration float64,
|
||||
segmentName string,
|
||||
mutationAt time.Time,
|
||||
) (bool, error) {
|
||||
if patchStart == nil && patchEnd == nil {
|
||||
return false, nil
|
||||
@@ -1024,23 +1029,62 @@ func applySegmentPatch(
|
||||
if effectiveSource == nil && state.start != nil && state.end != nil {
|
||||
effectiveSource = legacySharedSource
|
||||
}
|
||||
if !markers.CanWriteMarker(effectiveSource, state.confidence, update.MarkersSource, update.MarkersConfidence) {
|
||||
if !markers.CanWriteMarker(effectiveSource, state.confidence, source, confidence) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
src := update.MarkersSource
|
||||
algo := markerAlgorithm(update)
|
||||
src := source
|
||||
algo := algorithm
|
||||
nextState := segmentState{
|
||||
start: nextStart,
|
||||
end: nextEnd,
|
||||
source: &src,
|
||||
provider: provider,
|
||||
confidence: confidence,
|
||||
algorithm: &algo,
|
||||
detectedAt: &mutationAt,
|
||||
}
|
||||
if segmentEqual(*state, nextState) {
|
||||
return false, nil
|
||||
}
|
||||
state.start = nextStart
|
||||
state.end = nextEnd
|
||||
state.source = &src
|
||||
state.provider = update.MarkersProvider
|
||||
state.confidence = update.MarkersConfidence
|
||||
state.provider = provider
|
||||
state.confidence = confidence
|
||||
state.algorithm = &algo
|
||||
state.detectedAt = &mutationAt
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// segmentEqual reports whether two segment states are byte-equivalent. Used to
|
||||
// detect no-op writes so the transaction can short-circuit without a SQL UPDATE.
|
||||
// resolveSegmentProvenance returns the source/provider/confidence/algorithm to
|
||||
// write for a segment: the per-segment override when present, otherwise the
|
||||
// update's shared Markers* values. The algorithm always falls back to
|
||||
// external:<source> so writes carry an algorithm tag.
|
||||
func resolveSegmentProvenance(update MarkerUpdate, override *SegmentProvenance) (source string, provider *string, confidence *float64, algorithm string) {
|
||||
source = update.MarkersSource
|
||||
provider = update.MarkersProvider
|
||||
confidence = update.MarkersConfidence
|
||||
algorithm = update.MarkersAlgorithm
|
||||
if override != nil {
|
||||
if override.Source != "" {
|
||||
source = override.Source
|
||||
}
|
||||
provider = override.Provider
|
||||
confidence = override.Confidence
|
||||
if override.Algorithm != "" {
|
||||
algorithm = override.Algorithm
|
||||
}
|
||||
}
|
||||
if algorithm == "" {
|
||||
algorithm = "external:" + source
|
||||
}
|
||||
return source, provider, confidence, algorithm
|
||||
}
|
||||
|
||||
// segmentEqual reports whether two segment states are semantically equivalent.
|
||||
// detected_at is intentionally ignored so writing the same marker value does
|
||||
// not refresh provenance timestamps or create audit noise.
|
||||
func segmentEqual(a, b segmentState) bool {
|
||||
return ptrFloatEqual(a.start, b.start) &&
|
||||
ptrFloatEqual(a.end, b.end) &&
|
||||
@@ -1055,22 +1099,243 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
if update.MarkersSource == "" {
|
||||
return false, fmt.Errorf("marker source is required")
|
||||
}
|
||||
return r.UpsertAndClearMarkers(ctx, fileID, update, nil)
|
||||
}
|
||||
|
||||
// ClearMarkers nulls the given segment kinds (intro|credits|recap|preview) for
|
||||
// a file, including their provenance columns. Used by the admin manual-marker
|
||||
// API to remove a marker so detection/online fetch can repopulate it. Returns
|
||||
// whether a row was updated.
|
||||
func (r *FileRepository) ClearMarkers(ctx context.Context, fileID int, segments []string) (bool, error) {
|
||||
return r.upsertAndClearMarkers(ctx, fileID, nil, segments)
|
||||
}
|
||||
|
||||
// UpsertAndClearMarkers applies manual marker sets and clears in one row-locking
|
||||
// transaction so mixed PUT bodies cannot partially persist.
|
||||
func (r *FileRepository) UpsertAndClearMarkers(ctx context.Context, fileID int, update MarkerUpdate, clearSegments []string) (bool, error) {
|
||||
return r.upsertAndClearMarkers(ctx, fileID, &update, clearSegments)
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListMarkerEditAudit(ctx context.Context, fileIDs []int, limit int) ([]MarkerEditAuditRow, error) {
|
||||
if len(fileIDs) == 0 {
|
||||
return []MarkerEditAuditRow{}, nil
|
||||
}
|
||||
return r.listMarkerEditAudit(ctx, "WHERE a.media_file_id = ANY ($1)", []any{fileIDs}, limit)
|
||||
}
|
||||
|
||||
func (r *FileRepository) ListAllMarkerEditAudit(ctx context.Context, limit int) ([]MarkerEditAuditRow, error) {
|
||||
return r.listMarkerEditAudit(ctx, "", nil, limit)
|
||||
}
|
||||
|
||||
func (r *FileRepository) listMarkerEditAudit(ctx context.Context, whereClause string, args []any, limit int) ([]MarkerEditAuditRow, error) {
|
||||
limit = normalizeMarkerAuditLimit(limit)
|
||||
args = append(args, limit)
|
||||
limitPlaceholder := len(args)
|
||||
rows, err := r.pool.Query(ctx, fmt.Sprintf(`
|
||||
SELECT a.id,
|
||||
a.media_file_id,
|
||||
NULLIF(CASE WHEN mf.episode_id <> '' THEN mf.episode_id ELSE mf.content_id END, '') AS item_id,
|
||||
NULLIF(COALESCE(CASE WHEN mf.episode_id <> '' THEN 'episode' ELSE mi.type END, mf.base_type), '') AS item_type,
|
||||
NULLIF(COALESCE(e.title, mi.title, mf.base_title), '') AS media_title,
|
||||
NULLIF(mf.file_path, '') AS file_path,
|
||||
a.segment_kind,
|
||||
a.action,
|
||||
a.before_marker,
|
||||
a.after_marker,
|
||||
a.user_id,
|
||||
u.username,
|
||||
a.impersonator_user_id,
|
||||
iu.username,
|
||||
a.api_key_id,
|
||||
a.request_id,
|
||||
a.client_ip::text,
|
||||
a.user_agent,
|
||||
a.created_at
|
||||
FROM marker_edit_audit a
|
||||
LEFT JOIN media_files mf ON mf.id = a.media_file_id
|
||||
LEFT JOIN media_items mi ON mi.content_id = mf.content_id
|
||||
LEFT JOIN episodes e ON e.content_id = mf.episode_id
|
||||
LEFT JOIN users u ON u.id = a.user_id
|
||||
LEFT JOIN users iu ON iu.id = a.impersonator_user_id
|
||||
%s
|
||||
ORDER BY a.created_at DESC, a.id DESC
|
||||
LIMIT $%d`, whereClause, limitPlaceholder), args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("list marker edit audit: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
out := make([]MarkerEditAuditRow, 0, limit)
|
||||
for rows.Next() {
|
||||
var row MarkerEditAuditRow
|
||||
var beforeJSON []byte
|
||||
var afterJSON []byte
|
||||
if err := rows.Scan(
|
||||
&row.ID,
|
||||
&row.MediaFileID,
|
||||
&row.ItemID,
|
||||
&row.ItemType,
|
||||
&row.MediaTitle,
|
||||
&row.FilePath,
|
||||
&row.SegmentKind,
|
||||
&row.Action,
|
||||
&beforeJSON,
|
||||
&afterJSON,
|
||||
&row.UserID,
|
||||
&row.Username,
|
||||
&row.ImpersonatorUserID,
|
||||
&row.ImpersonatorUsername,
|
||||
&row.APIKeyID,
|
||||
&row.RequestID,
|
||||
&row.ClientIP,
|
||||
&row.UserAgent,
|
||||
&row.CreatedAt,
|
||||
); err != nil {
|
||||
return nil, fmt.Errorf("scan marker edit audit row: %w", err)
|
||||
}
|
||||
before, err := unmarshalMarkerAuditSegment(beforeJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
after, err := unmarshalMarkerAuditSegment(afterJSON)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
row.Before = before
|
||||
row.After = after
|
||||
out = append(out, row)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate marker edit audit rows: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func normalizeMarkerAuditLimit(limit int) int {
|
||||
if limit <= 0 {
|
||||
return 25
|
||||
}
|
||||
if limit > 100 {
|
||||
return 100
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
type markerSegmentFlags struct {
|
||||
intro bool
|
||||
credits bool
|
||||
recap bool
|
||||
preview bool
|
||||
}
|
||||
|
||||
func (f markerSegmentFlags) any() bool {
|
||||
return f.intro || f.credits || f.recap || f.preview
|
||||
}
|
||||
|
||||
type markerMutationState struct {
|
||||
duration float64
|
||||
existingSource *string
|
||||
existingConfidence *float64
|
||||
intro segmentState
|
||||
credits segmentState
|
||||
recap segmentState
|
||||
preview segmentState
|
||||
}
|
||||
|
||||
func (r *FileRepository) upsertAndClearMarkers(ctx context.Context, fileID int, update *MarkerUpdate, clearSegments []string) (bool, error) {
|
||||
hasUpdate := update != nil && update.HasAnySegment()
|
||||
if hasUpdate && update.MarkersSource == "" {
|
||||
return false, fmt.Errorf("marker source is required")
|
||||
}
|
||||
clearFlags, err := markerClearFlags(clearSegments)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !hasUpdate && !clearFlags.any() {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("begin marker upsert transaction: %w", err)
|
||||
return false, fmt.Errorf("begin marker mutation transaction: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var (
|
||||
duration float64
|
||||
existingSource *string
|
||||
existingConfidence *float64
|
||||
intro segmentState
|
||||
credits segmentState
|
||||
recap segmentState
|
||||
preview segmentState
|
||||
)
|
||||
state, err := loadMarkerMutationState(ctx, tx, fileID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
before := state
|
||||
mutationAt := time.Now().UTC()
|
||||
changed := markerSegmentFlags{}
|
||||
|
||||
if hasUpdate {
|
||||
applied, err := applyMarkerUpdateToMutationState(update, &state, mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
changed.intro = changed.intro || applied.intro
|
||||
changed.credits = changed.credits || applied.credits
|
||||
changed.recap = changed.recap || applied.recap
|
||||
changed.preview = changed.preview || applied.preview
|
||||
}
|
||||
if clearFlags.intro {
|
||||
changed.intro = clearSegmentState(&state.intro) || changed.intro
|
||||
}
|
||||
if clearFlags.credits {
|
||||
changed.credits = clearSegmentState(&state.credits) || changed.credits
|
||||
}
|
||||
if clearFlags.recap {
|
||||
changed.recap = clearSegmentState(&state.recap) || changed.recap
|
||||
}
|
||||
if clearFlags.preview {
|
||||
changed.preview = clearSegmentState(&state.preview) || changed.preview
|
||||
}
|
||||
if !changed.any() {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker no-op transaction: %w", err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
wrote, err := writeMarkerMutationState(ctx, tx, fileID, state)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if wrote {
|
||||
if audit, ok := MarkerAuditContextFromContext(ctx); ok {
|
||||
if err := insertMarkerEditAuditRows(ctx, tx, fileID, before, state, changed, audit, mutationAt); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker mutation transaction: %w", err)
|
||||
}
|
||||
return wrote, nil
|
||||
}
|
||||
|
||||
func markerClearFlags(segments []string) (markerSegmentFlags, error) {
|
||||
var flags markerSegmentFlags
|
||||
for _, seg := range segments {
|
||||
switch seg {
|
||||
case "intro":
|
||||
flags.intro = true
|
||||
case "credits":
|
||||
flags.credits = true
|
||||
case "recap":
|
||||
flags.recap = true
|
||||
case "preview":
|
||||
flags.preview = true
|
||||
default:
|
||||
return markerSegmentFlags{}, fmt.Errorf("invalid marker segment %q", seg)
|
||||
}
|
||||
}
|
||||
return flags, nil
|
||||
}
|
||||
|
||||
func loadMarkerMutationState(ctx context.Context, tx pgx.Tx, fileID int) (markerMutationState, error) {
|
||||
var state markerMutationState
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT COALESCE(duration, 0),
|
||||
markers_source,
|
||||
@@ -1081,120 +1346,132 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
intro_markers_provider,
|
||||
intro_markers_confidence,
|
||||
intro_markers_algorithm,
|
||||
intro_markers_detected_at,
|
||||
credits_start,
|
||||
credits_end,
|
||||
credits_markers_source,
|
||||
credits_markers_provider,
|
||||
credits_markers_confidence,
|
||||
credits_markers_algorithm,
|
||||
credits_markers_detected_at,
|
||||
recap_start,
|
||||
recap_end,
|
||||
recap_markers_source,
|
||||
recap_markers_provider,
|
||||
recap_markers_confidence,
|
||||
recap_markers_algorithm,
|
||||
recap_markers_detected_at,
|
||||
preview_start,
|
||||
preview_end,
|
||||
preview_markers_source,
|
||||
preview_markers_provider,
|
||||
preview_markers_confidence,
|
||||
preview_markers_algorithm
|
||||
preview_markers_algorithm,
|
||||
preview_markers_detected_at
|
||||
FROM media_files WHERE id = $1 FOR UPDATE`,
|
||||
fileID,
|
||||
).Scan(
|
||||
&duration,
|
||||
&existingSource,
|
||||
&existingConfidence,
|
||||
&intro.start, &intro.end, &intro.source, &intro.provider, &intro.confidence, &intro.algorithm,
|
||||
&credits.start, &credits.end, &credits.source, &credits.provider, &credits.confidence, &credits.algorithm,
|
||||
&recap.start, &recap.end, &recap.source, &recap.provider, &recap.confidence, &recap.algorithm,
|
||||
&preview.start, &preview.end, &preview.source, &preview.provider, &preview.confidence, &preview.algorithm,
|
||||
&state.duration,
|
||||
&state.existingSource,
|
||||
&state.existingConfidence,
|
||||
&state.intro.start, &state.intro.end, &state.intro.source, &state.intro.provider, &state.intro.confidence, &state.intro.algorithm, &state.intro.detectedAt,
|
||||
&state.credits.start, &state.credits.end, &state.credits.source, &state.credits.provider, &state.credits.confidence, &state.credits.algorithm, &state.credits.detectedAt,
|
||||
&state.recap.start, &state.recap.end, &state.recap.source, &state.recap.provider, &state.recap.confidence, &state.recap.algorithm, &state.recap.detectedAt,
|
||||
&state.preview.start, &state.preview.end, &state.preview.source, &state.preview.provider, &state.preview.confidence, &state.preview.algorithm, &state.preview.detectedAt,
|
||||
); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, ErrFileNotFound
|
||||
return markerMutationState{}, ErrFileNotFound
|
||||
}
|
||||
return false, fmt.Errorf("load existing marker source: %w", err)
|
||||
return markerMutationState{}, fmt.Errorf("load existing marker source: %w", err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
originalIntro, originalCredits, originalRecap, originalPreview := intro, credits, recap, preview
|
||||
|
||||
introApplied, err := applySegmentPatch(&intro, existingSource, update, update.IntroStart, update.IntroEnd, duration, "intro")
|
||||
func applyMarkerUpdateToMutationState(update *MarkerUpdate, state *markerMutationState, mutationAt time.Time) (markerSegmentFlags, error) {
|
||||
var applied markerSegmentFlags
|
||||
introSrc, introProv, introConf, introAlgo := resolveSegmentProvenance(*update, update.IntroProvenance)
|
||||
var err error
|
||||
applied.intro, err = applySegmentPatch(&state.intro, state.existingSource, introSrc, introProv, introConf, introAlgo, update.IntroStart, update.IntroEnd, state.duration, "intro", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
creditsApplied, err := applySegmentPatch(&credits, existingSource, update, update.CreditsStart, update.CreditsEnd, duration, "credits")
|
||||
creditsSrc, creditsProv, creditsConf, creditsAlgo := resolveSegmentProvenance(*update, update.CreditsProvenance)
|
||||
applied.credits, err = applySegmentPatch(&state.credits, state.existingSource, creditsSrc, creditsProv, creditsConf, creditsAlgo, update.CreditsStart, update.CreditsEnd, state.duration, "credits", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
recapApplied, err := applySegmentPatch(&recap, existingSource, update, update.RecapStart, update.RecapEnd, duration, "recap")
|
||||
recapSrc, recapProv, recapConf, recapAlgo := resolveSegmentProvenance(*update, update.RecapProvenance)
|
||||
applied.recap, err = applySegmentPatch(&state.recap, state.existingSource, recapSrc, recapProv, recapConf, recapAlgo, update.RecapStart, update.RecapEnd, state.duration, "recap", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
previewApplied, err := applySegmentPatch(&preview, existingSource, update, update.PreviewStart, update.PreviewEnd, duration, "preview")
|
||||
previewSrc, previewProv, previewConf, previewAlgo := resolveSegmentProvenance(*update, update.PreviewProvenance)
|
||||
applied.preview, err = applySegmentPatch(&state.preview, state.existingSource, previewSrc, previewProv, previewConf, previewAlgo, update.PreviewStart, update.PreviewEnd, state.duration, "preview", mutationAt)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
anyApplied := introApplied || creditsApplied || recapApplied || previewApplied
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(existingSource, existingConfidence, update, anyApplied)
|
||||
|
||||
if segmentEqual(intro, originalIntro) &&
|
||||
segmentEqual(credits, originalCredits) &&
|
||||
segmentEqual(recap, originalRecap) &&
|
||||
segmentEqual(preview, originalPreview) &&
|
||||
ptrStringEqual(existingSource, nextSource) &&
|
||||
ptrFloatEqual(existingConfidence, nextConfidence) {
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker no-op transaction: %w", err)
|
||||
}
|
||||
return false, nil
|
||||
return markerSegmentFlags{}, err
|
||||
}
|
||||
return applied, nil
|
||||
}
|
||||
|
||||
func writeMarkerMutationState(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
fileID int,
|
||||
state markerMutationState,
|
||||
) (bool, error) {
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
state.existingSource,
|
||||
state.existingConfidence,
|
||||
state.intro,
|
||||
state.credits,
|
||||
state.recap,
|
||||
state.preview,
|
||||
)
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE media_files
|
||||
SET intro_start = $2,
|
||||
intro_end = $3,
|
||||
credits_start = $4,
|
||||
credits_end = $5,
|
||||
recap_start = $6,
|
||||
recap_end = $7,
|
||||
preview_start = $8,
|
||||
preview_end = $9,
|
||||
markers_source = $10,
|
||||
markers_confidence = $11,
|
||||
intro_markers_source = $12,
|
||||
intro_markers_provider = $13,
|
||||
intro_markers_confidence = $14,
|
||||
intro_markers_algorithm = $15,
|
||||
intro_markers_detected_at = CASE WHEN $16 THEN NOW() ELSE intro_markers_detected_at END,
|
||||
credits_markers_source = $17,
|
||||
credits_markers_provider = $18,
|
||||
credits_markers_confidence = $19,
|
||||
credits_markers_algorithm = $20,
|
||||
credits_markers_detected_at = CASE WHEN $21 THEN NOW() ELSE credits_markers_detected_at END,
|
||||
recap_markers_source = $22,
|
||||
recap_markers_provider = $23,
|
||||
recap_markers_confidence = $24,
|
||||
recap_markers_algorithm = $25,
|
||||
recap_markers_detected_at = CASE WHEN $26 THEN NOW() ELSE recap_markers_detected_at END,
|
||||
preview_markers_source = $27,
|
||||
preview_markers_provider = $28,
|
||||
preview_markers_confidence = $29,
|
||||
preview_markers_algorithm = $30,
|
||||
preview_markers_detected_at = CASE WHEN $31 THEN NOW() ELSE preview_markers_detected_at END,
|
||||
SET intro_start = $2::double precision,
|
||||
intro_end = $3::double precision,
|
||||
credits_start = $4::double precision,
|
||||
credits_end = $5::double precision,
|
||||
recap_start = $6::double precision,
|
||||
recap_end = $7::double precision,
|
||||
preview_start = $8::double precision,
|
||||
preview_end = $9::double precision,
|
||||
markers_source = $10::text,
|
||||
markers_confidence = $11::double precision,
|
||||
intro_markers_source = $12::text,
|
||||
intro_markers_provider = $13::text,
|
||||
intro_markers_confidence = $14::double precision,
|
||||
intro_markers_algorithm = $15::text,
|
||||
intro_markers_detected_at = $16::timestamptz,
|
||||
credits_markers_source = $17::text,
|
||||
credits_markers_provider = $18::text,
|
||||
credits_markers_confidence = $19::double precision,
|
||||
credits_markers_algorithm = $20::text,
|
||||
credits_markers_detected_at = $21::timestamptz,
|
||||
recap_markers_source = $22::text,
|
||||
recap_markers_provider = $23::text,
|
||||
recap_markers_confidence = $24::double precision,
|
||||
recap_markers_algorithm = $25::text,
|
||||
recap_markers_detected_at = $26::timestamptz,
|
||||
preview_markers_source = $27::text,
|
||||
preview_markers_provider = $28::text,
|
||||
preview_markers_confidence = $29::double precision,
|
||||
preview_markers_algorithm = $30::text,
|
||||
preview_markers_detected_at = $31::timestamptz,
|
||||
updated_at = NOW()
|
||||
WHERE id = $1
|
||||
`,
|
||||
fileID,
|
||||
intro.start, intro.end,
|
||||
credits.start, credits.end,
|
||||
recap.start, recap.end,
|
||||
preview.start, preview.end,
|
||||
state.intro.start, state.intro.end,
|
||||
state.credits.start, state.credits.end,
|
||||
state.recap.start, state.recap.end,
|
||||
state.preview.start, state.preview.end,
|
||||
nextSource, nextConfidence,
|
||||
intro.source, intro.provider, intro.confidence, intro.algorithm, introApplied,
|
||||
credits.source, credits.provider, credits.confidence, credits.algorithm, creditsApplied,
|
||||
recap.source, recap.provider, recap.confidence, recap.algorithm, recapApplied,
|
||||
preview.source, preview.provider, preview.confidence, preview.algorithm, previewApplied,
|
||||
state.intro.source, state.intro.provider, state.intro.confidence, state.intro.algorithm, state.intro.detectedAt,
|
||||
state.credits.source, state.credits.provider, state.credits.confidence, state.credits.algorithm, state.credits.detectedAt,
|
||||
state.recap.source, state.recap.provider, state.recap.confidence, state.recap.algorithm, state.recap.detectedAt,
|
||||
state.preview.source, state.preview.provider, state.preview.confidence, state.preview.algorithm, state.preview.detectedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("updating media markers: %w", err)
|
||||
@@ -1202,13 +1479,227 @@ func (r *FileRepository) UpsertMarkers(ctx context.Context, fileID int, update M
|
||||
if tag.RowsAffected() == 0 {
|
||||
return false, ErrFileNotFound
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, fmt.Errorf("commit marker upsert transaction: %w", err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func clearSegmentState(state *segmentState) bool {
|
||||
if segmentEqual(*state, segmentState{}) {
|
||||
return false
|
||||
}
|
||||
state.start = nil
|
||||
state.end = nil
|
||||
state.source = nil
|
||||
state.provider = nil
|
||||
state.confidence = nil
|
||||
state.algorithm = nil
|
||||
state.detectedAt = nil
|
||||
return true
|
||||
}
|
||||
|
||||
func insertMarkerEditAuditRows(
|
||||
ctx context.Context,
|
||||
tx pgx.Tx,
|
||||
fileID int,
|
||||
before markerMutationState,
|
||||
after markerMutationState,
|
||||
changed markerSegmentFlags,
|
||||
audit MarkerAuditContext,
|
||||
mutationAt time.Time,
|
||||
) error {
|
||||
type changedSegment struct {
|
||||
kind string
|
||||
before segmentState
|
||||
after segmentState
|
||||
}
|
||||
segments := make([]changedSegment, 0, 4)
|
||||
if changed.intro {
|
||||
segments = append(segments, changedSegment{"intro", before.intro, after.intro})
|
||||
}
|
||||
if changed.credits {
|
||||
segments = append(segments, changedSegment{"credits", before.credits, after.credits})
|
||||
}
|
||||
if changed.recap {
|
||||
segments = append(segments, changedSegment{"recap", before.recap, after.recap})
|
||||
}
|
||||
if changed.preview {
|
||||
segments = append(segments, changedSegment{"preview", before.preview, after.preview})
|
||||
}
|
||||
|
||||
for _, segment := range segments {
|
||||
beforeJSON, err := marshalMarkerAuditSegment(markerAuditSegmentForState(segment.before))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
afterJSON, err := marshalMarkerAuditSegment(markerAuditSegmentForState(segment.after))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
action := "set"
|
||||
if len(afterJSON) == 0 {
|
||||
action = "clear"
|
||||
}
|
||||
_, err = tx.Exec(ctx, `
|
||||
INSERT INTO marker_edit_audit (
|
||||
media_file_id,
|
||||
segment_kind,
|
||||
action,
|
||||
before_marker,
|
||||
after_marker,
|
||||
user_id,
|
||||
impersonator_user_id,
|
||||
api_key_id,
|
||||
request_id,
|
||||
client_ip,
|
||||
user_agent,
|
||||
created_at
|
||||
)
|
||||
VALUES ($1, $2, $3, $4::jsonb, $5::jsonb, $6, $7, $8, $9, $10::inet, $11, $12)`,
|
||||
fileID,
|
||||
segment.kind,
|
||||
action,
|
||||
nullableJSON(beforeJSON),
|
||||
nullableJSON(afterJSON),
|
||||
audit.UserID,
|
||||
audit.ImpersonatorUserID,
|
||||
audit.APIKeyID,
|
||||
nullableString(audit.RequestID),
|
||||
nullableString(audit.ClientIP),
|
||||
nullableString(audit.UserAgent),
|
||||
mutationAt,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("insert marker edit audit row: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func markerAuditSegmentForState(state segmentState) *MarkerAuditSegment {
|
||||
if state.start == nil || state.end == nil {
|
||||
return nil
|
||||
}
|
||||
return &MarkerAuditSegment{
|
||||
Start: cloneFloat(state.start),
|
||||
End: cloneFloat(state.end),
|
||||
Source: cloneString(state.source),
|
||||
Provider: cloneString(state.provider),
|
||||
Confidence: cloneFloat(state.confidence),
|
||||
Algorithm: cloneString(state.algorithm),
|
||||
DetectedAt: cloneTime(state.detectedAt),
|
||||
}
|
||||
}
|
||||
|
||||
func marshalMarkerAuditSegment(segment *MarkerAuditSegment) ([]byte, error) {
|
||||
if segment == nil {
|
||||
return nil, nil
|
||||
}
|
||||
data, err := json.Marshal(segment)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal marker audit segment: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func nullableJSON(data []byte) any {
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
func nullableString(value string) any {
|
||||
if value == "" {
|
||||
return nil
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
func recomputeSharedMarkerAttribution(
|
||||
legacySource *string,
|
||||
legacyConfidence *float64,
|
||||
states ...segmentState,
|
||||
) (*string, *float64) {
|
||||
var (
|
||||
bestSource *string
|
||||
bestConfidence *float64
|
||||
bestPriority int
|
||||
found bool
|
||||
)
|
||||
for _, state := range states {
|
||||
if state.start == nil || state.end == nil {
|
||||
continue
|
||||
}
|
||||
source := state.source
|
||||
confidence := state.confidence
|
||||
if source == nil {
|
||||
source = legacySource
|
||||
if confidence == nil {
|
||||
confidence = legacyConfidence
|
||||
}
|
||||
}
|
||||
if source == nil || *source == "" {
|
||||
continue
|
||||
}
|
||||
priority := models.MarkerSourcePriority(*source)
|
||||
if !found || priority > bestPriority || (priority == bestPriority && confidenceGreater(confidence, bestConfidence)) {
|
||||
sourceCopy := *source
|
||||
bestSource = &sourceCopy
|
||||
bestConfidence = cloneFloat(confidence)
|
||||
bestPriority = priority
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
}
|
||||
return bestSource, bestConfidence
|
||||
}
|
||||
|
||||
func confidenceGreater(a, b *float64) bool {
|
||||
if a == nil {
|
||||
return false
|
||||
}
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
return *a > *b
|
||||
}
|
||||
|
||||
func cloneFloat(v *float64) *float64 {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneString(v *string) *string {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func cloneTime(v *time.Time) *time.Time {
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
out := *v
|
||||
return &out
|
||||
}
|
||||
|
||||
func unmarshalMarkerAuditSegment(data []byte) (*MarkerAuditSegment, error) {
|
||||
if len(data) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var segment MarkerAuditSegment
|
||||
if err := json.Unmarshal(data, &segment); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal marker audit segment: %w", err)
|
||||
}
|
||||
return &segment, nil
|
||||
}
|
||||
|
||||
func ptrFloatEqual(a, b *float64) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
@@ -1216,31 +1707,6 @@ func ptrFloatEqual(a, b *float64) bool {
|
||||
return *a == *b
|
||||
}
|
||||
|
||||
func nextSharedMarkerAttribution(
|
||||
existingSource *string,
|
||||
existingConfidence *float64,
|
||||
update MarkerUpdate,
|
||||
markerApplied bool,
|
||||
) (*string, *float64) {
|
||||
if !markerApplied {
|
||||
return existingSource, existingConfidence
|
||||
}
|
||||
if existingSource == nil || models.MarkerSourcePriority(update.MarkersSource) > models.MarkerSourcePriority(*existingSource) {
|
||||
return &update.MarkersSource, update.MarkersConfidence
|
||||
}
|
||||
if models.MarkerSourcePriority(update.MarkersSource) == models.MarkerSourcePriority(*existingSource) && update.MarkersConfidence != nil {
|
||||
return existingSource, update.MarkersConfidence
|
||||
}
|
||||
return existingSource, existingConfidence
|
||||
}
|
||||
|
||||
func markerAlgorithm(update MarkerUpdate) string {
|
||||
if update.MarkersAlgorithm != "" {
|
||||
return update.MarkersAlgorithm
|
||||
}
|
||||
return "external:" + update.MarkersSource
|
||||
}
|
||||
|
||||
func ptrStringEqual(a, b *string) bool {
|
||||
if a == nil || b == nil {
|
||||
return a == b
|
||||
|
||||
@@ -2,73 +2,115 @@ package scanner
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestNextSharedMarkerAttributionPreservesHigherPriorityConfidence(t *testing.T) {
|
||||
existingSource := models.MarkerSourceManual
|
||||
existingConfidence := 1.0
|
||||
updateConfidence := 0.5
|
||||
func TestRecomputeSharedMarkerAttributionPreservesHigherPrioritySegment(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manualConfidence := 1.0
|
||||
scannerConfidence := 0.5
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &manual,
|
||||
confidence: &manualConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceManual {
|
||||
t.Fatalf("next source = %v, want manual", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != existingConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, existingConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != manualConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, manualConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionUpdatesSamePriorityConfidence(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.5
|
||||
updateConfidence := 0.8
|
||||
func TestRecomputeSharedMarkerAttributionUpdatesSamePriorityConfidence(t *testing.T) {
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
lowerConfidence := 0.5
|
||||
higherConfidence := 0.8
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &scannerSource,
|
||||
confidence: &lowerConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &scannerSource,
|
||||
confidence: &higherConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != updateConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, updateConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != higherConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, higherConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionPromotesHigherPrioritySource(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.5
|
||||
updateConfidence := 0.9
|
||||
func TestRecomputeSharedMarkerAttributionPromotesHigherPrioritySource(t *testing.T) {
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manual := models.MarkerSourceManual
|
||||
scannerConfidence := 0.5
|
||||
manualConfidence := 0.9
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceManual,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, true)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
nil,
|
||||
nil,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
segmentState{
|
||||
start: floatPtr(20),
|
||||
end: floatPtr(30),
|
||||
source: &manual,
|
||||
confidence: &manualConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceManual {
|
||||
t.Fatalf("next source = %v, want manual", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != updateConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, updateConfidence)
|
||||
if nextConfidence == nil || *nextConfidence != manualConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, manualConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNextSharedMarkerAttributionDoesNotDowngradeConfidenceWhenMarkerRejected(t *testing.T) {
|
||||
func TestRecomputeSharedMarkerAttributionUsesLegacyAttributionForUnattributedSegment(t *testing.T) {
|
||||
existingSource := models.MarkerSourceScanner
|
||||
existingConfidence := 0.9
|
||||
updateConfidence := 0.4
|
||||
|
||||
nextSource, nextConfidence := nextSharedMarkerAttribution(&existingSource, &existingConfidence, MarkerUpdate{
|
||||
MarkersSource: models.MarkerSourceScanner,
|
||||
MarkersConfidence: &updateConfidence,
|
||||
}, false)
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
&existingSource,
|
||||
&existingConfidence,
|
||||
segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(10),
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner", nextSource)
|
||||
@@ -78,3 +120,122 @@ func TestNextSharedMarkerAttributionDoesNotDowngradeConfidenceWhenMarkerRejected
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplySegmentPatchSkipsSemanticNoop(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
confidence := 1.0
|
||||
algorithm := "manual:v1"
|
||||
detectedAt := time.Unix(100, 0).UTC()
|
||||
state := segmentState{
|
||||
start: floatPtr(0),
|
||||
end: floatPtr(60),
|
||||
source: &manual,
|
||||
confidence: &confidence,
|
||||
algorithm: &algorithm,
|
||||
detectedAt: &detectedAt,
|
||||
}
|
||||
|
||||
changed, err := applySegmentPatch(
|
||||
&state,
|
||||
nil,
|
||||
manual,
|
||||
nil,
|
||||
&confidence,
|
||||
algorithm,
|
||||
floatPtr(0),
|
||||
floatPtr(60),
|
||||
1800,
|
||||
"intro",
|
||||
time.Unix(200, 0).UTC(),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("applySegmentPatch returned error: %v", err)
|
||||
}
|
||||
if changed {
|
||||
t.Fatal("identical marker patch should be a semantic no-op")
|
||||
}
|
||||
if state.detectedAt == nil || !state.detectedAt.Equal(detectedAt) {
|
||||
t.Fatalf("detected_at changed on no-op: %v", state.detectedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClearSegmentStateSkipsSemanticNoop(t *testing.T) {
|
||||
empty := segmentState{}
|
||||
if clearSegmentState(&empty) {
|
||||
t.Fatal("clearing an empty segment should be a no-op")
|
||||
}
|
||||
|
||||
manual := models.MarkerSourceManual
|
||||
state := segmentState{start: floatPtr(0), end: floatPtr(60), source: &manual}
|
||||
if !clearSegmentState(&state) {
|
||||
t.Fatal("clearing a populated segment should report a change")
|
||||
}
|
||||
if state.start != nil || state.end != nil || state.source != nil {
|
||||
t.Fatalf("segment was not cleared: %+v", state)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarkerAuditSegmentForStatePreservesBeforeAfterShape(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
algorithm := "manual:v1"
|
||||
confidence := 1.0
|
||||
detectedAt := time.Unix(300, 0).UTC()
|
||||
|
||||
segment := markerAuditSegmentForState(segmentState{
|
||||
start: floatPtr(5),
|
||||
end: floatPtr(65),
|
||||
source: &manual,
|
||||
confidence: &confidence,
|
||||
algorithm: &algorithm,
|
||||
detectedAt: &detectedAt,
|
||||
})
|
||||
if segment == nil {
|
||||
t.Fatal("expected audit segment")
|
||||
}
|
||||
if segment.Start == nil || *segment.Start != 5 || segment.End == nil || *segment.End != 65 {
|
||||
t.Fatalf("audit segment bounds = %v..%v, want 5..65", segment.Start, segment.End)
|
||||
}
|
||||
if segment.Source == nil || *segment.Source != manual || segment.Algorithm == nil || *segment.Algorithm != algorithm {
|
||||
t.Fatalf("audit provenance = source %v algorithm %v", segment.Source, segment.Algorithm)
|
||||
}
|
||||
if segment.DetectedAt == nil || !segment.DetectedAt.Equal(detectedAt) {
|
||||
t.Fatalf("detected_at = %v, want %v", segment.DetectedAt, detectedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeSharedMarkerAttributionDropsClearedManualSegment(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
scannerSource := models.MarkerSourceScanner
|
||||
manualConfidence := 1.0
|
||||
scannerConfidence := 0.8
|
||||
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(
|
||||
&manual,
|
||||
&manualConfidence,
|
||||
segmentState{},
|
||||
segmentState{
|
||||
start: floatPtr(120),
|
||||
end: floatPtr(180),
|
||||
source: &scannerSource,
|
||||
confidence: &scannerConfidence,
|
||||
},
|
||||
)
|
||||
|
||||
if nextSource == nil || *nextSource != models.MarkerSourceScanner {
|
||||
t.Fatalf("next source = %v, want scanner after manual segment clear", nextSource)
|
||||
}
|
||||
if nextConfidence == nil || *nextConfidence != scannerConfidence {
|
||||
t.Fatalf("next confidence = %v, want %v", nextConfidence, scannerConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecomputeSharedMarkerAttributionClearsWhenNoSegmentsRemain(t *testing.T) {
|
||||
manual := models.MarkerSourceManual
|
||||
manualConfidence := 1.0
|
||||
|
||||
nextSource, nextConfidence := recomputeSharedMarkerAttribution(&manual, &manualConfidence, segmentState{})
|
||||
if nextSource != nil || nextConfidence != nil {
|
||||
t.Fatalf("shared attribution = %v/%v, want nil/nil with no remaining segments", nextSource, nextConfidence)
|
||||
}
|
||||
}
|
||||
|
||||
func floatPtr(v float64) *float64 { return &v }
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
type MarkerAuditContext struct {
|
||||
UserID *int
|
||||
ImpersonatorUserID *int
|
||||
APIKeyID *int64
|
||||
RequestID string
|
||||
ClientIP string
|
||||
UserAgent string
|
||||
}
|
||||
|
||||
type markerAuditContextKey struct{}
|
||||
|
||||
func WithMarkerAuditContext(ctx context.Context, audit MarkerAuditContext) context.Context {
|
||||
return context.WithValue(ctx, markerAuditContextKey{}, audit)
|
||||
}
|
||||
|
||||
func MarkerAuditContextFromContext(ctx context.Context) (MarkerAuditContext, bool) {
|
||||
audit, ok := ctx.Value(markerAuditContextKey{}).(MarkerAuditContext)
|
||||
return audit, ok
|
||||
}
|
||||
|
||||
type MarkerAuditSegment struct {
|
||||
Start *float64 `json:"start"`
|
||||
End *float64 `json:"end"`
|
||||
Source *string `json:"source"`
|
||||
Provider *string `json:"provider"`
|
||||
Confidence *float64 `json:"confidence"`
|
||||
Algorithm *string `json:"algorithm"`
|
||||
DetectedAt *time.Time `json:"detected_at"`
|
||||
}
|
||||
|
||||
type MarkerEditAuditRow struct {
|
||||
ID int64
|
||||
MediaFileID int
|
||||
ItemID *string
|
||||
ItemType *string
|
||||
MediaTitle *string
|
||||
FilePath *string
|
||||
SegmentKind string
|
||||
Action string
|
||||
Before *MarkerAuditSegment
|
||||
After *MarkerAuditSegment
|
||||
UserID *int
|
||||
Username *string
|
||||
ImpersonatorUserID *int
|
||||
ImpersonatorUsername *string
|
||||
APIKeyID *int64
|
||||
RequestID *string
|
||||
ClientIP *string
|
||||
UserAgent *string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/titleutil"
|
||||
@@ -26,10 +25,18 @@ func (s *Scanner) ScanPodcastFolder(ctx context.Context, folder *models.MediaFol
|
||||
return fmt.Errorf("ScanPodcastFolder: nil scanner or folder")
|
||||
}
|
||||
|
||||
var attempted int
|
||||
var succeeded int
|
||||
var failures []error
|
||||
for _, root := range folder.Paths {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
slog.Warn("podcast scan: read root failed", "root", root, "error", err)
|
||||
attempted++
|
||||
failures = append(failures, fmt.Errorf("read root %s: %w", root, err))
|
||||
continue
|
||||
}
|
||||
for _, entry := range entries {
|
||||
@@ -37,16 +44,29 @@ func (s *Scanner) ScanPodcastFolder(ctx context.Context, folder *models.MediaFol
|
||||
continue
|
||||
}
|
||||
subPath := filepath.Join(root, entry.Name())
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
attempted++
|
||||
if err := s.reconcilePodcastShow(ctx, folder, subPath); err != nil {
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return err
|
||||
}
|
||||
slog.Warn("podcast scan: show failed",
|
||||
"folder_id", folder.ID,
|
||||
"path", subPath,
|
||||
"error", err,
|
||||
)
|
||||
failures = append(failures, fmt.Errorf("%s: %w", subPath, err))
|
||||
// Continue with siblings — one bad show should not stop the scan.
|
||||
continue
|
||||
}
|
||||
succeeded++
|
||||
}
|
||||
}
|
||||
if attempted > 0 && succeeded == 0 && len(failures) > 0 {
|
||||
return fmt.Errorf("podcast scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -59,7 +79,7 @@ func (s *Scanner) reconcilePodcastShow(ctx context.Context, folder *models.Media
|
||||
return fmt.Errorf("parse podcast show %s: %w", folderPath, err)
|
||||
}
|
||||
|
||||
showContentID, err := s.upsertPodcastMediaItem(ctx, parsed)
|
||||
showContentID, err := s.upsertPodcastMediaItem(ctx, folder.ID, folderPath, parsed)
|
||||
if err != nil {
|
||||
return fmt.Errorf("upsert podcast item: %w", err)
|
||||
}
|
||||
@@ -84,32 +104,42 @@ func (s *Scanner) reconcilePodcastShow(ctx context.Context, folder *models.Media
|
||||
return nil
|
||||
}
|
||||
|
||||
// upsertPodcastMediaItem looks up an existing media_items row by
|
||||
// title+year+type='podcast', updates it if found, or creates a new row.
|
||||
// upsertPodcastMediaItem reuses an item already linked to the same filesystem
|
||||
// root, or creates a new row. It intentionally avoids title/year-only dedupe
|
||||
// because filesystem podcasts can have duplicate titles and missing years.
|
||||
// Returns the content_id used.
|
||||
func (s *Scanner) upsertPodcastMediaItem(ctx context.Context, show *parsedPodcastShow) (string, error) {
|
||||
func (s *Scanner) upsertPodcastMediaItem(ctx context.Context, folderID int, folderPath string, show *parsedPodcastShow) (string, error) {
|
||||
if s.itemRepo == nil {
|
||||
return "", fmt.Errorf("itemRepo not configured on Scanner")
|
||||
}
|
||||
|
||||
existing, err := s.itemRepo.GetByTitleYearType(ctx, show.Title, show.Year, "podcast")
|
||||
if err == nil && existing != nil {
|
||||
existing.Title = show.Title
|
||||
existing.Year = show.Year
|
||||
existing.Type = "podcast"
|
||||
if existing.SortTitle == "" {
|
||||
existing.SortTitle = titleutil.DeriveDefaultSortTitle(show.Title)
|
||||
}
|
||||
if err := s.itemRepo.Upsert(ctx, existing); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return existing.ContentID, nil
|
||||
if s.fileRepo == nil {
|
||||
return "", fmt.Errorf("fileRepo not configured on Scanner")
|
||||
}
|
||||
if err != nil && !errors.Is(err, catalog.ErrItemNotFound) {
|
||||
return "", fmt.Errorf("GetByTitleYearType: %w", err)
|
||||
return resolvePodcastMediaItem(ctx, s.fileRepo, s.itemRepo, folderID, folderPath, show)
|
||||
}
|
||||
|
||||
func resolvePodcastMediaItem(
|
||||
ctx context.Context,
|
||||
rootFinder filesystemRootContentFinder,
|
||||
itemWriter filesystemMediaItemWriter,
|
||||
folderID int,
|
||||
folderPath string,
|
||||
show *parsedPodcastShow,
|
||||
) (string, error) {
|
||||
if rootFinder == nil {
|
||||
return "", fmt.Errorf("root content finder not configured")
|
||||
}
|
||||
if itemWriter == nil {
|
||||
return "", fmt.Errorf("media item writer not configured")
|
||||
}
|
||||
existingID, err := rootFinder.FindContentIDByRootPath(ctx, folderID, folderPath, "podcast")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("find podcast by root path: %w", err)
|
||||
}
|
||||
if existingID != "" {
|
||||
return existingID, nil
|
||||
}
|
||||
|
||||
// Not found — create.
|
||||
id, err := idgen.NextID()
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("generate content_id: %w", err)
|
||||
@@ -121,7 +151,7 @@ func (s *Scanner) upsertPodcastMediaItem(ctx context.Context, show *parsedPodcas
|
||||
SortTitle: titleutil.DeriveDefaultSortTitle(show.Title),
|
||||
Year: show.Year,
|
||||
}
|
||||
if err := s.itemRepo.Upsert(ctx, item); err != nil {
|
||||
if err := itemWriter.Upsert(ctx, item); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return id, nil
|
||||
@@ -176,7 +206,7 @@ func (s *Scanner) upsertPodcastEpisodesAndFiles(
|
||||
BaseTitle: show.Title,
|
||||
BaseYear: show.Year,
|
||||
BaseType: "podcast",
|
||||
IdentityConfidence: "high",
|
||||
IdentityConfidence: podcastIdentityConfidence(show, ep),
|
||||
FilePath: ep.Path,
|
||||
ProbeSource: "local",
|
||||
}
|
||||
@@ -186,3 +216,35 @@ func (s *Scanner) upsertPodcastEpisodesAndFiles(
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func podcastIdentityConfidence(show *parsedPodcastShow, episode parsedPodcastEpisode) string {
|
||||
if show == nil {
|
||||
return "low"
|
||||
}
|
||||
|
||||
score := 0
|
||||
if show.Title != "" {
|
||||
score++
|
||||
}
|
||||
if show.Author != "" {
|
||||
score++
|
||||
}
|
||||
if show.Year > 0 {
|
||||
score++
|
||||
}
|
||||
if episode.Title != "" {
|
||||
score++
|
||||
}
|
||||
if episode.Track > 0 {
|
||||
score++
|
||||
}
|
||||
|
||||
switch {
|
||||
case score >= 5:
|
||||
return "high"
|
||||
case score > 0:
|
||||
return "medium"
|
||||
default:
|
||||
return "low"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,14 @@ package scanner
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
func TestParsePodcastShow(t *testing.T) {
|
||||
@@ -39,3 +45,125 @@ func TestParsePodcastShow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPodcastIdentityConfidenceReflectsMetadataCompleteness(t *testing.T) {
|
||||
show := &parsedPodcastShow{Title: "Tagged Show", Author: "Host", Year: 2024}
|
||||
episode := parsedPodcastEpisode{Title: "Episode", Track: 3}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "high" {
|
||||
t.Fatalf("complete metadata confidence = %q, want high", got)
|
||||
}
|
||||
|
||||
show = &parsedPodcastShow{Title: "Tagged Show"}
|
||||
episode = parsedPodcastEpisode{Title: "Episode"}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "medium" {
|
||||
t.Fatalf("partial metadata confidence = %q, want medium", got)
|
||||
}
|
||||
|
||||
show = &parsedPodcastShow{}
|
||||
episode = parsedPodcastEpisode{}
|
||||
if got := podcastIdentityConfidence(show, episode); got != "low" {
|
||||
t.Fatalf("empty metadata confidence = %q, want low", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanPodcastFolderReturnsErrorWhenEveryReconcileFails(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
showDir := filepath.Join(root, "bad-show")
|
||||
if err := os.Mkdir(showDir, 0o755); err != nil {
|
||||
t.Fatalf("mkdir show dir: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(showDir, "episode.mp3"), []byte("not real audio"), 0o644); err != nil {
|
||||
t.Fatalf("write fake audio: %v", err)
|
||||
}
|
||||
|
||||
s := &Scanner{ffprobePath: "definitely-missing-ffprobe"}
|
||||
err := s.ScanPodcastFolder(context.Background(), &models.MediaFolder{ID: 43, Paths: []string{root}})
|
||||
if err == nil {
|
||||
t.Fatal("ScanPodcastFolder returned nil, want aggregate failure")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "folder_id=43") {
|
||||
t.Fatalf("error = %q, want folder id", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanPodcastFolderReturnsCanceledContext(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
s := &Scanner{}
|
||||
err := s.ScanPodcastFolder(ctx, &models.MediaFolder{ID: 43, Paths: []string{root}})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("ScanPodcastFolder error = %v, want context.Canceled", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemReusesRootScopedContentID(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{contentID: "podcast-root-id"}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Same Show",
|
||||
&parsedPodcastShow{Title: "Same Show", Year: 0, Author: "Host A"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePodcastMediaItem: %v", err)
|
||||
}
|
||||
if got != "podcast-root-id" {
|
||||
t.Fatalf("contentID = %q, want root-scoped id", got)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("unexpected item upsert for existing root: %d", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemCreatesNewWhenRootHasNoClaim(t *testing.T) {
|
||||
finder := &fakeRootContentFinder{}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
got, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Another Same Show",
|
||||
&parsedPodcastShow{Title: "Same Show", Year: 0, Author: "Host B"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("resolvePodcastMediaItem: %v", err)
|
||||
}
|
||||
if got == "" {
|
||||
t.Fatal("contentID is empty")
|
||||
}
|
||||
if len(writer.upserts) != 1 {
|
||||
t.Fatalf("upserts = %d, want 1", len(writer.upserts))
|
||||
}
|
||||
if writer.upserts[0].ContentID != got || writer.upserts[0].Type != "podcast" {
|
||||
t.Fatalf("upserted item = %+v, contentID %q", writer.upserts[0], got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePodcastMediaItemPropagatesRootLookupError(t *testing.T) {
|
||||
wantErr := errors.New("root lookup failed")
|
||||
finder := &fakeRootContentFinder{err: wantErr}
|
||||
writer := &fakeFilesystemItemWriter{}
|
||||
|
||||
_, err := resolvePodcastMediaItem(
|
||||
context.Background(),
|
||||
finder,
|
||||
writer,
|
||||
8,
|
||||
"/library/Show",
|
||||
&parsedPodcastShow{Title: "Show"},
|
||||
)
|
||||
if !errors.Is(err, wantErr) {
|
||||
t.Fatalf("error = %v, want %v", err, wantErr)
|
||||
}
|
||||
if len(writer.upserts) != 0 {
|
||||
t.Fatalf("upserts = %d, want 0", len(writer.upserts))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -125,6 +125,26 @@ type MarkerUpdate struct {
|
||||
MarkersProvider *string
|
||||
MarkersConfidence *float64
|
||||
MarkersAlgorithm string
|
||||
|
||||
// Optional per-segment provenance overrides. When set for a segment,
|
||||
// UpsertMarkers writes these source/provider/confidence/algorithm values
|
||||
// for that segment instead of the shared Markers* fields above. Used by
|
||||
// merged multi-provider results where each segment may come from a
|
||||
// different provider/source class; left nil for single-source writes
|
||||
// (scanner/s3/local).
|
||||
IntroProvenance *SegmentProvenance
|
||||
CreditsProvenance *SegmentProvenance
|
||||
RecapProvenance *SegmentProvenance
|
||||
PreviewProvenance *SegmentProvenance
|
||||
}
|
||||
|
||||
// SegmentProvenance is a per-segment attribution override for MarkerUpdate.
|
||||
// When Source is empty, the shared MarkerUpdate.MarkersSource applies.
|
||||
type SegmentProvenance struct {
|
||||
Source string
|
||||
Provider *string
|
||||
Confidence *float64
|
||||
Algorithm string
|
||||
}
|
||||
|
||||
// HasAnySegment reports whether the update would write at least one segment.
|
||||
|
||||
@@ -6,17 +6,14 @@ package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
)
|
||||
|
||||
// SubtitleCue is one timed subtitle entry. Times are absolute media positions.
|
||||
// Translation only rewrites Lines; Start/End are preserved verbatim so timing
|
||||
// can never drift.
|
||||
type SubtitleCue struct {
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Lines []string
|
||||
}
|
||||
type SubtitleCue = subtitles.SubtitleCue
|
||||
|
||||
// Translator converts subtitle cues from one language to another, preserving
|
||||
// cue count, order, and timing. The built-in implementation is LLMTranslator
|
||||
|
||||
@@ -1,161 +1,13 @@
|
||||
package ai
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
import "github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
|
||||
// ParseCues parses SRT or WebVTT subtitle bytes into cues. It tolerates both
|
||||
// comma (SRT) and period (VTT) millisecond separators, a leading WEBVTT header,
|
||||
// optional cue identifiers, VTT cue settings after the end timestamp, a UTF-8
|
||||
// BOM, and CRLF line endings. Malformed individual cues are skipped rather than
|
||||
// failing the whole document.
|
||||
// ParseCues keeps the AI package API stable while using the shared parser.
|
||||
func ParseCues(data []byte) ([]SubtitleCue, error) {
|
||||
text := string(data)
|
||||
text = strings.TrimPrefix(text, "\ufeff") // strip UTF-8 BOM
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
|
||||
var cues []SubtitleCue
|
||||
for _, block := range strings.Split(text, "\n\n") {
|
||||
block = strings.Trim(block, "\n")
|
||||
if strings.TrimSpace(block) == "" {
|
||||
continue
|
||||
}
|
||||
lines := strings.Split(block, "\n")
|
||||
|
||||
timingIdx := -1
|
||||
for i, l := range lines {
|
||||
if strings.Contains(l, "-->") {
|
||||
timingIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if timingIdx < 0 {
|
||||
continue // WEBVTT header, NOTE/STYLE blocks, etc.
|
||||
}
|
||||
|
||||
start, end, err := parseTimingLine(lines[timingIdx])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
textLines := append([]string(nil), lines[timingIdx+1:]...)
|
||||
for len(textLines) > 0 && strings.TrimSpace(textLines[len(textLines)-1]) == "" {
|
||||
textLines = textLines[:len(textLines)-1]
|
||||
}
|
||||
if len(textLines) == 0 {
|
||||
continue
|
||||
}
|
||||
cues = append(cues, SubtitleCue{Start: start, End: end, Lines: textLines})
|
||||
}
|
||||
|
||||
if len(cues) == 0 {
|
||||
return nil, fmt.Errorf("no subtitle cues found")
|
||||
}
|
||||
return cues, nil
|
||||
return subtitles.ParseCues(data)
|
||||
}
|
||||
|
||||
// SerializeSRT renders cues as a well-formed SRT document. Output always uses
|
||||
// comma millisecond separators; the serving layer converts SRT to VTT on read.
|
||||
// SerializeSRT keeps the AI package API stable while using the shared writer.
|
||||
func SerializeSRT(cues []SubtitleCue) []byte {
|
||||
var buf bytes.Buffer
|
||||
for i, cue := range cues {
|
||||
fmt.Fprintf(&buf, "%d\n%s --> %s\n", i+1, formatSRTTimestamp(cue.Start), formatSRTTimestamp(cue.End))
|
||||
for _, line := range cue.Lines {
|
||||
buf.WriteString(line)
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func parseTimingLine(line string) (time.Duration, time.Duration, error) {
|
||||
parts := strings.SplitN(line, "-->", 2)
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("invalid timing line")
|
||||
}
|
||||
start, err := parseTimestamp(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
// The end timestamp may be followed by VTT cue settings (e.g. "line:0%").
|
||||
endField := strings.TrimSpace(parts[1])
|
||||
if sp := strings.IndexAny(endField, " \t"); sp >= 0 {
|
||||
endField = endField[:sp]
|
||||
}
|
||||
end, err := parseTimestamp(endField)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return 0, 0, fmt.Errorf("invalid cue range")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
// parseTimestamp accepts HH:MM:SS,mmm / HH:MM:SS.mmm / MM:SS.mmm forms.
|
||||
func parseTimestamp(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(strings.Replace(s, ",", ".", 1))
|
||||
colon := strings.Split(s, ":")
|
||||
if len(colon) < 2 || len(colon) > 3 {
|
||||
return 0, fmt.Errorf("invalid timestamp %q", s)
|
||||
}
|
||||
|
||||
var hours int
|
||||
idx := 0
|
||||
if len(colon) == 3 {
|
||||
h, err := strconv.Atoi(colon[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid hours in %q", s)
|
||||
}
|
||||
hours = h
|
||||
idx = 1
|
||||
}
|
||||
|
||||
minutes, err := strconv.Atoi(colon[idx])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid minutes in %q", s)
|
||||
}
|
||||
|
||||
secFrac := strings.SplitN(colon[idx+1], ".", 2)
|
||||
seconds, err := strconv.Atoi(secFrac[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid seconds in %q", s)
|
||||
}
|
||||
|
||||
millis := 0
|
||||
if len(secFrac) == 2 {
|
||||
frac := secFrac[1]
|
||||
for len(frac) < 3 {
|
||||
frac += "0"
|
||||
}
|
||||
millis, err = strconv.Atoi(frac[:3])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid milliseconds in %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
return time.Duration(hours)*time.Hour +
|
||||
time.Duration(minutes)*time.Minute +
|
||||
time.Duration(seconds)*time.Second +
|
||||
time.Duration(millis)*time.Millisecond, nil
|
||||
}
|
||||
|
||||
func formatSRTTimestamp(d time.Duration) string {
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
totalMS := int64(d / time.Millisecond)
|
||||
ms := totalMS % 1000
|
||||
totalSec := totalMS / 1000
|
||||
s := totalSec % 60
|
||||
totalMin := totalSec / 60
|
||||
m := totalMin % 60
|
||||
h := totalMin / 60
|
||||
return fmt.Sprintf("%02d:%02d:%02d,%03d", h, m, s, ms)
|
||||
return subtitles.SerializeSRT(cues)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
package subtitles
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SubtitleCue is one timed subtitle entry. Times are absolute media positions.
|
||||
type SubtitleCue struct {
|
||||
Start time.Duration
|
||||
End time.Duration
|
||||
Lines []string
|
||||
}
|
||||
|
||||
// ParseCues parses SRT or WebVTT subtitle bytes into cues. It tolerates both
|
||||
// comma (SRT) and period (VTT) millisecond separators, a leading WEBVTT header,
|
||||
// optional cue identifiers, VTT cue settings after the end timestamp, a UTF-8
|
||||
// BOM, and CRLF line endings. Malformed individual cues are skipped rather than
|
||||
// failing the whole document.
|
||||
func ParseCues(data []byte) ([]SubtitleCue, error) {
|
||||
text := string(data)
|
||||
text = strings.TrimPrefix(text, "\ufeff")
|
||||
text = strings.ReplaceAll(text, "\r\n", "\n")
|
||||
text = strings.ReplaceAll(text, "\r", "\n")
|
||||
|
||||
var cues []SubtitleCue
|
||||
for _, block := range strings.Split(text, "\n\n") {
|
||||
block = strings.Trim(block, "\n")
|
||||
if strings.TrimSpace(block) == "" {
|
||||
continue
|
||||
}
|
||||
lines := strings.Split(block, "\n")
|
||||
|
||||
timingIdx := -1
|
||||
for i, line := range lines {
|
||||
if strings.Contains(line, "-->") {
|
||||
timingIdx = i
|
||||
break
|
||||
}
|
||||
}
|
||||
if timingIdx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
start, end, err := parseTimingLine(lines[timingIdx])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
textLines := append([]string(nil), lines[timingIdx+1:]...)
|
||||
for len(textLines) > 0 && strings.TrimSpace(textLines[len(textLines)-1]) == "" {
|
||||
textLines = textLines[:len(textLines)-1]
|
||||
}
|
||||
if len(textLines) == 0 {
|
||||
continue
|
||||
}
|
||||
cues = append(cues, SubtitleCue{Start: start, End: end, Lines: textLines})
|
||||
}
|
||||
|
||||
if len(cues) == 0 {
|
||||
return nil, fmt.Errorf("no subtitle cues found")
|
||||
}
|
||||
return cues, nil
|
||||
}
|
||||
|
||||
// SerializeSRT renders cues as a well-formed SRT document. Output always uses
|
||||
// comma millisecond separators; the serving layer converts SRT to VTT on read.
|
||||
func SerializeSRT(cues []SubtitleCue) []byte {
|
||||
var buf bytes.Buffer
|
||||
for i, cue := range cues {
|
||||
fmt.Fprintf(&buf, "%d\n%s --> %s\n", i+1, formatSRTTimestamp(cue.Start), formatSRTTimestamp(cue.End))
|
||||
for _, line := range cue.Lines {
|
||||
buf.WriteString(line)
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
buf.WriteByte('\n')
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func parseTimingLine(line string) (time.Duration, time.Duration, error) {
|
||||
parts := strings.SplitN(line, "-->", 2)
|
||||
if len(parts) != 2 {
|
||||
return 0, 0, fmt.Errorf("invalid timing line")
|
||||
}
|
||||
start, err := parseTimestamp(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
endField := strings.TrimSpace(parts[1])
|
||||
if sp := strings.IndexAny(endField, " \t"); sp >= 0 {
|
||||
endField = endField[:sp]
|
||||
}
|
||||
end, err := parseTimestamp(endField)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
if start < 0 || end < 0 || end < start {
|
||||
return 0, 0, fmt.Errorf("invalid cue range")
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
// parseTimestamp accepts HH:MM:SS,mmm / HH:MM:SS.mmm / MM:SS.mmm forms.
|
||||
func parseTimestamp(s string) (time.Duration, error) {
|
||||
s = strings.TrimSpace(strings.Replace(s, ",", ".", 1))
|
||||
colon := strings.Split(s, ":")
|
||||
if len(colon) < 2 || len(colon) > 3 {
|
||||
return 0, fmt.Errorf("invalid timestamp %q", s)
|
||||
}
|
||||
|
||||
var hours int
|
||||
idx := 0
|
||||
if len(colon) == 3 {
|
||||
h, err := strconv.Atoi(colon[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid hours in %q", s)
|
||||
}
|
||||
hours = h
|
||||
idx = 1
|
||||
}
|
||||
|
||||
minutes, err := strconv.Atoi(colon[idx])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid minutes in %q", s)
|
||||
}
|
||||
|
||||
secFrac := strings.SplitN(colon[idx+1], ".", 2)
|
||||
seconds, err := strconv.Atoi(secFrac[0])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid seconds in %q", s)
|
||||
}
|
||||
|
||||
millis := 0
|
||||
if len(secFrac) == 2 {
|
||||
frac := secFrac[1]
|
||||
for len(frac) < 3 {
|
||||
frac += "0"
|
||||
}
|
||||
millis, err = strconv.Atoi(frac[:3])
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("invalid milliseconds in %q", s)
|
||||
}
|
||||
}
|
||||
|
||||
return time.Duration(hours)*time.Hour +
|
||||
time.Duration(minutes)*time.Minute +
|
||||
time.Duration(seconds)*time.Second +
|
||||
time.Duration(millis)*time.Millisecond, nil
|
||||
}
|
||||
|
||||
func formatSRTTimestamp(d time.Duration) string {
|
||||
if d < 0 {
|
||||
d = 0
|
||||
}
|
||||
totalMS := int64(d / time.Millisecond)
|
||||
ms := totalMS % 1000
|
||||
totalSec := totalMS / 1000
|
||||
seconds := totalSec % 60
|
||||
totalMin := totalSec / 60
|
||||
minutes := totalMin % 60
|
||||
hours := totalMin / 60
|
||||
return fmt.Sprintf("%02d:%02d:%02d,%03d", hours, minutes, seconds, ms)
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/taskmanager"
|
||||
)
|
||||
|
||||
// ContributionRunner submits a file's eligible markers (satisfied by
|
||||
// *markers.ContributionService).
|
||||
type ContributionRunner interface {
|
||||
ContributeFile(ctx context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error)
|
||||
}
|
||||
|
||||
// AutoContributeConfigReader exposes per-provider contribution config (satisfied
|
||||
// by *markers.ProviderConfigStore).
|
||||
type AutoContributeConfigReader interface {
|
||||
List() []markers.ProviderConfig
|
||||
}
|
||||
|
||||
// ContributionCandidateSource lists local-intro files eligible for auto
|
||||
// contribution (satisfied by *markers.ContributionStore).
|
||||
type ContributionCandidateSource interface {
|
||||
CandidateLocalIntroFiles(ctx context.Context, minConfidence float64, afterID, limit int) ([]int, error)
|
||||
}
|
||||
|
||||
// ContributionFileLoader loads files by id (satisfied by *scanner.FileRepository).
|
||||
type ContributionFileLoader interface {
|
||||
GetByIDs(ctx context.Context, ids []int) ([]*models.MediaFile, error)
|
||||
}
|
||||
|
||||
// ContributeMarkersTask submits high-confidence local intro markers to providers
|
||||
// that have auto-contribution enabled. It is a no-op when no provider opts in.
|
||||
type ContributeMarkersTask struct {
|
||||
service ContributionRunner
|
||||
config AutoContributeConfigReader
|
||||
candidates ContributionCandidateSource
|
||||
files ContributionFileLoader
|
||||
}
|
||||
|
||||
// NewContributeMarkersTask constructs the task.
|
||||
func NewContributeMarkersTask(service ContributionRunner, config AutoContributeConfigReader, candidates ContributionCandidateSource, files ContributionFileLoader) *ContributeMarkersTask {
|
||||
return &ContributeMarkersTask{service: service, config: config, candidates: candidates, files: files}
|
||||
}
|
||||
|
||||
func (t *ContributeMarkersTask) Key() string { return "contribute_markers" }
|
||||
func (t *ContributeMarkersTask) Name() string { return "Contribute Markers" }
|
||||
func (t *ContributeMarkersTask) Description() string {
|
||||
return "Submits high-confidence local intro markers to enabled contribution providers"
|
||||
}
|
||||
func (t *ContributeMarkersTask) Category() taskmanager.TaskCategory {
|
||||
return taskmanager.TaskCategoryLibrary
|
||||
}
|
||||
func (t *ContributeMarkersTask) IsHidden() bool { return false }
|
||||
|
||||
func (t *ContributeMarkersTask) DefaultTriggers() []taskmanager.TriggerConfig {
|
||||
// Runs after the 03:30 local-detection task so freshly detected markers are
|
||||
// eligible the same night.
|
||||
return []taskmanager.TriggerConfig{{Type: taskmanager.TriggerTypeDaily, TimeOfDay: "04:00"}}
|
||||
}
|
||||
|
||||
func (t *ContributeMarkersTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error {
|
||||
if t.service == nil || t.config == nil || t.candidates == nil || t.files == nil {
|
||||
progress.Report(100, "Contribution is not configured")
|
||||
return nil
|
||||
}
|
||||
|
||||
minConfidence := 0.0
|
||||
autoEnabled := false
|
||||
for _, c := range t.config.List() {
|
||||
if !c.ContributeEnabled || !c.ContributeAutoLocal {
|
||||
continue
|
||||
}
|
||||
if !autoEnabled || c.ContributeMinConfidence < minConfidence {
|
||||
minConfidence = c.ContributeMinConfidence
|
||||
}
|
||||
autoEnabled = true
|
||||
}
|
||||
if !autoEnabled {
|
||||
progress.Report(100, "No provider has auto-contribution enabled")
|
||||
return nil
|
||||
}
|
||||
|
||||
const batch = 100
|
||||
afterID := 0
|
||||
submitted, skipped, failed := 0, 0, 0
|
||||
|
||||
for {
|
||||
ids, err := t.candidates.CandidateLocalIntroFiles(ctx, minConfidence, afterID, batch)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load contribution candidates: %w", err)
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
break
|
||||
}
|
||||
files, err := t.files.GetByIDs(ctx, ids)
|
||||
if err != nil {
|
||||
return fmt.Errorf("load candidate files: %w", err)
|
||||
}
|
||||
byID := make(map[int]*models.MediaFile, len(files))
|
||||
for _, f := range files {
|
||||
byID[f.ID] = f
|
||||
}
|
||||
for _, id := range ids {
|
||||
afterID = id
|
||||
file := byID[id]
|
||||
if file == nil {
|
||||
continue
|
||||
}
|
||||
outcomes, err := t.service.ContributeFile(ctx, file, markers.ContributeOptions{Auto: true})
|
||||
if err != nil {
|
||||
failed++
|
||||
continue
|
||||
}
|
||||
for _, o := range outcomes {
|
||||
switch o.Status {
|
||||
case markers.OutcomeStatusSkipped:
|
||||
skipped++
|
||||
case markers.OutcomeStatusRateLimited:
|
||||
failed++
|
||||
writeContributionTaskResult(progress, submitted, skipped, failed, o.RetryAfter)
|
||||
progress.Report(100, fmt.Sprintf("Contribution usage-limited; retry after %s", formatRetryAfter(o.RetryAfter)))
|
||||
return nil
|
||||
case markers.OutcomeStatusError:
|
||||
failed++
|
||||
default:
|
||||
submitted++
|
||||
}
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
writeContributionTaskResult(progress, submitted, skipped, failed, 0)
|
||||
progress.Report(100, fmt.Sprintf("Contributed %d, skipped %d, failed %d", submitted, skipped, failed))
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeContributionTaskResult(progress taskmanager.ProgressReporter, submitted, skipped, failed int, retryAfter time.Duration) {
|
||||
result := map[string]int{"submitted": submitted, "skipped": skipped, "failed": failed}
|
||||
if retryAfter > 0 {
|
||||
result["retry_after_seconds"] = int(retryAfter.Seconds())
|
||||
}
|
||||
if data, err := json.Marshal(result); err == nil {
|
||||
progress.SetResultData(data)
|
||||
}
|
||||
}
|
||||
|
||||
func formatRetryAfter(d time.Duration) string {
|
||||
if d <= 0 {
|
||||
return "later"
|
||||
}
|
||||
return d.String()
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package tasks
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/markers"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
type contribTestProgress struct{ data json.RawMessage }
|
||||
|
||||
func (p *contribTestProgress) Report(float64, string) {}
|
||||
func (p *contribTestProgress) SetResultData(d json.RawMessage) { p.data = d }
|
||||
|
||||
type fakeContribRunner struct {
|
||||
calls []int
|
||||
autoSeen bool
|
||||
outcomes []markers.ContributionOutcome
|
||||
}
|
||||
|
||||
func (f *fakeContribRunner) ContributeFile(_ context.Context, file *models.MediaFile, opts markers.ContributeOptions) ([]markers.ContributionOutcome, error) {
|
||||
f.calls = append(f.calls, file.ID)
|
||||
f.autoSeen = opts.Auto
|
||||
return f.outcomes, nil
|
||||
}
|
||||
|
||||
type fakeAutoConfig []markers.ProviderConfig
|
||||
|
||||
func (f fakeAutoConfig) List() []markers.ProviderConfig { return f }
|
||||
|
||||
type fakeCandidates struct {
|
||||
ids []int
|
||||
gotMin float64
|
||||
delivered bool
|
||||
}
|
||||
|
||||
func (f *fakeCandidates) CandidateLocalIntroFiles(_ context.Context, minConfidence float64, _, _ int) ([]int, error) {
|
||||
f.gotMin = minConfidence
|
||||
if f.delivered {
|
||||
return nil, nil
|
||||
}
|
||||
f.delivered = true
|
||||
return f.ids, nil
|
||||
}
|
||||
|
||||
type fakeFileLoader struct{}
|
||||
|
||||
func (fakeFileLoader) GetByIDs(_ context.Context, ids []int) ([]*models.MediaFile, error) {
|
||||
out := make([]*models.MediaFile, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
out = append(out, &models.MediaFile{ID: id})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func TestContributeMarkersTaskNoAutoProvider(t *testing.T) {
|
||||
runner := &fakeContribRunner{}
|
||||
cfg := fakeAutoConfig{{Provider: "introdb", ContributeEnabled: true, ContributeAutoLocal: false}}
|
||||
task := NewContributeMarkersTask(runner, cfg, &fakeCandidates{ids: []int{1}}, fakeFileLoader{})
|
||||
|
||||
if err := task.Execute(context.Background(), &contribTestProgress{}); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if len(runner.calls) != 0 {
|
||||
t.Errorf("expected no contributions when auto disabled, got %d", len(runner.calls))
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeMarkersTaskSubmitsCandidates(t *testing.T) {
|
||||
runner := &fakeContribRunner{outcomes: []markers.ContributionOutcome{{Status: markers.SubmissionStatusPending}}}
|
||||
cands := &fakeCandidates{ids: []int{10, 11}}
|
||||
cfg := fakeAutoConfig{{Provider: "introdb", ContributeEnabled: true, ContributeAutoLocal: true, ContributeMinConfidence: 0.95}}
|
||||
task := NewContributeMarkersTask(runner, cfg, cands, fakeFileLoader{})
|
||||
|
||||
prog := &contribTestProgress{}
|
||||
if err := task.Execute(context.Background(), prog); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if len(runner.calls) != 2 {
|
||||
t.Fatalf("expected 2 contributions, got %d", len(runner.calls))
|
||||
}
|
||||
if !runner.autoSeen {
|
||||
t.Error("ContributeFile should be called with Auto=true")
|
||||
}
|
||||
if cands.gotMin != 0.95 {
|
||||
t.Errorf("min confidence passed = %v, want 0.95", cands.gotMin)
|
||||
}
|
||||
if prog.data == nil {
|
||||
t.Error("expected result summary data")
|
||||
}
|
||||
}
|
||||
|
||||
func TestContributeMarkersTaskStopsOnRateLimit(t *testing.T) {
|
||||
runner := &fakeContribRunner{outcomes: []markers.ContributionOutcome{{
|
||||
Status: markers.OutcomeStatusRateLimited, RetryAfter: 90 * time.Second,
|
||||
}}}
|
||||
cands := &fakeCandidates{ids: []int{10, 11}}
|
||||
cfg := fakeAutoConfig{{Provider: "introdb", ContributeEnabled: true, ContributeAutoLocal: true, ContributeMinConfidence: 0.95}}
|
||||
task := NewContributeMarkersTask(runner, cfg, cands, fakeFileLoader{})
|
||||
|
||||
prog := &contribTestProgress{}
|
||||
if err := task.Execute(context.Background(), prog); err != nil {
|
||||
t.Fatalf("Execute: %v", err)
|
||||
}
|
||||
if len(runner.calls) != 1 {
|
||||
t.Fatalf("expected task to stop after rate limit, calls=%d", len(runner.calls))
|
||||
}
|
||||
var data map[string]int
|
||||
if err := json.Unmarshal(prog.data, &data); err != nil {
|
||||
t.Fatalf("decode result data: %v", err)
|
||||
}
|
||||
if data["retry_after_seconds"] != 90 {
|
||||
t.Fatalf("retry_after_seconds = %d, want 90", data["retry_after_seconds"])
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user