From 39ba284c9d2ee57ad86ce7403d510c6e29bbeb64 Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Wed, 10 Jun 2026 14:58:54 -0400 Subject: [PATCH] =?UTF-8?q?feat(ai):=20shared=20AI=20core=20=E2=80=94=20me?= =?UTF-8?q?tadata=20translation,=20Whisper=20ASR,=20per-profile=20language?= =?UTF-8?q?,=20on-view=20translation=20(#127)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: design + plan for shared AI core, metadata translation, Whisper ASR Co-Authored-By: Claude Fable 5 * refactor(ai): shared LLM client, segment translator, and job runner packages internal/ai/llm: OpenAI-compatible chat client moved out of subtitles/ai, plus /v1/audio/transcriptions (verbose_json) for the ASR work; one shared retry/backoff loop for both. internal/ai/translate: the batched indexed-JSON translation protocol generalized to text segments. internal/ai/jobrunner: dispatch/heartbeat/reaper/cancel lifecycle extracted behind a minimal store interface, with a semaphore shareable across job services. Co-Authored-By: Claude Fable 5 * refactor(subtitles): consume shared AI core LLMTranslator becomes a thin cue<->segment adapter over aitranslate; the service delegates dispatch/heartbeat/reaper/cancel to jobrunner; the local OpenAI client is gone in favor of internal/ai/llm. Behavior (prompts, wire protocol, job rows, recovery semantics) is unchanged. NewService now takes the dispatch semaphore so all AI job services can share one bound. Co-Authored-By: Claude Fable 5 * feat(config): shared ai.* settings, metadata translation job table, localization provenance columns ai.* connection keys (chat + optional separate ASR endpoint) load with a fallback to the legacy subtitle_ai.* rows — those are never renamed in SQL because encrypted values are GCM-bound to their setting key. New toggles: subtitle_ai.transcribe_enabled, metadata_ai.enabled. Migration adds metadata_translation_jobs, per-field provenance (provider|ai|manual) on the localization tables, and media_folders.auto_translate_metadata. Co-Authored-By: Claude Fable 5 * feat(catalog): localization field provenance with provider/ai/manual precedence Provider upserts keep manual values and never blank a field with an empty incoming value; new UpsertAITranslation/UpsertAIOverview methods write AI fields only over empty or ai-sourced values (force adds provider, never manual) — all enforced in single-statement SQL. Serving now merges only non-empty localized fields onto the base item, since localization rows are legitimately partial (AI rows carry no titles/artwork). Co-Authored-By: Claude Fable 5 * feat(metadata): AI translation service, refresh auto-fallback, and admin API internal/metadata/translation: job service over the shared AI core that expands an item to its season/episode overviews, skips already-localized fields (zero model calls on repeat runs), batches paragraphs through the generic translator, and persists per batch with provenance-aware upserts. MetadataService gains an AutoTranslator seam invoked after each refresh for libraries with auto_translate_metadata. Admin endpoints under the metadata curation guard: enqueue, list (poll), cancel; plus a status probe. Co-Authored-By: Claude Fable 5 * feat(subtitles): Whisper ASR transcribe and transcribe_translate jobs New WhisperTranscriber: one ffmpeg pass extracts the audio track to 10-min 16kHz mono WAV chunks (temp dir cleaned on every exit path), each chunk goes to the OpenAI-compatible /v1/audio/transcriptions endpoint (verbose_json, per-request timeout sized to 3x chunk duration), segment timestamps are offset and built into wrapped cues. Chunks process playhead-first and stream live to the requesting session. The transcript is stored as an ordinary downloaded subtitle (provider 'transcribed'); transcribe_translate chains the existing translator and stores the translated track as the job result. Enqueue accepts an optional kind; status reports transcribe_enabled. Co-Authored-By: Claude Fable 5 * feat(web): AI services settings, metadata translate action, library auto-translate, generate-from-audio New AI Services admin page hosts the shared endpoint config (reads fall back to legacy subtitle_ai.* values, writes target ai.*) and the three feature toggles; the AI card moves out of Subtitles settings. The metadata editor gains a Translate-with-AI panel with job polling and force/re-translate. The library form gains the auto-translate toggle (threaded through the libraries API). The player translate modal gains a From-audio mode that lists audio tracks and submits transcribe / transcribe_translate jobs. Co-Authored-By: Claude Fable 5 * style: gofmt import grouping in router and translation tests Co-Authored-By: Claude Fable 5 * feat(catalog): per-profile metadata language and viewer-triggered description translation user_profiles.preferred_metadata_language threads through the access scope into catalog serving: presentation language now resolves explicit param -> profile preference -> library metadata language (native API and jellycompat). ItemDetail gains pending_translation_language when the viewer's language is missing a localized overview. New metadata_ai.on_view setting (off|button| auto) gates POST /items/{id}/translate-description: any profile with item access may request its language, with in-flight dedup and a 15-minute failure cooldown so page views never hammer a broken endpoint. Co-Authored-By: Claude Fable 5 * feat(web): on-view description translation with per-profile metadata language Profile playback settings gain a Metadata language picker (library default inherit). Detail pages: when the server reports pending_translation_language and metadata_ai.on_view is 'auto', the description translates on view with a pulse animation until the refetched detail comes back localized (45s timeout); in 'button' mode a small Translate chip triggers the same flow. Co-Authored-By: Claude Fable 5 * fix(web): expose metadata_ai.on_view in AI Services settings The on-view translation mode had no UI control, so it could only ever be 'off' — viewers got neither the auto translation nor the fallback button. Adds the off/button/auto selector to the Features card, and the config loader now warns and falls back to 'off' on a bad row instead of refusing to start. Co-Authored-By: Claude Fable 5 * fix(ai): clear configuration hint when the transcription endpoint is chat-only A blank Transcription base URL falls back to the chat endpoint; chat-only gateways reject the multipart upload with an opaque 400 that reads like a pipeline bug. 400/404/405 transcription failures now carry a hint to set a Whisper-compatible endpoint in AI Services. Co-Authored-By: Claude Fable 5 * fix(subtitles): wrap ASR cue text by rune count, not bytes Arabic/Cyrillic/Greek text is 2+ bytes per character in UTF-8, so byte-based wrapping broke lines at roughly half the intended visual width. Co-Authored-By: Claude Fable 5 * docs(web): steer transcription base URL hint away from chat-only gateways Co-Authored-By: Claude Fable 5 * feat(ai): block chat-only gateways for transcription, add endpoint presets llm.IsChatOnlyGateway (OpenRouter et al — no timestamped transcription API) is enforced in three layers: the settings API rejects ai.asr_base_url values pointing at one, the router disables ASR with a warning when the blank-URL fallback would land on one, and llm.Transcribe refuses outright. The AI Services page gains one-click transcription presets (Groq turbo/accurate, OpenAI, self-hosted speaches) plus the mirrored client-side check, and the settings API now also validates metadata_ai.on_view. Co-Authored-By: Claude Fable 5 * fix(subtitles): tighten ASR subtitle sync Three systematic timing-error sources addressed: cue offsets now use the segment muxer's exact per-chunk start times (segment_list CSV) instead of assuming index*chunk_seconds; the audio stream's start delay relative to the container timeline (common in TS remuxes) is probed via ffprobe and added to every cue; and the chunk length is now operator-tunable via subtitle_ai.asr_chunk_seconds (60-600s, default 600) since shorter chunks bound Whisper's within-chunk timestamp drift. Playhead-first ordering now pivots on real chunk starts, and a beyond-end playhead starts at the final chunk instead of restarting from zero. Co-Authored-By: Claude Fable 5 * fix(ai): tolerate base URLs that already include the /v1 segment Providers like DeepInfra expose their OpenAI-compatible API under a base that contains the version segment (api.deepinfra.com/v1/openai); always appending /v1/... mangled those. endpointURL now appends bare paths when the base already carries /v1. Co-Authored-By: Claude Fable 5 * feat(web): prefer self-hosted transcription in presets and hints Preset order becomes self-hosted (recommended) -> Groq turbo -> Groq large-v3 -> OpenAI, and the settings hint plus the job-error hint lead with the self-hosted option. The self-hosted preset now fills the turbo CT2 model to match the recommended speaches setup. Co-Authored-By: Claude Fable 5 * fix(subtitles): request VAD and word timestamps for ASR cue accuracy Without vad_filter, faster-whisper servers report wall-to-wall segment times: cues linger on screen through silence (verified up to 91s) and paragraph-length segments become single 400+ char cues. Request vad_filter=true (skipped for hosted providers that reject non-OpenAI fields and run VAD server-side) plus timestamp_granularities word+segment, and rebuild cues from word timings: split at speech pauses, sentence ends, text capacity, and a 7s max duration; cap word-less segments instead of trusting their reported end; stretch sub-second cues to a readable minimum. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../2026-06-10-ai-translation-and-asr.md | 377 +++++++++++ ...026-06-10-ai-translation-and-asr-design.md | 311 +++++++++ ...-on-view-description-translation-design.md | 78 +++ internal/access/resolver.go | 1 + internal/access/types.go | 7 +- internal/ai/jobrunner/runner.go | 197 ++++++ internal/ai/jobrunner/runner_test.go | 158 +++++ internal/ai/jobrunner/status.go | 22 + internal/ai/llm/client.go | 297 +++++++++ internal/ai/llm/client_test.go | 350 +++++++++++ internal/ai/llm/transcribe.go | 286 +++++++++ .../ai => ai/translate}/languages.go | 24 +- internal/ai/translate/translate.go | 212 +++++++ internal/ai/translate/translate_test.go | 153 +++++ internal/api/handlers/admin.go | 16 + internal/api/handlers/items.go | 17 +- internal/api/handlers/libraries.go | 4 + internal/api/handlers/metadata_ai.go | 212 +++++++ internal/api/handlers/profiles.go | 6 + internal/api/handlers/subtitle_ai.go | 29 +- internal/api/router.go | 100 ++- internal/catalog/access_filter.go | 6 +- internal/catalog/detail.go | 57 +- internal/catalog/encrypted_settings_repo.go | 6 +- internal/catalog/folder_repo.go | 10 +- internal/catalog/localization_repo.go | 187 +++++- internal/catalog/localize.go | 75 +++ internal/catalog/localize_test.go | 109 ++++ internal/config/config.go | 45 +- internal/config/db_loader.go | 51 +- internal/jellycompat/access_filter.go | 13 +- internal/metadata/service.go | 44 ++ internal/metadata/translation/job.go | 139 ++++ internal/metadata/translation/pgrepo.go | 279 ++++++++ internal/metadata/translation/prompt.go | 34 + internal/metadata/translation/service.go | 593 ++++++++++++++++++ internal/metadata/translation/service_test.go | 544 ++++++++++++++++ internal/metadata/translation/store.go | 51 ++ internal/models/media.go | 15 + internal/playback/audio_extract.go | 180 ++++++ internal/subtitles/ai/client.go | 190 ------ internal/subtitles/ai/config.go | 32 +- internal/subtitles/ai/job.go | 45 +- internal/subtitles/ai/service.go | 441 ++++++++----- internal/subtitles/ai/service_test.go | 6 +- internal/subtitles/ai/transcriber.go | 327 ++++++++++ internal/subtitles/ai/transcriber_test.go | 374 +++++++++++ internal/subtitles/ai/translator.go | 193 ++---- internal/userstore/pgstore/profiles.go | 15 +- internal/userstore/types.go | 2 + ...143224_ai_metadata_translation_and_asr.sql | 73 +++ ...38_profile_preferred_metadata_language.sql | 13 + web/src/api/types.ts | 10 + web/src/components/EditMetadataDialog.tsx | 5 + web/src/components/MetadataTranslatePanel.tsx | 134 ++++ .../admin/libraries/LibraryFormSections.tsx | 15 + .../admin/libraries/useLibraryForm.ts | 6 + web/src/hooks/queries/items.ts | 78 +++ web/src/hooks/useOnViewTranslation.ts | 85 +++ web/src/pages/ItemDetail/DetailHero.tsx | 39 +- .../pages/ItemDetail/MovieContent.test.tsx | 4 + web/src/pages/ItemDetail/MovieContent.tsx | 5 + .../pages/ItemDetail/SeriesContent.test.tsx | 4 + web/src/pages/ItemDetail/SeriesContent.tsx | 5 + .../admin-settings/AIServicesSettings.tsx | 408 ++++++++++++ .../admin-settings/AdminSettingsLayout.tsx | 3 + .../admin-settings/SubtitlesSettings.tsx | 130 +--- web/src/pages/settings/PlaybackSettings.tsx | 34 + web/src/player/components/PlayerControls.tsx | 1 + web/src/player/components/SubtitleMenu.tsx | 58 +- .../components/SubtitleTranslateModal.tsx | 173 +++-- 71 files changed, 7388 insertions(+), 815 deletions(-) create mode 100644 docs/superpowers/plans/2026-06-10-ai-translation-and-asr.md create mode 100644 docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md create mode 100644 docs/superpowers/specs/2026-06-10-on-view-description-translation-design.md create mode 100644 internal/ai/jobrunner/runner.go create mode 100644 internal/ai/jobrunner/runner_test.go create mode 100644 internal/ai/jobrunner/status.go create mode 100644 internal/ai/llm/client.go create mode 100644 internal/ai/llm/client_test.go create mode 100644 internal/ai/llm/transcribe.go rename internal/{subtitles/ai => ai/translate}/languages.go (64%) create mode 100644 internal/ai/translate/translate.go create mode 100644 internal/ai/translate/translate_test.go create mode 100644 internal/api/handlers/metadata_ai.go create mode 100644 internal/catalog/localize.go create mode 100644 internal/catalog/localize_test.go create mode 100644 internal/metadata/translation/job.go create mode 100644 internal/metadata/translation/pgrepo.go create mode 100644 internal/metadata/translation/prompt.go create mode 100644 internal/metadata/translation/service.go create mode 100644 internal/metadata/translation/service_test.go create mode 100644 internal/metadata/translation/store.go create mode 100644 internal/playback/audio_extract.go delete mode 100644 internal/subtitles/ai/client.go create mode 100644 internal/subtitles/ai/transcriber.go create mode 100644 internal/subtitles/ai/transcriber_test.go create mode 100644 migrations/sql/20260610143224_ai_metadata_translation_and_asr.sql create mode 100644 migrations/sql/20260610153638_profile_preferred_metadata_language.sql create mode 100644 web/src/components/MetadataTranslatePanel.tsx create mode 100644 web/src/hooks/useOnViewTranslation.ts create mode 100644 web/src/pages/admin-settings/AIServicesSettings.tsx diff --git a/docs/superpowers/plans/2026-06-10-ai-translation-and-asr.md b/docs/superpowers/plans/2026-06-10-ai-translation-and-asr.md new file mode 100644 index 00000000..97ba7243 --- /dev/null +++ b/docs/superpowers/plans/2026-06-10-ai-translation-and-asr.md @@ -0,0 +1,377 @@ +# AI Services Core, Metadata Translation & Whisper ASR — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Design:** `docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md` (read it first; this plan does not repeat the rationale). + +**Goal:** Extract the subtitle-AI LLM plumbing into shared packages (`internal/ai/llm`, `internal/ai/translate`, `internal/ai/jobrunner`), add AI translation of overviews/taglines into the existing localization tables with field provenance, and add Whisper ASR subtitle generation (`transcribe`, `transcribe_translate`). Single PR. + +**Architecture:** New `internal/ai/*` packages consumed by a refactored `internal/subtitles/ai` and a new `internal/metadata/translation` service. One new Goose migration (jobs table, provenance columns, library auto-translate flag). New `ai.*` settings keys with loader fallback to legacy `subtitle_ai.*` rows (encrypted values are GCM-bound to their key — never rename rows in SQL). Frontend: AI Services settings page, metadata-editor translate action, library toggle, player "generate from audio" mode. + +**Tech stack:** Go (chi, pgx), PostgreSQL via Goose SQL migrations, React + TypeScript (TanStack Query), Vitest. + +Commands assume the repository root is the cwd. + +--- + +## Ordering and verification + +Tasks 1–4 are a pure refactor and must leave subtitle translation behaviorally unchanged — run `go build ./... && go test ./internal/...` after each. Tasks 5–11 are backend features; 12–15 frontend; 16 final verification. + +Before opening the MR: `make lint`, `cd web && pnpm run lint && pnpm run format:check`, `make verify-local-paths`. + +--- + +## File structure + +**Create** +- `internal/ai/llm/client.go`, `client_test.go`, `config.go` — shared OpenAI-compatible client (chat + transcription) +- `internal/ai/llm/transcribe.go` — multipart `/v1/audio/transcriptions`, `verbose_json` types +- `internal/ai/translate/translate.go`, `translate_test.go` — generic segment batch translator (package `aitranslate`) +- `internal/ai/jobrunner/runner.go`, `runner_test.go` — dispatch/semaphore/heartbeat/reaper/cancel registry +- `internal/metadata/translation/{job.go,pgrepo.go,service.go,prompt.go,service_test.go}` +- `internal/subtitles/ai/transcriber.go`, `transcriber_test.go` — ASR pipeline +- `internal/playback/audio_extract.go` — ffmpeg chunked audio extraction helper +- `internal/api/handlers/metadata_ai.go` +- `migrations/sql/_ai_metadata_translation_and_asr.sql` (via `make migrate-create`) +- `web/src/pages/admin-settings/AIServicesSettings.tsx` + +**Modify** +- `internal/subtitles/ai/{client.go→deleted,translator.go,service.go,engine.go,job.go,config.go,pgrepo.go}` +- `internal/config/{config.go,db_loader.go}`; `internal/catalog/encrypted_settings_repo.go` +- `internal/catalog/{localization_repo.go,detail.go}`; `internal/models/media.go` +- `internal/metadata/service.go` (provider provenance + auto-enqueue hook) +- `internal/api/handlers/subtitle_ai.go`; router registration; `cmd/silo/main.go` wiring +- `web/src/pages/admin-settings/SubtitlesSettings.tsx`, `web/src/components/EditMetadataDialog.tsx`, `web/src/player/components/SubtitleTranslateModal.tsx`, library settings form, `web/src/api/` types/client, admin settings nav/route registration + +--- + +## Task 1: `internal/ai/llm` — shared client + +- [ ] **Step 1: Move the chat client.** Create `internal/ai/llm` (package `llm`). Move `Client`, `chat` (export as `Chat`), `chatMessage` (export as `Message`), retry helpers (`sleepCtx`, `rateLimitBackoff`, `truncate`) from `internal/subtitles/ai/client.go`. New `llm.Config`: + +```go +type Config struct { + BaseURL string // chat endpoint, no trailing /v1 + APIKey string + ChatModel string + ASRBaseURL string // empty = BaseURL + ASRAPIKey string // empty = APIKey + ASRModel string + MaxConcurrentJobs int +} +``` + +Log lines lose the "subtitle" wording (they are shared now). + +- [ ] **Step 2: Add `Transcribe`.** In `transcribe.go`: + +```go +type TranscribeRequest struct { + Filename string + Audio io.Reader + Language string // optional ISO-639-1 hint + Timeout time.Duration // per-request; sized to chunk length by the caller +} +type TranscriptionSegment struct{ Start, End float64; Text string } +type Transcription struct{ Language string; Segments []TranscriptionSegment } +func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (*Transcription, error) +``` + +Multipart fields: `file`, `model` (=`ASRModel`), `response_format=verbose_json`, `temperature=0`, optional `language`. Reuse the same 429/5xx/transport retry-with-backoff loop as `Chat` (extract a shared `doWithRetry` rather than duplicating it). Empty `segments` in the response is an error (`transcription returned no segments`), not a silent fallback. + +- [ ] **Step 3: Tests** (`client_test.go`, httptest): chat retries on 429 (honors `Retry-After`), 5xx, 200-with-error-object, empty choices; transcribe happy path (multipart fields present, segments parsed), missing-segments error, ASR base-url/key override falling back to chat values. + +- [ ] **Step 4: Commit.** `refactor(ai): extract shared OpenAI-compatible LLM client with transcription support` + +--- + +## Task 2: `internal/ai/translate` — generic segment translator + +- [ ] **Step 1: Move the batch logic.** Package `aitranslate`. Move `buildIndexedJSON`, `extractJSONObject`, and the batch loop from `internal/subtitles/ai/translator.go`, generalized: + +```go +type Segment struct{ ID, Text string } +type Request struct { + Segments []Segment + SystemPrompt string // caller-supplied; domain-specific + BatchSize int + ContextNeighbors int // preceding source segments sent untranslated +} +type ChatFn func(ctx context.Context, system, user string) (string, error) +func Translate(ctx context.Context, chat ChatFn, req Request, + onBatch func(batch []Segment, done, total int)) ([]Segment, error) +``` + +Wire protocol unchanged: 1-based indexed JSON per batch, same-keys response, malformed-response retries (`maxRetries = 2`), completeness check. `ChatFn` keeps the package free of an `llm` dependency and trivially testable. + +- [ ] **Step 2: Tests:** batch splitting/boundaries, context neighbors, code-fence tolerance, omitted-key retry then failure, ctx cancellation between batches. + +- [ ] **Step 3: Commit.** `refactor(ai): extract generic batched segment translator` + +--- + +## Task 3: `internal/ai/jobrunner` — shared job lifecycle + +- [ ] **Step 1: Extract the runner.** Move dispatch/heartbeat/reaper/cancel mechanics from `internal/subtitles/ai/service.go` (lines ~136–319) behind: + +```go +type Store interface { + Heartbeat(ctx context.Context, id int64) error + ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error) + MarkCancelled(ctx context.Context, id int64, message string) error +} +type Runner struct{ /* baseCtx, sem (shared), store, logger, cancels, wg */ } +func New(appCtx context.Context, sem chan struct{}, store Store, logger *slog.Logger) *Runner +func (r *Runner) Recover() // reap + background reaper loop +func (r *Runner) Dispatch(id int64, run func(ctx context.Context)) // semaphore + heartbeat + cancel registry +func (r *Runner) Cancel(id int64) bool // true if an in-flight goroutine was cancelled +``` + +Heartbeat-while-queued behavior (the comment block in `dispatch`) must be preserved verbatim — it is load-bearing for multi-instance safety. Constants (30 s heartbeat, 2 min stale, 1 min reaper) move here. + +- [ ] **Step 2: Shared semaphore.** The semaphore is **constructed by the caller** (`cmd/silo`) at size `ai.max_concurrent_jobs` and passed to every `Runner`, so subtitle + metadata + ASR jobs share one bound. + +- [ ] **Step 3: Tests:** dispatch bounded by a size-1 shared semaphore across two runners; cancel of queued job marks cancelled without running; reaper resets a stale heartbeat row (fake store). + +- [ ] **Step 4: Commit.** `refactor(ai): extract shared job lifecycle runner` + +--- + +## Task 4: Refactor `internal/subtitles/ai` onto the shared core + +- [ ] **Step 1:** Delete `client.go`; `LLMTranslator` becomes an adapter: cues → `aitranslate.Segment` (ID = 1-based index, Text = joined lines), subtitle system prompt stays here, `splitCueLines` maps back. `Service` keeps job semantics but delegates lifecycle to `jobrunner.Runner`; `pgrepo` satisfies `jobrunner.Store`. +- [ ] **Step 2:** Update `cmd/silo/main.go` wiring (build `llm.Client` + shared semaphore once; pass into the subtitle service). +- [ ] **Step 3:** `go build ./... && go test ./internal/...` — subtitle translation behavior unchanged (same prompts, same batching, same job rows). +- [ ] **Step 4: Commit.** `refactor(subtitles): consume shared AI core` + +--- + +## Task 5: Migration + +- [ ] **Step 1:** `make migrate-create NAME=ai_metadata_translation_and_asr`, then fill the generated file: + +```sql +-- +goose Up +-- +goose StatementBegin +CREATE TABLE public.metadata_translation_jobs ( + id bigserial PRIMARY KEY, + target_kind text NOT NULL, -- 'item' | 'season' | 'episode' + content_id text NOT NULL, + include_children boolean NOT NULL DEFAULT true, + source_language text NOT NULL DEFAULT '', + target_language text NOT NULL, + engine text NOT NULL DEFAULT 'openai', + model text NOT NULL DEFAULT '', + status text NOT NULL DEFAULT 'pending', -- pending|running|completed|failed|cancelled + progress double precision NOT NULL DEFAULT 0, + progress_message text NOT NULL DEFAULT '', + fields_done integer NOT NULL DEFAULT 0, + fields_total integer NOT NULL DEFAULT 0, + force boolean NOT NULL DEFAULT false, + error_message text NOT NULL DEFAULT '', + idempotency_key text NOT NULL, + requested_by integer, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + heartbeat_at timestamptz NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX metadata_translation_jobs_active_idempotency_idx + ON public.metadata_translation_jobs (idempotency_key) + WHERE status IN ('pending', 'running'); +CREATE INDEX metadata_translation_jobs_content_idx + ON public.metadata_translation_jobs (content_id, created_at DESC); +CREATE INDEX metadata_translation_jobs_status_idx + ON public.metadata_translation_jobs (status) WHERE status IN ('pending', 'running'); + +-- Field provenance: 'provider' | 'ai' | 'manual'. manual > provider > ai. +ALTER TABLE media_item_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')), + ADD COLUMN tagline_source text NOT NULL DEFAULT 'provider' + CHECK (tagline_source IN ('provider', 'ai', 'manual')); +ALTER TABLE season_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')); +ALTER TABLE episode_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')); + +ALTER TABLE media_folders + ADD COLUMN auto_translate_metadata boolean NOT NULL DEFAULT false; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE media_folders DROP COLUMN auto_translate_metadata; +ALTER TABLE episode_localizations DROP COLUMN overview_source; +ALTER TABLE season_localizations DROP COLUMN overview_source; +ALTER TABLE media_item_localizations DROP COLUMN tagline_source, DROP COLUMN overview_source; +DROP TABLE IF EXISTS public.metadata_translation_jobs; +-- +goose StatementEnd +``` + +- [ ] **Step 2:** `make migrate-up` against the local stack (`docker compose up -d postgres redis`), then `make migrate-status`. +- [ ] **Step 3: Commit.** `feat(metadata): migration for translation jobs, localization provenance, library auto-translate` + +--- + +## Task 6: Settings & config + +- [ ] **Step 1: Config structs** (`internal/config/config.go`): add `AIConfig` (fields mirroring `llm.Config`); slim `SubtitleAIConfig` to `Enabled`, `TranscribeEnabled`, `BatchSize`, `ContextNeighbors`; add `MetadataAIConfig{Enabled bool}`. +- [ ] **Step 2: Loader** (`internal/config/db_loader.go`): load `ai.*` with legacy fallback, following the `recommendations.embedding_auth_token` precedent at line ~409: + +```go +cfg.AI.BaseURL = stringOr(m, "ai.base_url", stringOr(m, "subtitle_ai.base_url", "https://api.openai.com")) +cfg.AI.APIKey = stringOr(m, "ai.api_key", stringOr(m, "subtitle_ai.api_key", "")) +cfg.AI.ChatModel = stringOr(m, "ai.chat_model", stringOr(m, "subtitle_ai.chat_model", "gpt-4o-mini")) +// ai.max_concurrent_jobs ← subtitle_ai.max_concurrent_jobs ← 2 +// ai.asr_model (default "whisper-1"), ai.asr_base_url, ai.asr_api_key (default "") +// subtitle_ai.transcribe_enabled, metadata_ai.enabled (default false) +``` + +**Never** rename the legacy rows in SQL — encrypted values are GCM-bound to their setting key (`internal/catalog/encrypted_settings_repo.go`). + +- [ ] **Step 3:** Add `ai.api_key`, `ai.asr_api_key` to `sensitiveSettingKeys`. +- [ ] **Step 4:** Readiness helpers: subtitle translate = `SubtitleAI.Enabled && AI chat ready`; ASR = `SubtitleAI.TranscribeEnabled && AI.ASRModel != "" && (asr or chat base URL set)`; metadata = `MetadataAI.Enabled && AI chat ready`. +- [ ] **Step 5: Commit.** `feat(config): shared ai.* settings with legacy subtitle_ai fallback` + +--- + +## Task 7: Provenance-aware localization writes + +- [ ] **Step 1: Models** (`internal/models/media.go`): add `OverviewSource`/`TaglineSource` to `MediaItemLocalization`, `OverviewSource` to `SeasonLocalization`/`EpisodeLocalization`. +- [ ] **Step 2: Provider upsert rules** (`internal/catalog/localization_repo.go`): rewrite the three `Upsert` statements so, per AI-writable field: + - existing source `manual` → keep existing value and source; + - incoming value empty → keep existing value and source; + - otherwise → take incoming value, set source `provider`. + +Single-statement `ON CONFLICT … DO UPDATE` with `CASE` (no read-modify-write). Pattern for one field: + +```sql +overview = CASE + WHEN media_item_localizations.overview_source = 'manual' THEN media_item_localizations.overview + WHEN EXCLUDED.overview = '' THEN media_item_localizations.overview + ELSE EXCLUDED.overview END, +overview_source = CASE + WHEN media_item_localizations.overview_source = 'manual' THEN media_item_localizations.overview_source + WHEN EXCLUDED.overview = '' THEN media_item_localizations.overview_source + ELSE 'provider' END +``` + +- [ ] **Step 3: AI upsert methods**: `UpsertAITranslation(ctx, contentID, language string, overview, tagline *string, force bool)` per repo (nil pointer = field not part of this write). Insert path populates only the translated fields (other text columns empty). Update path per field: write when existing source is `ai`, existing value is empty, or `force` — never when `manual`; `force` may overwrite `provider` (the admin asked). Sets source `ai`. +- [ ] **Step 4: Serving hardening** (`internal/catalog/detail.go`): verify `LocalizeItemModel` / `LocalizeSeasonModel` / `LocalizeEpisodeModel` only override base fields with **non-empty** localization values (AI rows carry empty titles). Fix any field that clobbers; add a regression test. +- [ ] **Step 5: Tests** for the upsert matrix (provider-over-ai, ai-skips-provider-unless-force, manual-untouchable, empty-never-blanks), following the existing repo/service test patterns in `internal/catalog`. +- [ ] **Step 6: Commit.** `feat(catalog): field provenance for localizations with provider/ai/manual precedence` + +--- + +## Task 8: `internal/metadata/translation` service + +- [ ] **Step 1: Job model + repo** (`job.go`, `pgrepo.go`): mirror `internal/subtitles/ai/{job.go,pgrepo.go}` shapes against `metadata_translation_jobs`; satisfy `jobrunner.Store`. Idempotency key = SHA-256 of `content_id|target_kind|target_language|model`. +- [ ] **Step 2: Service** (`service.go`): `Enqueue`, `GetJob`, `ListJobs(contentID)`, `Cancel`, `Recover`, using `jobrunner.Runner` + the shared semaphore. The run loop: + 1. Expand targets: `item` → item overview+tagline; series item with `include_children` → + every season overview + every episode overview; `season` (+children → its episodes); `episode` → its overview. Source text comes from the **base** rows (default metadata language); `source_language` recorded from the item's `default_metadata_language`. + 2. Skip-if-filled: drop any field whose target-language localization value is already non-empty, unless `force`. All skipped → complete immediately ("Nothing to translate"). + 3. Build `aitranslate.Segment`s (IDs `item:overview`, `item:tagline`, `season::overview`, `episode::overview`), translate with `prompt.go`'s system prompt (names the title/year, "translate media catalog descriptions", preserve proper nouns/character names/tone, no added information), `BatchSize` = package constant `metadataBatchSize = 10`, `ContextNeighbors = 0`. + 4. Per batch: provenance-aware AI upserts (Task 7) + `fields_done`/progress update. Persisting per batch means a cancelled job keeps completed fields. +- [ ] **Step 3: Tests** (fake repos + fake `ChatFn`): series expansion counts, skip-if-filled short-circuit (zero chat calls), force overwrites `ai`+`provider` but never `manual`, per-batch persistence on cancellation mid-job. +- [ ] **Step 4: Commit.** `feat(metadata): AI translation service for overviews and taglines` + +--- + +## Task 9: Ingestion hook + library flag + +- [ ] **Step 1:** Switch the provider localization writes in `internal/metadata/service.go` (item ~line 1476, season ~2921, episode ~3080) to the provenance-aware upserts — no other behavior change. +- [ ] **Step 2:** Thread `auto_translate_metadata` through the `MediaFolder` model, folder repo scan/update, and the folder settings API payload. +- [ ] **Step 3: Auto-enqueue hook.** After an item's refresh persists (single point at the end of the item flow — not per season/episode), when: folder flag set, folder `metadata_language` non-empty and ≠ item `default_metadata_language`, metadata AI ready, and the item's target-language localization is missing overview (or tagline) → `Enqueue` a non-force `item` job with `include_children=true`. Fire-and-forget (log on error); the active-idempotency index plus skip-if-filled make repeat refreshes free. +- [ ] **Step 4: Commit.** `feat(metadata): per-library auto-translate fallback on refresh` + +--- + +## Task 10: Metadata AI API + +- [ ] **Step 1:** `internal/api/handlers/metadata_ai.go`, mirroring `subtitle_ai.go` shapes: + - `GET /api/v1/metadata/ai/status` → `{enabled}` + - `POST /api/v1/metadata/ai/translate` `{content_id, target_kind, target_language, include_children, force}` → `{job}` (joins an in-flight duplicate, same as subtitles) + - `GET /api/v1/metadata/ai/jobs/{job_id}`, `POST /api/v1/metadata/ai/jobs/{job_id}/cancel`, `GET /api/v1/metadata/ai/jobs?content_id=…` +- [ ] **Step 2:** Gate with the same permission as metadata editing (metadata curation permission; see `internal/catalog/update.go` callers). Register routes; wire the service in `cmd/silo/main.go`. +- [ ] **Step 3:** Handler tests following `internal/api/handlers` conventions (validation: bad kind, missing language, disabled engine → 400/503 mapping). +- [ ] **Step 4: Commit.** `feat(api): metadata AI translation endpoints` + +--- + +## Task 11: Whisper ASR + +- [ ] **Step 1: Audio extraction** (`internal/playback/audio_extract.go`), alongside the existing subtitle extraction helpers: + +```go +// ExtractAudioChunks extracts one audio track to 16 kHz mono WAV chunks in dir. +// Returns ordered chunk paths; chunkSeconds = 600 in production. +func ExtractAudioChunks(ctx context.Context, filePath string, audioTrackIndex int, + dir, ffmpegPath string, chunkSeconds int) ([]string, error) +``` + +Single ffmpeg pass: `-vn -map 0:a: -ac 1 -ar 16000 -c:a pcm_s16le -f segment -segment_time /chunk%05d.wav`. + +- [ ] **Step 2: Transcriber** (`internal/subtitles/ai/transcriber.go`): + +```go +type Transcriber interface { + Transcribe(ctx context.Context, req TranscribeJobRequest, + onChunk func(cues []SubtitleCue, done, total int)) ([]SubtitleCue, string /*detected lang*/, error) +} +``` + +`WhisperTranscriber` implementation: temp dir (`os.MkdirTemp`, removed on all exit paths) → `ExtractAudioChunks` → per chunk `llm.Client.Transcribe` (per-request timeout ∝ chunk duration; pass the job's language hint) → offset segment times by `chunkIndex*chunkSeconds` → cues (one per segment, wrapped to ≤2 lines ×~42 chars via a `wrapCueText` helper; whitespace-only segments dropped). Chunk processing order starts at the chunk containing `StartPosition`, then wraps (mirrors `reorderFromPosition` semantics). + +- [ ] **Step 3: Wire kinds into the service.** `Service.run` branches on `job.Kind`: + - `translate`: existing path. + - `transcribe`: resolve audio track (`source_index` = audio index, `-1` → default/first; reject files without audio), transcribe (progress 10–70 %, live cue streaming via the existing notifier callbacks), store SRT as provider `transcribed`, language = hint or detected, release name `" (AI transcribed)"`, notify `SubtitleReady`. + - `transcribe_translate`: `transcribe`, store the transcript track, then run the existing `Translator` on the cues (70–95 %), store provider `translated`, `result_subtitle_id` = translated track. + +Enqueue validation per kind (ASR requires `TranscribeEnabled`; idempotency model component = `asr_model` or `asr_model+chat_model` for the chained kind). + +- [ ] **Step 4:** API surface: `kind` field (optional, default `translate`) on the existing enqueue request in `subtitle_ai.go`; `transcribe_enabled` on the status response. +- [ ] **Step 5: Tests** (`transcriber_test.go`, fake client): timestamp stitching across chunks, playhead-first chunk ordering, cue wrapping, empty-segments chunk → job error, temp-dir cleanup on failure (assert via `t.TempDir` layout). +- [ ] **Step 6: Commit.** `feat(subtitles): Whisper ASR transcribe and transcribe_translate jobs` + +--- + +## Task 12: Frontend — AI Services settings page + +- [ ] **Step 1:** `web/src/pages/admin-settings/AIServicesSettings.tsx`: connection card (`ai.base_url`, `ai.api_key` via the sensitive-key pattern already used in `SubtitlesSettings.tsx:264`, `ai.chat_model`, `ai.asr_model`, optional `ai.asr_base_url`/`ai.asr_api_key`, `ai.max_concurrent_jobs`) + features card (`subtitle_ai.enabled`, `subtitle_ai.transcribe_enabled`, `metadata_ai.enabled`, `subtitle_ai.batch_size`, `subtitle_ai.context_neighbors`). Reads show effective values (new key, falling back to the legacy key when unset — same fallback order as the loader); writes always target the new `ai.*` keys. +- [ ] **Step 2:** Remove the AI card from `SubtitlesSettings.tsx` (leave a link/hint to the new page); register the page in the admin settings nav + route following the existing page registrations. +- [ ] **Step 3: Commit.** `feat(web): AI services settings page` + +--- + +## Task 13: Frontend — metadata translate action + +- [ ] **Step 1:** API client functions + types for the Task 10 endpoints. +- [ ] **Step 2:** `EditMetadataDialog.tsx`: "Translate with AI" action (visible when `metadata_ai` status is enabled): target language (default = library `metadata_language`), `include_children` (series only), `force` checkbox; enqueue, poll the job every ~1.5 s, show `progress_message`/`fields_done`, invalidate the item detail query on completion. +- [ ] **Step 3: Commit.** `feat(web): translate descriptions from the metadata editor` + +--- + +## Task 14: Frontend — library auto-translate toggle + +- [ ] **Step 1:** Add the `auto_translate_metadata` switch to the library settings form next to the metadata-language field (helper text: "When metadata providers have no translation for this library's language, translate descriptions with AI"). Disabled state with hint when `metadata_ai` is off. +- [ ] **Step 2: Commit.** `feat(web): library auto-translate toggle` + +--- + +## Task 15: Frontend — player "Generate from audio" + +- [ ] **Step 1:** `SubtitleTranslateModal.tsx`: when `transcribe_enabled`, add a source option "Generate from audio (AI)" listing the file's audio tracks; submit with `kind: "transcribe"` (or `transcribe_translate` when the chosen output language differs from the audio language). Surface it prominently when no text subtitle source exists (today's dead end for bitmap-only files). +- [ ] **Step 2: Commit.** `feat(web): generate subtitles from audio in the player` + +--- + +## Task 16: Verification + +- [ ] `go build ./... && go test ./...` +- [ ] `make lint` +- [ ] `cd web && pnpm run lint && pnpm run format:check` +- [ ] `make verify-local-paths` +- [ ] Manual smoke against the local stack: enqueue a metadata translation for a series (verify localization rows + provenance values), re-run a refresh (verify provider values overwrite `ai` rows and `manual` simulation survives), run a `transcribe` job on a short file against a local Whisper-compatible server. +- [ ] MR description: problem, approach, link to the design doc, risks (Whisper endpoint variance, chunk-boundary artifacts), AI-use disclosure, screenshots of the new settings page / editor action / player mode. diff --git a/docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md b/docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md new file mode 100644 index 00000000..22d97ab9 --- /dev/null +++ b/docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md @@ -0,0 +1,311 @@ +# AI services: shared core, metadata translation, Whisper ASR — design + +**Date:** 2026-06-10 +**Status:** Draft, pending review +**Scope:** Extract the AI subtitle translation engine into a shared AI core, add AI translation of movie/series/season/episode descriptions into the existing localization tables, and add Whisper ASR subtitle generation (`transcribe`, `transcribe_translate`). One PR. + +Commands assume the repository root is the cwd. + +## Problem + +`internal/subtitles/ai` ships an on-demand LLM subtitle translator. The LLM plumbing inside +it (OpenAI-compatible chat client with retry/backoff, batched indexed-JSON translation, +job lifecycle with idempotency/heartbeat/reaper/bounded concurrency) is generic, but it is +buried in a subtitle-specific package, so nothing else can use it. + +Two features want exactly that plumbing: + +1. **Metadata translation.** Libraries whose `metadata_language` the providers cannot fully + serve end up with overviews/taglines in the item's default language (usually English). + The localization tables (`media_item_localizations`, `season_localizations`, + `episode_localizations`) and the serving path (`LocalizeItemModel` and friends in + `internal/catalog/detail.go`) already exist — the gap is producing rows for languages the + providers don't have. +2. **Whisper ASR.** `subtitle_ai_jobs.kind` already reserves `transcribe` and + `transcribe_translate`; media with no usable text subtitle track (foreign audio with only + bitmap subs, or no subs at all) currently has no AI path to a subtitle track. + +## Goals + +- One shared AI core: a single OpenAI-compatible endpoint configuration (chat + audio + transcription), one retry/backoff implementation, one generic batched text translator, + one job-lifecycle runner — consumed by subtitle translation, metadata translation, and ASR. +- AI-translated overviews and taglines stored in the **existing** localization tables, so + web, Android, Apple, and jellycompat clients get them through the current serving path + with **zero client changes**. +- Provenance on AI-writable localization fields so refreshes never regress data quality: + `manual` beats `provider` beats `ai`. A later provider localization overwrites an AI + translation; the reverse never happens. +- Two metadata-translation triggers: a manual admin action, and an opt-in per-library + fallback ("provider had no localization for the library language → AI-translate") wired + into metadata ingestion. +- ASR jobs that produce ordinary `downloaded_subtitles` rows (provider `transcribed`), + with `transcribe_translate` chaining the existing LLM translator for a target-language + track; live cue streaming to the requesting playback session, same as translation today. +- Total AI load on the configured endpoint bounded by **one** shared semaphore. + +## Non-goals + +- **Episode/movie/series title translation.** Titles often have official localized forms + that are not literal translations; provider data only. +- **Collection names or descriptions.** Deferred: collections have no localization table + yet. The metadata translation service is written against content-ID + field-list inputs, + so a `collection_localizations` table can be added in a follow-up without reshaping it. +- **Genre translation.** If ever wanted, that is a static mapping, not an LLM call. +- **Per-profile metadata language.** Presentation language stays library-scoped + (`media_folders.metadata_language`); changing that is an orthogonal feature. +- **A localization editor UI.** The `manual` provenance value is honored by the write + rules but no UI sets it yet (the existing metadata editor edits base fields, not + localizations). +- **Whisper word-level timing, diarization, VAD-aligned chunking.** v1 uses fixed-length + chunks; boundary-word artifacts are an accepted limitation noted for follow-up. + +## Architecture + +### Package layout + +``` +internal/ai/llm Client: OpenAI-compatible /v1/chat/completions (moved from + internal/subtitles/ai/client.go) + new /v1/audio/transcriptions. + Shared retry/backoff/429 handling. Config for the endpoint(s). +internal/ai/translate Generic batched text translation: []Segment{ID, Text} in, + translated segments out. Indexed-JSON protocol, context + neighbors, malformed-response retries — moved from + internal/subtitles/ai/translator.go and de-subtitled. +internal/ai/jobrunner Generic job lifecycle: bounded dispatch off a shared semaphore, + heartbeat loop, stale-job reaper, cancel registry — extracted from + internal/subtitles/ai/service.go behind a small repo interface. +internal/subtitles/ai Stays: subtitle job semantics. Translate path becomes a thin + adapter (cues ↔ segments); gains the transcribe paths. +internal/metadata/translation + New: metadata translation job service + repo + provenance-aware + writes into the localization repos. +``` + +Package names: `llm`, `aitranslate`, `jobrunner` (import paths above; `aitranslate` avoids +a name collision when imported from package `ai`). + +### Shared semaphore + +`cmd/silo` builds one `chan struct{}` of size `ai.max_concurrent_jobs` and hands it to both +job services via `jobrunner`. Subtitle, metadata, and ASR jobs all draw from it, so the +operator's endpoint sees a bounded number of concurrent jobs regardless of job mix. +(An ASR job holds one slot for its whole pipeline: extract → transcribe → translate.) + +## Settings + +Connection settings move to a shared `ai.*` namespace. Because encrypted values are +GCM-bound to their setting key (see `internal/catalog/encrypted_settings_repo.go`), the +legacy rows are **not renamed**; the loader falls back to them, following the precedent of +`recommendations.embedding_auth_token` → `recommendations.openai_api_key` in +`internal/config/db_loader.go`. + +| Key | Default | Fallback | Notes | +| --- | --- | --- | --- | +| `ai.base_url` | `https://api.openai.com` | `subtitle_ai.base_url` | | +| `ai.api_key` | `""` | `subtitle_ai.api_key` | sensitive | +| `ai.chat_model` | `gpt-4o-mini` | `subtitle_ai.chat_model` | | +| `ai.max_concurrent_jobs` | `2` | `subtitle_ai.max_concurrent_jobs` | shared semaphore size | +| `ai.asr_model` | `whisper-1` | — | | +| `ai.asr_base_url` | `""` (= `ai.base_url`) | — | separate local Whisper server | +| `ai.asr_api_key` | `""` (= `ai.api_key`) | — | sensitive | +| `subtitle_ai.enabled` | `false` | — | existing; subtitle translation toggle | +| `subtitle_ai.transcribe_enabled` | `false` | — | new; ASR toggle | +| `subtitle_ai.batch_size` | `40` | — | existing; cues per chat request | +| `subtitle_ai.context_neighbors` | `2` | — | existing | +| `metadata_ai.enabled` | `false` | — | new; metadata translation toggle | + +`ai.api_key` and `ai.asr_api_key` join `sensitiveSettingKeys`. The admin UI reads through +the same fallback (effective values come from the loaded config) and writes the new keys. + +Config structs: `AIConfig` (connection + concurrency), `SubtitleAIConfig` (slims down to +toggles + batch tuning), new `MetadataAIConfig`. Readiness: subtitle translate requires +`subtitle_ai.enabled` + chat config; ASR requires `subtitle_ai.transcribe_enabled` + ASR +config; metadata requires `metadata_ai.enabled` + chat config. + +## Metadata translation + +### What gets translated + +| Target | Fields | +| --- | --- | +| movie / series (`media_item_localizations`) | `overview`, `tagline` | +| season (`season_localizations`) | `overview` | +| episode (`episode_localizations`) | `overview` | + +Titles, sort titles, and artwork columns in those rows are never written by AI. + +### Provenance + +New columns (default `'provider'`, check-constrained to `provider|ai|manual`): + +- `media_item_localizations.overview_source`, `media_item_localizations.tagline_source` +- `season_localizations.overview_source` +- `episode_localizations.overview_source` + +Write rules, enforced in the localization repos (single-statement upserts with CASE): + +| Writer | May overwrite | Sets source | Notes | +| --- | --- | --- | --- | +| provider ingestion | `provider`, `ai` | `provider` | a non-empty provider value always replaces an AI value; an **empty** provider value never blanks an existing one | +| AI translation | `ai`, empty | `ai` | with `force`, may also overwrite `provider` (admin explicitly re-translating) — never `manual` | +| manual (future editor) | anything | `manual` | | + +Serving hardening: `LocalizeItemModel` / `LocalizeSeasonModel` / `LocalizeEpisodeModel` +must only override base fields with **non-empty** localization values, since AI-created +rows legitimately carry empty titles. (Verify current behavior; fix if it clobbers.) + +### Job model + +New table `metadata_translation_jobs`, mirroring `subtitle_ai_jobs` conventions +(status enum, progress, idempotency partial-unique index, heartbeat): + +- `target_kind` (`item` | `season` | `episode`), `content_id`, `include_children` +- `target_language`, `source_language` (default language of the base row) +- `engine`, `model`, `status`, `progress`, `progress_message`, `fields_done`, `fields_total` +- `error_message`, `idempotency_key`, `requested_by`, `force`, timestamps, `heartbeat_at` + +A job for a series item with `include_children` expands to: item overview + tagline, every +season overview, every episode overview. All fields become segments +(`item:overview`, `season::overview`, `episode::overview`, …) fed +to the generic translator in batches (metadata batch size constant, smaller than the +subtitle one — paragraphs, not cue lines), so episode descriptions share terminology in one +prompt context. The system prompt names the series/movie and year for grounding. + +Skip logic in the run loop (not at enqueue): a field whose localization value is already +non-empty is skipped unless `force` — so the auto trigger costs zero LLM calls on repeat +refreshes. If every field is skipped the job completes immediately. + +Idempotency key: `content_id | target_kind | target_language | model`, active-only partial +unique index (completed/failed rows never block a retry). + +### Triggers + +1. **Manual.** Admin action on the item detail metadata editor: pick target language + (defaults to the library's `metadata_language`), `include_children`, `force`. Gated by + the same permission as metadata editing. +2. **Auto fallback.** New `media_folders.auto_translate_metadata boolean NOT NULL DEFAULT + false`. At the end of a metadata refresh for an item (after provider localizations are + persisted in `internal/metadata/service.go`), if the folder has the flag, has a + `metadata_language`, that language differs from the item's default metadata language, + and the localization row for it is missing a translated field — enqueue a non-force job + for the item (children included for series). Enqueue is fire-and-forget; refresh latency + is unaffected. + +### API + +Admin-gated, mirroring the subtitle AI handler shapes: + +- `GET /api/v1/metadata/ai/status` +- `POST /api/v1/metadata/ai/translate` `{content_id, target_kind, target_language, include_children, force}` +- `GET /api/v1/metadata/ai/jobs/{job_id}` / `POST …/cancel` +- `GET /api/v1/metadata/ai/jobs?content_id=…` + +Progress UI polls the job endpoint (jobs are short; no websocket events in v1). + +## Whisper ASR + +### Kinds + +`subtitle_ai_jobs.kind` gains its reserved values: + +- `transcribe` — audio track → subtitle track in the spoken language. +- `transcribe_translate` — `transcribe`, then the existing LLM translation chained to + `target_language`. Both the transcript track and the translated track are stored (the + transcript is a cache: a second target language skips ASR entirely because the transcript + is now a selectable text source for a plain `translate` job). `result_subtitle_id` + points at the translated track. + +For ASR kinds, `source_index` holds the **audio** track index (`-1` = default/first audio +track); `source_language` is an optional hint passed to Whisper, else taken from the +detected `verbose_json.language`. + +### Pipeline + +1. Resolve the audio track from `media_files`; reject files with no audio. +2. Extract once with ffmpeg into fixed chunks in a job temp dir + (`-vn -map 0:a: -ac 1 -ar 16000 -c:a pcm_s16le -f segment -segment_time 600`): + 16 kHz mono WAV, 10-minute chunks (~19 MB, under typical 25 MB API limits). Temp dir + removed on every exit path. +3. Per chunk: `POST /v1/audio/transcriptions` (multipart: file, model, + `response_format=verbose_json`, optional `language`, `temperature=0`) via the shared + `llm.Client` with the shared retry/backoff. Segment timestamps are offset by the chunk + start and merged. +4. Cue building: one cue per Whisper segment; text wrapped to at most two lines + (~42 chars/line). Empty/whitespace segments dropped. +5. Store as `downloaded_subtitles` (provider `transcribed`, SRT, release name + `" (AI transcribed)"`), notify `SubtitleReady`. +6. `transcribe_translate`: feed the cues to the existing `Translator`; store the translated + track (provider `translated`) and complete. + +Chunks are processed starting from the chunk containing the viewer's playhead, then +wrapping — same UX as translation — and cues stream live to the requesting session through +the existing notifier. Progress bands: extract 0–10 %, transcribe 10–70 % (per chunk), +translate 70–95 %. + +Known v1 limitation: fixed chunk boundaries can clip a word at the seam; acceptable, noted +for a silence-aligned follow-up. + +### API / UI + +The existing enqueue endpoint `POST /api/v1/subtitles/ai/translate` gains an optional +`kind` (default `translate`; backward compatible). `GET /api/v1/subtitles/ai/status` gains +`transcribe_enabled`. The player's translate modal gains a "Generate from audio" mode that +lists audio tracks and a target language — the natural path when the only subtitle tracks +are bitmap (today's hard error) or none exist. + +## Frontend + +- **AI Services settings page** (`web/src/pages/admin-settings/`): connection card + (base URL, API key, chat model, ASR model, optional ASR base URL/key), features card + (subtitle translation, ASR, metadata translation toggles; concurrency; batch tuning). + The AI card currently inside `SubtitlesSettings.tsx` moves here; subtitle settings keep + the download-provider config. +- **Metadata editor**: "Translate with AI" action in `EditMetadataDialog.tsx` (language, + include children, force), with job-poll progress and query invalidation on completion. +- **Library settings**: `auto_translate_metadata` toggle next to the metadata-language + field. +- **Player**: "Generate from audio" mode in `SubtitleTranslateModal.tsx`. + +## Client (Android / Apple) impact + +None required. Translated overviews/taglines arrive through the existing localized detail +responses; ASR/translated tracks arrive as ordinary downloaded subtitles. Optional +follow-ups in the client repos: surface "Generate from audio" in their subtitle menus +(the jobs API is shared) and the library auto-translate toggle in their admin surfaces. + +## Reliability + +- All jobs: idempotency dedup of in-flight work, heartbeat + stale reaper (crash-safe, + multi-instance safe), terminal-state handling on cancel/shutdown — one implementation in + `jobrunner`, no copy-paste between the two services. +- Provider-wins/manual-wins provenance is enforced in SQL (single-statement upserts), not + in racy read-modify-write Go code. +- The shared semaphore bounds total endpoint load; ASR temp space is cleaned on success, + failure, cancel, and shutdown. +- Auto-translate re-runs are free: skip-if-filled happens before any LLM call. + +## Migrations + +One Goose migration (`make migrate-create NAME=ai_metadata_translation_and_asr`): + +1. `metadata_translation_jobs` table + active-idempotency / content / status indexes. +2. Provenance columns on the three localization tables (+ check constraints). +3. `media_folders.auto_translate_metadata`. + +Down: drop the table, columns, and constraint. + +## Risks / open questions + +- **Whisper-endpoint compatibility.** OpenAI, Groq, faster-whisper-server/speaches all + speak `verbose_json`, but segment quality varies; the cue builder must tolerate missing + segment arrays (fall back to one cue per chunk worth of `text` is *not* acceptable — + fail the job with a clear message instead). +- **`LocalizeItemModel` empty-field semantics** must be verified before AI rows (empty + titles) ship; this is called out as an explicit plan task. +- **Long-running ASR on slow local Whisper servers** can exceed the 10-minute HTTP client + timeout per chunk; the transcription call uses a per-request timeout sized for chunk + duration rather than the client default. +- **Settings sprawl.** The `ai.*`/legacy fallback leaves old `subtitle_ai.*` connection + rows in place indefinitely; harmless, but a future settings-GC could prune them once the + new keys are written. diff --git a/docs/superpowers/specs/2026-06-10-on-view-description-translation-design.md b/docs/superpowers/specs/2026-06-10-on-view-description-translation-design.md new file mode 100644 index 00000000..31fc9381 --- /dev/null +++ b/docs/superpowers/specs/2026-06-10-on-view-description-translation-design.md @@ -0,0 +1,78 @@ +# On-view description translation + per-profile metadata language — design + +**Date:** 2026-06-10 +**Status:** Approved direction (per-profile preference; auto with button fallback; any user behind an opt-in setting) +**Builds on:** `docs/superpowers/specs/2026-06-10-ai-translation-and-asr-design.md` + +Commands assume the repository root is the cwd. + +## Problem + +Metadata translation currently triggers from the admin editor or the per-library +refresh fallback, always targeting the library's metadata language. A viewer whose +language differs from the library's (or whose library was never auto-translated) has +no way to get descriptions in their language, and there is no per-person language +preference at all — presentation language is library-scoped. + +## Design + +### Per-profile preferred metadata language + +- New column `user_profiles.preferred_metadata_language text NOT NULL DEFAULT ''` + (empty = inherit the library's metadata language). Threaded through the profile + model/repo/API exactly like the existing `language` (UI language) field, plus a + picker in the profile settings UI with an "Inherit library default" option. +- `access.Scope` gains `PreferredMetadataLanguage`, populated when the access + resolver loads the profile, so every request already carries it. +- `catalog.AccessFilter` gains `ProfilePreferredLanguage`, and presentation language + resolution becomes: explicit `PresentationLanguage` (request param) → + **profile preference** → library `metadata_language`. Applied wherever the scope + is available (native API and jellycompat), so all clients get profile-localized + metadata server-side. + +### "Not in your language" signal + +`ItemDetail` gains `pending_translation_language` (omitted when empty): set when the +resolved presentation language differs from the item's default metadata language, +the base overview is non-empty, and no localization overview exists for that +language. It is pure data — clients combine it with the AI feature status to decide +what to render. + +### On-view trigger + +- New setting `metadata_ai.on_view`: `off` (default) | `button` | `auto`, surfaced + on `GET /api/v1/metadata/ai/status` as `on_view`. +- New authenticated endpoint `POST /api/v1/items/{id}/translate-description` + `{target_language}` — any profile, gated by `metadata_ai` readiness + + `on_view != off`, item access enforced with the same `EnsureAccessible` check the + watched-state endpoints use. The client echoes back the language the server + reported in `pending_translation_language`. +- The endpoint goes through a new `Service.RequestOnView`, which adds one guard on + top of the existing pipeline: if the latest job for (content, language) failed + within a cooldown window, return it instead of enqueuing — a broken endpoint must + not be retried on every page view. Existing guarantees do the rest: in-flight + idempotency collapses concurrent viewers onto one job, skip-if-filled makes + repeats free, and the shared semaphore caps endpoint load. + +### Web UX + +On the detail page, when `pending_translation_language` is set: + +- `on_view=auto`: fire the request on view (once per item), pulse-animate the + description while waiting, and poll the detail query until the flag clears (the + job's first batch translates the item's own overview, so this lands in seconds + even when a series job continues episodes in the background). Time out after ~45 s + and show the original text — no error states in the viewer's face. +- `on_view=button`: a small "Translate" chip near the description triggers the same + flow on click. + +No job-status endpoint for viewers: completion is observed as the flag clearing on +refetch; failures surface only as the shimmer timing out (the cooldown stops +re-triggering). + +## Non-goals + +- Per-profile language in the Android/Apple UIs (server-side serving means they + already *receive* localized metadata; surfacing the trigger there is a client + follow-up using the same endpoint). +- Translating titles, or any admin-side behavior changes. diff --git a/internal/access/resolver.go b/internal/access/resolver.go index 3da3d22f..d853ca7f 100644 --- a/internal/access/resolver.go +++ b/internal/access/resolver.go @@ -73,6 +73,7 @@ func (r *Resolver) Resolve(ctx context.Context, input ResolveInput) (Scope, erro scope.MaxContentRating = profile.MaxContentRating scope.MaxPlaybackQuality = MinQuality(scope.MaxPlaybackQuality, NormalizePlaybackQuality(profile.MaxPlaybackQuality)) + scope.PreferredMetadataLanguage = profile.PreferredMetadataLanguage scope.AllowedLibraryIDs, scope.LibrariesRestricted = effectiveLibraries(user.LibraryIDs, profile) scope.ProfileVerified = profile.PINHash == "" || input.SkipPINVerification diff --git a/internal/access/types.go b/internal/access/types.go index 8b25f259..3a34ffb4 100644 --- a/internal/access/types.go +++ b/internal/access/types.go @@ -9,8 +9,11 @@ type Scope struct { LibrariesRestricted bool MaxContentRating string MaxPlaybackQuality string - PolicyRevision int64 - ProfileVerified bool + // PreferredMetadataLanguage is the profile's metadata (presentation) + // language; "" inherits the library's metadata language. + PreferredMetadataLanguage string + PolicyRevision int64 + ProfileVerified bool } // ResolveInput is the request input for resolving a viewer access scope. diff --git a/internal/ai/jobrunner/runner.go b/internal/ai/jobrunner/runner.go new file mode 100644 index 00000000..172c62ee --- /dev/null +++ b/internal/ai/jobrunner/runner.go @@ -0,0 +1,197 @@ +// Package jobrunner owns the lifecycle mechanics shared by Silo's AI job +// services (subtitle translation/ASR, metadata translation): bounded dispatch +// off a semaphore shared across services, a heartbeat loop that keeps a row +// alive even while the job is queued, a stale-job reaper for rows orphaned by +// a crashed worker, and a per-job cancel registry. The services keep their own +// job tables, validation, and run logic; this package keeps them honest about +// concurrency and crash recovery without copy-pasting the trickiest code. +package jobrunner + +import ( + "context" + "log/slog" + "sync" + "time" +) + +const ( + // A running job refreshes its heartbeat every HeartbeatInterval; one whose + // heartbeat has not advanced for StaleJobThreshold is treated as orphaned + // by a crashed worker and reaped. The margin over HeartbeatInterval avoids + // reaping a job that is merely mid–LLM-call. + HeartbeatInterval = 30 * time.Second + StaleJobThreshold = 2 * time.Minute + // How often the background reaper scans for orphaned jobs. + ReaperInterval = time.Minute +) + +// Store is the minimal persistence surface the runner needs. Both AI job +// repositories satisfy it. +type Store interface { + Heartbeat(ctx context.Context, id int64) error + // ResetStaleJobs marks pending/running jobs whose heartbeat predates + // `before` as failed with the given message. Returns rows reset. + ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error) +} + +// NewSemaphore builds the dispatch semaphore shared across runners, so the +// configured endpoint sees one global bound regardless of job mix. size <= 0 +// falls back to 2. +func NewSemaphore(size int) chan struct{} { + if size <= 0 { + size = 2 + } + return make(chan struct{}, size) +} + +// Runner executes jobs with bounded concurrency, heartbeats, cancellation, +// and crash recovery. One Runner per job table; the semaphore may be shared +// across Runners. +type Runner struct { + // baseCtx is the application context; dispatched jobs and the reaper + // derive from it so they stop when the server shuts down. + baseCtx context.Context + sem chan struct{} + store Store + // label names the job family in log lines ("subtitle ai", "metadata translation"). + label string + logger *slog.Logger + + mu sync.Mutex + cancels map[int64]context.CancelFunc + wg sync.WaitGroup +} + +// New wires a runner. A nil appCtx falls back to context.Background(); a nil +// sem gets a private default-size semaphore (used by tests; production passes +// the shared one). +func New(appCtx context.Context, sem chan struct{}, store Store, label string, logger *slog.Logger) *Runner { + if appCtx == nil { + appCtx = context.Background() + } + if sem == nil { + sem = NewSemaphore(0) + } + if logger == nil { + logger = slog.Default() + } + return &Runner{ + baseCtx: appCtx, + sem: sem, + store: store, + label: label, + logger: logger, + cancels: make(map[int64]context.CancelFunc), + } +} + +// Recover clears jobs orphaned by a crashed worker and starts a background +// reaper that keeps doing so. Reaping is heartbeat-based (not "every active +// job"), so it is safe when multiple instances share one database: a job +// still being heartbeat-updated by a live worker is never reset. Call once at +// startup; jobs and the reaper derive from the application context, so they +// stop on shutdown. +func (r *Runner) Recover() { + r.reapStaleJobs() + go r.reaperLoop() +} + +func (r *Runner) reaperLoop() { + ticker := time.NewTicker(ReaperInterval) + defer ticker.Stop() + for { + select { + case <-r.baseCtx.Done(): + return + case <-ticker.C: + r.reapStaleJobs() + } + } +} + +func (r *Runner) reapStaleJobs() { + before := time.Now().Add(-StaleJobThreshold) + n, err := r.store.ResetStaleJobs(context.WithoutCancel(r.baseCtx), before, "interrupted by server restart") + if err != nil { + r.logger.Warn("failed to reset stale jobs", "jobs", r.label, "error", err) + return + } + if n > 0 { + r.logger.Info("reset stale jobs", "jobs", r.label, "count", n) + } +} + +// Dispatch launches a bounded background goroutine for job id. run executes +// once a semaphore slot is acquired; onAbort runs instead if the job is +// cancelled (user cancel or shutdown) while still waiting for a slot. Both +// receive a context derived from the application context that is cancelled by +// Cancel(id) or server shutdown. +func (r *Runner) Dispatch(id int64, run func(ctx context.Context), onAbort func(ctx context.Context)) { + runCtx, cancel := context.WithCancel(r.baseCtx) + r.mu.Lock() + r.cancels[id] = cancel + r.mu.Unlock() + + r.wg.Add(1) + go func() { + defer r.wg.Done() + defer func() { + r.mu.Lock() + delete(r.cancels, id) + r.mu.Unlock() + cancel() + }() + + // Heartbeat for the whole lifetime — crucially including while queued + // behind the semaphore — so the stale-job reaper never reaps a job that + // is alive but merely waiting for a slot (which would otherwise mark it + // failed and let it resurrect itself on acquire, or admit a duplicate). + stopHeartbeat := make(chan struct{}) + defer close(stopHeartbeat) + go r.heartbeatLoop(runCtx, id, stopHeartbeat) + + select { + case r.sem <- struct{}{}: + case <-runCtx.Done(): + if onAbort != nil { + onAbort(context.WithoutCancel(runCtx)) + } + return + } + defer func() { <-r.sem }() + + run(runCtx) + }() +} + +// Cancel cancels the in-flight goroutine for id, returning false when none is +// registered (job on another node, or never dispatched) so the caller can +// fall back to a best-effort terminal transition in the database. +func (r *Runner) Cancel(id int64) bool { + r.mu.Lock() + cancel := r.cancels[id] + r.mu.Unlock() + if cancel == nil { + return false + } + cancel() + return true +} + +// heartbeatLoop keeps a job's heartbeat_at fresh until the job ends or the +// context is cancelled, so the stale-job reaper only ever reaps jobs orphaned +// by a crashed worker. +func (r *Runner) heartbeatLoop(ctx context.Context, jobID int64, stop <-chan struct{}) { + ticker := time.NewTicker(HeartbeatInterval) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ctx.Done(): + return + case <-ticker.C: + _ = r.store.Heartbeat(context.WithoutCancel(ctx), jobID) + } + } +} diff --git a/internal/ai/jobrunner/runner_test.go b/internal/ai/jobrunner/runner_test.go new file mode 100644 index 00000000..7b7f9182 --- /dev/null +++ b/internal/ai/jobrunner/runner_test.go @@ -0,0 +1,158 @@ +package jobrunner + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" +) + +type fakeStore struct { + mu sync.Mutex + heartbeats int + resets int + lastBefore time.Time +} + +func (s *fakeStore) Heartbeat(context.Context, int64) error { + s.mu.Lock() + defer s.mu.Unlock() + s.heartbeats++ + return nil +} + +func (s *fakeStore) ResetStaleJobs(_ context.Context, before time.Time, _ string) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.resets++ + s.lastBefore = before + return 0, nil +} + +func (s *fakeStore) snapshot() (int, time.Time) { + s.mu.Lock() + defer s.mu.Unlock() + return s.resets, s.lastBefore +} + +func TestRecoverReapsImmediatelyWithStaleCutoff(t *testing.T) { + store := &fakeStore{} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + approxNow := time.Now() + New(ctx, nil, store, "test", nil).Recover() + + resets, before := store.snapshot() + if resets < 1 { + t.Fatalf("Recover did not reap immediately: resets=%d", resets) + } + want := approxNow.Add(-StaleJobThreshold) + if diff := before.Sub(want); diff > 2*time.Second || diff < -2*time.Second { + t.Errorf("stale cutoff = %v, want ~%v", before, want) + } +} + +// One shared semaphore bounds jobs across BOTH runners: with size 1, the +// second runner's job must not start until the first runner's job finishes. +func TestSharedSemaphoreBoundsAcrossRunners(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sem := NewSemaphore(1) + r1 := New(ctx, sem, &fakeStore{}, "one", nil) + r2 := New(ctx, sem, &fakeStore{}, "two", nil) + + release := make(chan struct{}) + aStarted := make(chan struct{}) + var bStarted atomic.Bool + bDone := make(chan struct{}) + + r1.Dispatch(1, func(context.Context) { + close(aStarted) + <-release + }, nil) + <-aStarted + + r2.Dispatch(2, func(context.Context) { + bStarted.Store(true) + close(bDone) + }, nil) + + time.Sleep(50 * time.Millisecond) + if bStarted.Load() { + t.Fatal("job B ran while job A held the only slot") + } + close(release) + select { + case <-bDone: + case <-time.After(2 * time.Second): + t.Fatal("job B never ran after the slot freed") + } +} + +// Cancelling a job that is queued behind the semaphore aborts it without +// running it, via the onAbort callback. +func TestCancelWhileQueuedCallsOnAbort(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sem := NewSemaphore(1) + r := New(ctx, sem, &fakeStore{}, "test", nil) + + release := make(chan struct{}) + defer close(release) + started := make(chan struct{}) + r.Dispatch(1, func(context.Context) { + close(started) + <-release + }, nil) + <-started + + var ran atomic.Bool + aborted := make(chan struct{}) + r.Dispatch(2, func(context.Context) { + ran.Store(true) + }, func(context.Context) { + close(aborted) + }) + + if !r.Cancel(2) { + t.Fatal("Cancel(2) found no in-flight goroutine") + } + select { + case <-aborted: + case <-time.After(2 * time.Second): + t.Fatal("onAbort never ran") + } + if ran.Load() { + t.Fatal("cancelled queued job still ran") + } +} + +func TestCancelUnknownJobReturnsFalse(t *testing.T) { + r := New(context.Background(), nil, &fakeStore{}, "test", nil) + if r.Cancel(99) { + t.Fatal("Cancel(99) = true for unknown job") + } +} + +// The run context is cancelled by Cancel(id) so an in-flight job can stop. +func TestCancelRunningJobCancelsContext(t *testing.T) { + r := New(context.Background(), NewSemaphore(1), &fakeStore{}, "test", nil) + started := make(chan struct{}) + stopped := make(chan struct{}) + r.Dispatch(7, func(ctx context.Context) { + close(started) + <-ctx.Done() + close(stopped) + }, nil) + <-started + if !r.Cancel(7) { + t.Fatal("Cancel(7) found no in-flight goroutine") + } + select { + case <-stopped: + case <-time.After(2 * time.Second): + t.Fatal("running job did not observe cancellation") + } +} diff --git a/internal/ai/jobrunner/status.go b/internal/ai/jobrunner/status.go new file mode 100644 index 00000000..5262809f --- /dev/null +++ b/internal/ai/jobrunner/status.go @@ -0,0 +1,22 @@ +package jobrunner + +// Status is the lifecycle state shared by every AI job table. +type Status string + +const ( + StatusPending Status = "pending" + StatusRunning Status = "running" + StatusCompleted Status = "completed" + StatusFailed Status = "failed" + StatusCancelled Status = "cancelled" +) + +// Terminal reports whether a status is final. +func (s Status) Terminal() bool { + switch s { + case StatusCompleted, StatusFailed, StatusCancelled: + return true + default: + return false + } +} diff --git a/internal/ai/llm/client.go b/internal/ai/llm/client.go new file mode 100644 index 00000000..41a8ef3a --- /dev/null +++ b/internal/ai/llm/client.go @@ -0,0 +1,297 @@ +// Package llm provides the shared OpenAI-compatible API client used by every +// AI feature in Silo (subtitle translation, metadata translation, Whisper ASR). +// One endpoint configuration, one retry/backoff implementation; the operator +// can point it at OpenAI, Groq, a local Ollama/llama.cpp/faster-whisper +// server, etc. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "time" +) + +// Config holds the connection settings for the shared OpenAI-compatible +// endpoints. The ASR fields are optional overrides for operators who run a +// separate Whisper-compatible server next to their chat endpoint; when empty, +// transcription uses the chat endpoint's base URL and key. +type Config struct { + BaseURL string // e.g. "https://api.openai.com" (no trailing /v1) + APIKey string // empty for keyless local servers + ChatModel string // chat-completions model used for translation + + ASRBaseURL string // optional; empty = BaseURL + ASRAPIKey string // optional; empty = APIKey + ASRModel string // audio-transcription model, e.g. "whisper-1" +} + +func (c Config) asrBaseURL() string { + if c.ASRBaseURL != "" { + return c.ASRBaseURL + } + return c.BaseURL +} + +func (c Config) asrAPIKey() string { + if c.ASRAPIKey != "" { + return c.ASRAPIKey + } + return c.APIKey +} + +// ChatConfigured reports whether chat completions are minimally configured. +func (c Config) ChatConfigured() bool { return c.BaseURL != "" && c.ChatModel != "" } + +// ASRConfigured reports whether audio transcription is minimally configured. +func (c Config) ASRConfigured() bool { return c.asrBaseURL() != "" && c.ASRModel != "" } + +// Client is a minimal OpenAI-compatible API client with shared retry/backoff +// conventions (429 with Retry-After, 5xx, transport errors, and gateway "200 +// with embedded error object" responses are retried; other 4xx fail fast). +type Client struct { + cfg Config + // chatHTTP caps a single chat completion at 10 minutes. asrHTTP has no + // client-level timeout: a transcription upload's deadline is set per request + // (sized to the chunk duration), which a client timeout would silently cap. + chatHTTP *http.Client + asrHTTP *http.Client +} + +// NewClient builds a client from the endpoint config. +func NewClient(cfg Config) *Client { + return &Client{ + cfg: cfg, + chatHTTP: &http.Client{Timeout: 10 * time.Minute}, + asrHTTP: &http.Client{}, + } +} + +// Message is one chat-completions message. +type Message struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type chatResponseFormat struct { + Type string `json:"type"` +} + +type chatCompletionRequest struct { + Model string `json:"model"` + Messages []Message `json:"messages"` + Temperature float32 `json:"temperature"` + ResponseFormat *chatResponseFormat `json:"response_format,omitempty"` +} + +type chatCompletionResponse struct { + Choices []struct { + Message Message `json:"message"` + } `json:"choices"` + // Some OpenAI-compatible gateways (e.g. OpenRouter) return a 200 with an + // error object instead of an HTTP error status when an upstream provider + // fails. We surface and retry on it. + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +// Chat performs one chat completion and returns the first choice's content. +// When jsonObject is true it requests response_format=json_object; providers +// that ignore the field still work because the prompt itself demands JSON. +func (c *Client) Chat(ctx context.Context, messages []Message, jsonObject bool) (string, error) { + reqBody := chatCompletionRequest{ + Model: c.cfg.ChatModel, + Messages: messages, + Temperature: 0.2, + } + if jsonObject { + reqBody.ResponseFormat = &chatResponseFormat{Type: "json_object"} + } + + body, err := json.Marshal(reqBody) + if err != nil { + return "", fmt.Errorf("marshal chat request: %w", err) + } + + url := endpointURL(c.cfg.BaseURL, "chat/completions") + + var content string + err = c.doWithRetry(ctx, c.chatHTTP, "chat API", + func() (*http.Request, error) { + httpReq, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if reqErr != nil { + return nil, fmt.Errorf("create request: %w", reqErr) + } + httpReq.Header.Set("Content-Type", "application/json") + if c.cfg.APIKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) + } + return httpReq, nil + }, + func(respBody []byte) error { + var parsed chatCompletionResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + return fmt.Errorf("decode chat response: %w", err) + } + if parsed.Error != nil && parsed.Error.Message != "" { + // 200 with an upstream error object — transient on gateways. + return fmt.Errorf("chat API error: %s", parsed.Error.Message) + } + if len(parsed.Choices) == 0 || parsed.Choices[0].Message.Content == "" { + slog.Warn("AI chat returned no choices, retrying", + "model", c.cfg.ChatModel, "response_bytes", len(respBody)) + return fmt.Errorf("chat API returned no choices") + } + content = parsed.Choices[0].Message.Content + return nil + }) + if err != nil { + return "", err + } + return content, nil +} + +// SystemUserChat performs one chat completion from a system + user prompt +// pair, requesting a JSON object response. Its signature matches +// aitranslate.ChatFn so a client method reference wires straight in. +func (c *Client) SystemUserChat(ctx context.Context, system, user string) (string, error) { + return c.Chat(ctx, []Message{ + {Role: "system", Content: system}, + {Role: "user", Content: user}, + }, true) +} + +// permanentError marks a parse failure that retrying cannot fix (e.g. an +// endpoint that structurally does not support the requested response format). +type permanentError struct{ err error } + +func (e *permanentError) Error() string { return e.err.Error() } +func (e *permanentError) Unwrap() error { return e.err } + +// doWithRetry runs the shared request/retry loop: build constructs a fresh +// request per attempt, parse consumes a 200 body. Transport errors, read +// errors, 429 (honoring Retry-After), 5xx, and retryable parse errors back +// off and retry; other 4xx and permanentError parse failures return at once. +func (c *Client) doWithRetry(ctx context.Context, httpClient *http.Client, label string, + build func() (*http.Request, error), parse func(body []byte) error) error { + const maxAttempts = 6 + var lastErr error + for attempt := 0; attempt < maxAttempts; attempt++ { + httpReq, err := build() + if err != nil { + return err + } + + resp, doErr := httpClient.Do(httpReq) + if doErr != nil { + lastErr = fmt.Errorf("%s request failed: %w", label, doErr) + if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { + return waitErr + } + continue + } + + respBody, readErr := io.ReadAll(resp.Body) + resp.Body.Close() + if readErr != nil { + // A truncated/failed read could otherwise be misparsed as a valid + // (empty) response; treat it as a retryable transport error. + lastErr = fmt.Errorf("read %s response: %w", label, readErr) + if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { + return waitErr + } + continue + } + + switch { + case resp.StatusCode == http.StatusTooManyRequests: + wait := rateLimitBackoff(resp, attempt) + slog.Warn("rate limited by AI API, waiting", "api", label, "attempt", attempt+1, "wait", wait) + lastErr = fmt.Errorf("%s returned 429: %s", label, Truncate(string(respBody), 300)) + if waitErr := sleepCtx(ctx, wait); waitErr != nil { + return waitErr + } + continue + case resp.StatusCode >= 500: + lastErr = fmt.Errorf("%s returned %d: %s", label, resp.StatusCode, Truncate(string(respBody), 300)) + if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { + return waitErr + } + continue + case resp.StatusCode != http.StatusOK: + // 4xx other than 429: not retryable. + return fmt.Errorf("%s returned %d: %s", label, resp.StatusCode, Truncate(string(respBody), 300)) + } + + parseErr := parse(respBody) + if parseErr == nil { + return nil + } + var perm *permanentError + if errors.As(parseErr, &perm) { + return perm.err + } + lastErr = parseErr + if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { + return waitErr + } + } + + if lastErr == nil { + lastErr = fmt.Errorf("%s: retries exhausted", label) + } + return lastErr +} + +// endpointURL joins a configured base URL with an OpenAI API path, +// tolerating bases that already include the version segment (e.g. +// DeepInfra's https://api.deepinfra.com/v1/openai) alongside bare hosts +// (https://api.openai.com) and prefixed hosts (https://api.groq.com/openai). +func endpointURL(base, path string) string { + base = strings.TrimRight(base, "/") + if strings.Contains(base, "/v1") { + return base + "/" + path + } + return base + "/v1/" + path +} + +// Truncate caps a string for inclusion in an error or log line. +func Truncate(s string, maxLen int) string { + if len(s) > maxLen { + return s[:maxLen] + "..." + } + return s +} + +// sleepCtx waits for d or until ctx is cancelled. +func sleepCtx(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + return nil + } +} + +// rateLimitBackoff returns how long to wait after a 429, honoring Retry-After +// and otherwise backing off exponentially (capped at 60s). +func rateLimitBackoff(resp *http.Response, attempt int) time.Duration { + if ra := resp.Header.Get("Retry-After"); ra != "" { + if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { + return time.Duration(secs) * time.Second + } + } + wait := 10 * time.Second * (1 << attempt) + if wait > 60*time.Second { + wait = 60 * time.Second + } + return wait +} diff --git a/internal/ai/llm/client_test.go b/internal/ai/llm/client_test.go new file mode 100644 index 00000000..2273f9d3 --- /dev/null +++ b/internal/ai/llm/client_test.go @@ -0,0 +1,350 @@ +package llm + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" +) + +func chatConfig(baseURL string) Config { + return Config{BaseURL: baseURL, APIKey: "test-key", ChatModel: "test-model"} +} + +const chatOK = `{"choices":[{"message":{"role":"assistant","content":"hello"}}]}` + +func TestChatSuccessSendsAuthAndModel(t *testing.T) { + var gotAuth, gotBody string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + buf := make([]byte, 4096) + n, _ := r.Body.Read(buf) + gotBody = string(buf[:n]) + w.Write([]byte(chatOK)) + })) + defer srv.Close() + + c := NewClient(chatConfig(srv.URL)) + out, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, true) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if out != "hello" { + t.Errorf("content = %q, want hello", out) + } + if gotAuth != "Bearer test-key" { + t.Errorf("auth = %q", gotAuth) + } + if !strings.Contains(gotBody, `"model":"test-model"`) || !strings.Contains(gotBody, `"json_object"`) { + t.Errorf("request body missing model/response_format: %s", gotBody) + } +} + +func TestChatRetriesOn429HonoringRetryAfter(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls.Add(1) == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.Write([]byte(chatOK)) + })) + defer srv.Close() + + start := time.Now() + c := NewClient(chatConfig(srv.URL)) + if _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false); err != nil { + t.Fatalf("Chat: %v", err) + } + if calls.Load() != 2 { + t.Errorf("calls = %d, want 2", calls.Load()) + } + if elapsed := time.Since(start); elapsed < time.Second { + t.Errorf("did not honor Retry-After: elapsed %v", elapsed) + } +} + +func TestChatRetriesOn5xxAndEmbeddedErrorAndEmptyChoices(t *testing.T) { + responses := []func(w http.ResponseWriter){ + func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadGateway) }, + func(w http.ResponseWriter) { w.Write([]byte(`{"error":{"message":"upstream sad"}}`)) }, + func(w http.ResponseWriter) { w.Write([]byte(`{"choices":[]}`)) }, + func(w http.ResponseWriter) { w.Write([]byte(chatOK)) }, + } + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + responses[calls.Add(1)-1](w) + })) + defer srv.Close() + + c := NewClient(chatConfig(srv.URL)) + out, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false) + if err != nil { + t.Fatalf("Chat: %v", err) + } + if out != "hello" || calls.Load() != 4 { + t.Errorf("out=%q calls=%d, want hello/4", out, calls.Load()) + } +} + +func TestChatFailsFastOnNon429ClientError(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusUnauthorized) + })) + defer srv.Close() + + c := NewClient(chatConfig(srv.URL)) + if _, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, false); err == nil { + t.Fatal("expected error") + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (no retry on 401)", calls.Load()) + } +} + +const verboseJSON = `{"language":"english","text":"hi there","segments":[{"start":0.0,"end":1.5,"text":" hi"},{"start":1.5,"end":3.0,"text":" there"}]}` + +func TestTranscribeParsesSegmentsAndMultipart(t *testing.T) { + var gotModel, gotFormat, gotLang, gotAuth, gotVAD string + var gotGranularities []string + var gotFile []byte + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + if err := r.ParseMultipartForm(32 << 20); err != nil { + t.Errorf("parse multipart: %v", err) + } + gotModel = r.FormValue("model") + gotFormat = r.FormValue("response_format") + gotLang = r.FormValue("language") + gotVAD = r.FormValue("vad_filter") + gotGranularities = r.MultipartForm.Value["timestamp_granularities[]"] + f, _, err := r.FormFile("file") + if err == nil { + buf := make([]byte, 64) + n, _ := f.Read(buf) + gotFile = buf[:n] + f.Close() + } + w.Write([]byte(verboseJSON)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + c := NewClient(cfg) + tr, err := c.Transcribe(context.Background(), TranscribeRequest{ + Filename: "chunk.wav", Audio: []byte("RIFFfake"), Language: "ja", + }) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if gotModel != "whisper-test" || gotFormat != "verbose_json" || gotLang != "ja" { + t.Errorf("fields model=%q format=%q lang=%q", gotModel, gotFormat, gotLang) + } + // Local (non-hosted) endpoint: faster-whisper VAD must be requested or + // segment timestamps stretch wall-to-wall across silence. + if gotVAD != "true" { + t.Errorf("vad_filter = %q, want true for a self-hosted endpoint", gotVAD) + } + if fmt.Sprint(gotGranularities) != "[segment word]" { + t.Errorf("timestamp_granularities[] = %v, want [segment word]", gotGranularities) + } + if gotAuth != "Bearer test-key" { + t.Errorf("auth = %q (should fall back to chat key)", gotAuth) + } + if string(gotFile) != "RIFFfake" { + t.Errorf("file payload = %q", gotFile) + } + if tr.Language != "english" || len(tr.Segments) != 2 || tr.Segments[1].Text != " there" || tr.Segments[1].End != 3.0 { + t.Errorf("unexpected transcription: %+v", tr) + } +} + +func TestTranscribeOmitsVADFilterForStrictHostedEndpoints(t *testing.T) { + for url, wantVAD := range map[string]bool{ + "https://api.openai.com": false, + "https://myorg.openai.azure.com": false, + "https://api.groq.com/openai": false, + "http://192.168.1.10:8000": true, + "https://whisper.example.com": true, + "https://api.deepinfra.com/v1/openai": true, + } { + got := !hostMatchesAny(url, strictHostedASRHosts) + if got != wantVAD { + t.Errorf("vad_filter for %s = %v, want %v", url, got, wantVAD) + } + } +} + +func TestTranscribeParsesPerSegmentWords(t *testing.T) { + // speaches/faster-whisper shape: words nested inside each segment. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"language":"en","text":"hi there","segments":[ + {"start":0.0,"end":3.0,"text":" hi there","words":[ + {"start":0.2,"end":0.5,"word":" hi"},{"start":0.6,"end":1.0,"word":" there"}]}]}`)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + words := tr.Segments[0].Words + if len(words) != 2 || words[1].Text != " there" || words[1].Start != 0.6 || words[1].End != 1.0 { + t.Errorf("segment words = %+v", words) + } +} + +func TestTranscribeAttachesTopLevelWordsBySegmentTime(t *testing.T) { + // OpenAI shape: words in a top-level array, segments without words. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"language":"en","text":"hi there friend","segments":[ + {"start":0.0,"end":1.5,"text":" hi there"},{"start":1.5,"end":3.0,"text":" friend"}], + "words":[{"start":0.2,"end":0.5,"word":"hi"},{"start":0.6,"end":1.0,"word":"there"}, + {"start":1.8,"end":2.2,"word":"friend"}]}`)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if len(tr.Segments[0].Words) != 2 || len(tr.Segments[1].Words) != 1 { + t.Errorf("word distribution = %d/%d, want 2/1", len(tr.Segments[0].Words), len(tr.Segments[1].Words)) + } + if tr.Segments[1].Words[0].Text != "friend" { + t.Errorf("segment 1 word = %+v", tr.Segments[1].Words[0]) + } +} + +func TestTranscribeEmptySegmentsIsNotAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"language":"english","text":"","segments":[]}`)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + tr, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if len(tr.Segments) != 0 { + t.Errorf("segments = %v, want empty", tr.Segments) + } +} + +func TestTranscribeMissingSegmentsFieldFailsFast(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Write([]byte(`{"text":"plain response without segments"}`)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + _, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err == nil || !strings.Contains(err.Error(), "verbose_json") { + t.Fatalf("err = %v, want verbose_json complaint", err) + } + if calls.Load() != 1 { + t.Errorf("calls = %d, want 1 (permanent error must not retry)", calls.Load()) + } +} + +func TestTranscribeUsesASROverrides(t *testing.T) { + var gotAuth atomic.Value + asrSrv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth.Store(r.Header.Get("Authorization")) + w.Write([]byte(verboseJSON)) + })) + defer asrSrv.Close() + + cfg := Config{ + BaseURL: "http://chat.invalid", APIKey: "chat-key", ChatModel: "m", + ASRBaseURL: asrSrv.URL, ASRAPIKey: "asr-key", ASRModel: "whisper-test", + } + if _, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}); err != nil { + t.Fatalf("Transcribe: %v", err) + } + if gotAuth.Load() != "Bearer asr-key" { + t.Errorf("auth = %q, want asr-key", gotAuth.Load()) + } +} + +func TestTranscribeRequiresConfig(t *testing.T) { + c := NewClient(Config{BaseURL: "http://x", ChatModel: "m"}) // no ASR model + if _, err := c.Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}); err == nil { + t.Fatal("expected not-configured error") + } +} + +func TestTranscribeChatOnlyGatewayGetsConfigHint(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"error":{"message":"invalid content-type: multipart/form-data","code":400}}`)) + })) + defer srv.Close() + + cfg := chatConfig(srv.URL) + cfg.ASRModel = "whisper-test" + _, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err == nil || !strings.Contains(err.Error(), "Whisper-compatible Transcription base URL") { + t.Fatalf("err = %v, want configuration hint", err) + } +} + +func TestIsChatOnlyGateway(t *testing.T) { + cases := map[string]bool{ + "https://openrouter.ai/api": true, + "https://openrouter.ai": true, + "openrouter.ai/api/v1": true, + "https://gateway.openrouter.ai": true, + "https://api.groq.com/openai": false, + "https://api.openai.com": false, + "http://localhost:8000": false, + "https://my-openrouter.ai.example.io": false, + "": false, + } + for in, want := range cases { + if got := IsChatOnlyGateway(in); got != want { + t.Errorf("IsChatOnlyGateway(%q) = %v, want %v", in, got, want) + } + } +} + +func TestTranscribeRefusesChatOnlyGateway(t *testing.T) { + cfg := Config{BaseURL: "https://openrouter.ai/api", ChatModel: "m", ASRModel: "whisper-1"} + _, err := NewClient(cfg).Transcribe(context.Background(), TranscribeRequest{Filename: "c.wav", Audio: []byte("x")}) + if err == nil || !strings.Contains(err.Error(), "cannot produce timestamped transcriptions") { + t.Fatalf("err = %v, want chat-only refusal", err) + } +} + +func TestEndpointURLToleratesVersionedBases(t *testing.T) { + cases := map[string]string{ + "https://api.openai.com": "https://api.openai.com/v1/chat/completions", + "https://api.groq.com/openai": "https://api.groq.com/openai/v1/chat/completions", + "https://api.deepinfra.com/v1/openai": "https://api.deepinfra.com/v1/openai/chat/completions", + "http://localhost:8969": "http://localhost:8969/v1/chat/completions", + "http://localhost:8969/v1": "http://localhost:8969/v1/chat/completions", + "https://api.openai.com/": "https://api.openai.com/v1/chat/completions", + } + for base, want := range cases { + if got := endpointURL(base, "chat/completions"); got != want { + t.Errorf("endpointURL(%q) = %q, want %q", base, got, want) + } + } +} diff --git a/internal/ai/llm/transcribe.go b/internal/ai/llm/transcribe.go new file mode 100644 index 00000000..5d33fb55 --- /dev/null +++ b/internal/ai/llm/transcribe.go @@ -0,0 +1,286 @@ +package llm + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/url" + "strings" + "time" +) + +// defaultTranscribeTimeout bounds a single transcription request when the +// caller does not size one to its chunk duration. Generous on purpose: local +// Whisper servers on modest hardware can run well below realtime. +const defaultTranscribeTimeout = 20 * time.Minute + +// TranscribeRequest is one audio-transcription call. Audio is held in memory +// so the request can be rebuilt across retries; callers chunk long files +// (a 10-minute 16 kHz mono WAV is ~19 MB). +type TranscribeRequest struct { + Filename string // e.g. "chunk00001.wav"; the extension hints the container + Audio []byte + Language string // optional ISO-639-1 hint; empty lets the model detect + Timeout time.Duration // per-request deadline; 0 uses defaultTranscribeTimeout +} + +// TranscriptionSegment is one timed segment of recognized speech, in seconds +// relative to the start of the submitted audio. +type TranscriptionSegment struct { + Start float64 + End float64 + Text string + // Words carries per-word timings when the endpoint honors + // timestamp_granularities[]=word; empty otherwise. Word-level times let + // cue building split paragraph-length segments and end cues when speech + // actually stops instead of when the next segment begins. + Words []TranscriptionWord +} + +// TranscriptionWord is one recognized word with timing, in seconds relative +// to the start of the submitted audio. +type TranscriptionWord struct { + Start float64 + End float64 + Text string +} + +// Transcription is the parsed verbose_json transcription result. +type Transcription struct { + // Language is the detected (or hinted) language as reported by the + // endpoint. OpenAI returns an English language name ("english"); other + // servers return ISO codes. Callers must normalize. + Language string + // Segments may legitimately be empty for speech-free audio (silence, + // music-only chunks). + Segments []TranscriptionSegment +} + +type transcriptionWordJSON struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Word string `json:"word"` +} + +type transcriptionResponse struct { + Language string `json:"language"` + Text string `json:"text"` + // Segments distinguishes "verbose_json honored, no speech" (empty array) + // from "endpoint ignored verbose_json" (field absent) — the latter cannot + // produce timed cues and must fail rather than silently emit nothing. + Segments *[]struct { + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Words []transcriptionWordJSON `json:"words"` + } `json:"segments"` + // Words is where OpenAI puts word timings; faster-whisper servers + // (speaches) nest them inside each segment instead. + Words []transcriptionWordJSON `json:"words"` + Error *struct { + Message string `json:"message"` + } `json:"error"` +} + +// chatOnlyGatewayHosts lists OpenAI-compatible gateways that cannot serve +// Silo's transcription needs: they either lack /v1/audio/transcriptions +// entirely or (OpenRouter) return plain text with no segment timestamps, +// which subtitle cues require. Matched by host suffix. +var chatOnlyGatewayHosts = []string{ + "openrouter.ai", +} + +// strictHostedASRHosts lists hosted transcription providers that reject +// multipart fields outside the OpenAI spec, so the faster-whisper-only +// vad_filter field must be omitted for them. Nothing is lost: hosted Whisper +// runs voice-activity detection server-side. Self-hosted servers (speaches, +// faster-whisper) need the explicit field — without it segment timestamps +// stretch wall-to-wall across silence. Matched by host suffix. +var strictHostedASRHosts = []string{ + "api.openai.com", + "openai.azure.com", + "api.groq.com", + "api.mistral.ai", +} + +// IsChatOnlyGateway reports whether baseURL points at a known chat-only +// gateway that cannot produce timestamped transcriptions. Used to validate +// the transcription settings and to disable ASR rather than fail jobs at +// runtime. +func IsChatOnlyGateway(baseURL string) bool { + return hostMatchesAny(baseURL, chatOnlyGatewayHosts) +} + +// hostMatchesAny reports whether baseURL's hostname equals or is a subdomain +// of any of the given host suffixes. +func hostMatchesAny(baseURL string, hosts []string) bool { + baseURL = strings.TrimSpace(baseURL) + if baseURL == "" { + return false + } + if !strings.Contains(baseURL, "://") { + baseURL = "https://" + baseURL + } + u, err := url.Parse(baseURL) + if err != nil || u.Host == "" { + return false + } + host := strings.ToLower(u.Hostname()) + for _, h := range hosts { + if host == h || strings.HasSuffix(host, "."+h) { + return true + } + } + return false +} + +// Transcribe performs one audio transcription against the ASR endpoint +// (falling back to the chat endpoint's base URL/key when no ASR override is +// configured), using the OpenAI-compatible /v1/audio/transcriptions API with +// response_format=verbose_json for segment timestamps. +func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (*Transcription, error) { + if !c.cfg.ASRConfigured() { + return nil, fmt.Errorf("transcription endpoint is not configured") + } + if IsChatOnlyGateway(c.cfg.asrBaseURL()) { + return nil, fmt.Errorf("the configured transcription endpoint (%s) cannot produce timestamped transcriptions; "+ + "set a Whisper-compatible Transcription base URL under Admin Settings → AI Services", c.cfg.asrBaseURL()) + } + if len(req.Audio) == 0 { + return nil, fmt.Errorf("no audio data to transcribe") + } + + timeout := req.Timeout + if timeout <= 0 { + timeout = defaultTranscribeTimeout + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + url := endpointURL(c.cfg.asrBaseURL(), "audio/transcriptions") + + var result *Transcription + doErr := c.doWithRetry(ctx, c.asrHTTP, "transcription API", + func() (*http.Request, error) { + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + fw, err := w.CreateFormFile("file", req.Filename) + if err != nil { + return nil, fmt.Errorf("create multipart file: %w", err) + } + if _, err := fw.Write(req.Audio); err != nil { + return nil, fmt.Errorf("write multipart audio: %w", err) + } + fields := [][2]string{ + {"model", c.cfg.ASRModel}, + {"response_format", "verbose_json"}, + {"temperature", "0"}, + // Word timings let cue building split paragraph-length + // segments and end cues when speech actually stops; segment + // granularity must be requested alongside or OpenAI omits it. + {"timestamp_granularities[]", "segment"}, + {"timestamp_granularities[]", "word"}, + } + if req.Language != "" { + fields = append(fields, [2]string{"language", req.Language}) + } + if !hostMatchesAny(c.cfg.asrBaseURL(), strictHostedASRHosts) { + fields = append(fields, [2]string{"vad_filter", "true"}) + } + for _, f := range fields { + if err := w.WriteField(f[0], f[1]); err != nil { + return nil, fmt.Errorf("write multipart field %s: %w", f[0], err) + } + } + if err := w.Close(); err != nil { + return nil, fmt.Errorf("finalize multipart body: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(buf.Bytes())) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + httpReq.Header.Set("Content-Type", w.FormDataContentType()) + if key := c.cfg.asrAPIKey(); key != "" { + httpReq.Header.Set("Authorization", "Bearer "+key) + } + return httpReq, nil + }, + func(respBody []byte) error { + var parsed transcriptionResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + return fmt.Errorf("decode transcription response: %w", err) + } + if parsed.Error != nil && parsed.Error.Message != "" { + return fmt.Errorf("transcription API error: %s", parsed.Error.Message) + } + if parsed.Segments == nil { + return &permanentError{err: fmt.Errorf( + "transcription endpoint did not return verbose_json segments (model %q); a verbose_json-capable Whisper endpoint is required", c.cfg.ASRModel)} + } + out := &Transcription{Language: parsed.Language} + for _, s := range *parsed.Segments { + out.Segments = append(out.Segments, TranscriptionSegment{ + Start: s.Start, End: s.End, Text: s.Text, Words: wordsFromJSON(s.Words), + }) + } + attachTopLevelWords(out.Segments, wordsFromJSON(parsed.Words)) + result = out + return nil + }) + if doErr != nil { + return nil, decorateTranscribeError(doErr) + } + return result, nil +} + +func wordsFromJSON(words []transcriptionWordJSON) []TranscriptionWord { + out := make([]TranscriptionWord, 0, len(words)) + for _, w := range words { + out = append(out, TranscriptionWord{Start: w.Start, End: w.End, Text: w.Word}) + } + return out +} + +// attachTopLevelWords distributes an OpenAI-style top-level word list onto +// segments by word midpoint, for endpoints that report words separately from +// segments. Segments that already carry their own words are left untouched. +func attachTopLevelWords(segments []TranscriptionSegment, words []TranscriptionWord) { + if len(segments) == 0 || len(words) == 0 { + return + } + for _, s := range segments { + if len(s.Words) > 0 { + return + } + } + si := 0 + for _, w := range words { + mid := (w.Start + w.End) / 2 + for si < len(segments)-1 && mid >= segments[si].End { + si++ + } + segments[si].Words = append(segments[si].Words, w) + } +} + +// decorateTranscribeError appends a configuration hint to the errors a +// chat-only gateway produces when it receives a transcription upload (no +// /v1/audio/transcriptions route, or multipart rejected). Operators routinely +// point the shared base URL at a chat-only provider; without the hint the raw +// 400/404 reads like a pipeline bug instead of "set a Whisper endpoint". +func decorateTranscribeError(err error) error { + msg := err.Error() + likelyUnsupported := strings.Contains(msg, "returned 400") || + strings.Contains(msg, "returned 404") || + strings.Contains(msg, "returned 405") + if !likelyUnsupported { + return err + } + return fmt.Errorf("%w — the configured endpoint likely does not support audio transcription; "+ + "set a Whisper-compatible Transcription base URL (and model) under Admin Settings → AI Services "+ + "(e.g. a self-hosted faster-whisper/speaches server, api.groq.com/openai with whisper-large-v3-turbo, or api.openai.com with whisper-1)", err) +} diff --git a/internal/subtitles/ai/languages.go b/internal/ai/translate/languages.go similarity index 64% rename from internal/subtitles/ai/languages.go rename to internal/ai/translate/languages.go index 6e561b3a..808f3fb7 100644 --- a/internal/subtitles/ai/languages.go +++ b/internal/ai/translate/languages.go @@ -1,4 +1,4 @@ -package ai +package aitranslate import "strings" @@ -18,9 +18,25 @@ var languageNames = map[string]string{ "tr": "Turkish", "uk": "Ukrainian", "vi": "Vietnamese", "zh": "Chinese", } -// languageDisplayName returns a human-readable language name for a code, or the -// trimmed code itself when unknown. An empty code yields an empty string. -func languageDisplayName(code string) string { +// LanguageCodeFromName maps an English language name back to its ISO 639-1 +// code — Whisper endpoints report detected languages as names ("english") +// while local servers report codes. Returns "" when unknown. +func LanguageCodeFromName(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + if name == "" { + return "" + } + for code, display := range languageNames { + if strings.ToLower(display) == name { + return code + } + } + return "" +} + +// LanguageDisplayName returns a human-readable language name for a code, or +// the trimmed code itself when unknown. An empty code yields an empty string. +func LanguageDisplayName(code string) string { code = strings.TrimSpace(code) if code == "" { return "" diff --git a/internal/ai/translate/translate.go b/internal/ai/translate/translate.go new file mode 100644 index 00000000..4faa8d94 --- /dev/null +++ b/internal/ai/translate/translate.go @@ -0,0 +1,212 @@ +// Package aitranslate implements generic batched text translation over an +// OpenAI-compatible chat model. Callers hand it ordered text segments and a +// domain-specific system prompt; it owns the wire protocol (indexed-JSON +// batches the model must echo back key-for-key), context carryover between +// batches, and retries on malformed model output. Subtitle cues and metadata +// descriptions both translate through this one implementation. +package aitranslate + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// Segment is one unit of translatable text. ID is the caller's identifier and +// is never sent to the model (batches are keyed by position), so any stable +// string works. +type Segment struct { + ID string + Text string +} + +// ChatFn performs one chat completion. Wiring this as a function keeps the +// package independent of the client and trivially testable. +type ChatFn func(ctx context.Context, system, user string) (string, error) + +// Request describes one translation run. +type Request struct { + Segments []Segment + SystemPrompt string // domain-specific; must demand same-keys JSON output + // TargetName is the human-readable target language name used in the + // per-batch user message. + TargetName string + // EntryNoun names the units in the user message ("cues", "descriptions"); + // empty defaults to "entries". + EntryNoun string + // BatchSize is the number of segments per chat request (required, > 0). + BatchSize int + // ContextNeighbors is how many preceding source segments are included, + // untranslated, for continuity across batch boundaries. + ContextNeighbors int +} + +// maxFormatRetries is how many times a batch is re-asked after the model +// returns unparsable or incomplete JSON. Transport/API errors are retried +// inside the chat client and are not part of this count. +const maxFormatRetries = 2 + +// Translate translates req.Segments in order, preserving count and IDs and +// only rewriting Text. onBatch, when non-nil, is called after each batch with +// that batch's translated segments and overall progress (done/total +// segments). Implementations must honor ctx cancellation between batches. +func Translate(ctx context.Context, chat ChatFn, req Request, onBatch func(batch []Segment, done, total int)) ([]Segment, error) { + if chat == nil { + return nil, fmt.Errorf("chat function is nil") + } + if req.BatchSize <= 0 { + return nil, fmt.Errorf("invalid batch size: %d", req.BatchSize) + } + total := len(req.Segments) + if total == 0 { + return nil, fmt.Errorf("no segments to translate") + } + noun := req.EntryNoun + if noun == "" { + noun = "entries" + } + + // Preserve IDs by copying input segments and only replacing Text. + out := make([]Segment, total) + copy(out, req.Segments) + + for start := 0; start < total; start += req.BatchSize { + if err := ctx.Err(); err != nil { + return nil, err + } + end := min(start+req.BatchSize, total) + + contextStart := max(0, start-req.ContextNeighbors) + translated, err := translateBatch(ctx, chat, req, noun, req.Segments[contextStart:start], req.Segments[start:end]) + if err != nil { + return nil, fmt.Errorf("translate %s %d-%d: %w", noun, start+1, end, err) + } + for i, text := range translated { + out[start+i].Text = text + } + if onBatch != nil { + onBatch(out[start:end], end, total) + } + } + + return out, nil +} + +func translateBatch(ctx context.Context, chat ChatFn, req Request, noun string, contextSegs, batch []Segment) ([]string, error) { + texts := make([]string, len(batch)) + for i, s := range batch { + texts[i] = s.Text + } + payload, err := buildIndexedJSON(texts) + if err != nil { + return nil, err + } + + var user strings.Builder + if len(contextSegs) > 0 { + user.WriteString("Preceding lines for context only — do not translate or include them in your output:\n") + for _, s := range contextSegs { + user.WriteString(strings.ReplaceAll(s.Text, "\n", " ")) + user.WriteByte('\n') + } + user.WriteByte('\n') + } + target := req.TargetName + if target == "" { + target = "the target language" + } + fmt.Fprintf(&user, "Translate these %d %s into %s. Respond with only the JSON object:\n%s", len(batch), noun, target, payload) + + var lastErr error + for attempt := 0; attempt <= maxFormatRetries; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + content, err := chat(ctx, req.SystemPrompt, user.String()) + if err != nil { + return nil, err // transport/API errors are already retried inside the client + } + + obj, err := extractJSONObject(content) + if err != nil { + lastErr = err + continue + } + var m map[string]string + if err := json.Unmarshal([]byte(obj), &m); err != nil { + lastErr = fmt.Errorf("decode translation JSON: %w", err) + continue + } + + out := make([]string, len(batch)) + complete := true + for i := range batch { + v, ok := m[strconv.Itoa(i+1)] + if !ok { + complete = false + break + } + out[i] = v + } + if !complete { + lastErr = fmt.Errorf("model omitted one or more %s", noun) + continue + } + return out, nil + } + + return nil, fmt.Errorf("invalid model response after %d attempts: %w", maxFormatRetries+1, lastErr) +} + +// buildIndexedJSON renders texts as a JSON object {"1":..., "2":...} keyed by +// 1-based position, escaping each value safely. It is built by hand rather +// than json.Marshal'ing a map so the keys stay in numeric order — that reads +// more naturally for the model than the lexicographic order Go emits for maps +// ("1","10","11",...,"2"). Correctness doesn't depend on order (results are +// mapped back by key), but ordered input gives the model better context. +func buildIndexedJSON(texts []string) (string, error) { + var b strings.Builder + b.WriteByte('{') + for i, text := range texts { + if i > 0 { + b.WriteByte(',') + } + key, err := json.Marshal(strconv.Itoa(i + 1)) + if err != nil { + return "", err + } + val, err := json.Marshal(text) + if err != nil { + return "", err + } + b.Write(key) + b.WriteByte(':') + b.Write(val) + } + b.WriteByte('}') + return b.String(), nil +} + +// extractJSONObject pulls the first balanced-looking JSON object out of a +// model reply, tolerating ``` code fences and surrounding prose. +func extractJSONObject(s string) (string, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "```") { + s = strings.TrimPrefix(s, "```") + if nl := strings.IndexByte(s, '\n'); nl >= 0 { + s = s[nl+1:] + } + if idx := strings.LastIndex(s, "```"); idx >= 0 { + s = s[:idx] + } + s = strings.TrimSpace(s) + } + start := strings.IndexByte(s, '{') + end := strings.LastIndexByte(s, '}') + if start < 0 || end < 0 || end < start { + return "", fmt.Errorf("no JSON object found in model response") + } + return s[start : end+1], nil +} diff --git a/internal/ai/translate/translate_test.go b/internal/ai/translate/translate_test.go new file mode 100644 index 00000000..7ad06474 --- /dev/null +++ b/internal/ai/translate/translate_test.go @@ -0,0 +1,153 @@ +package aitranslate + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "testing" +) + +// echoChat translates by upper-casing every value of the indexed JSON payload +// in the user message, mimicking a well-behaved model. +func echoChat(t *testing.T) ChatFn { + t.Helper() + return func(_ context.Context, _ string, user string) (string, error) { + obj, err := extractJSONObject(user) + if err != nil { + return "", fmt.Errorf("test chat: %w", err) + } + var m map[string]string + if err := json.Unmarshal([]byte(obj), &m); err != nil { + return "", fmt.Errorf("test chat decode: %w", err) + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = strings.ToUpper(v) + } + b, _ := json.Marshal(out) + return string(b), nil + } +} + +func segments(n int) []Segment { + out := make([]Segment, n) + for i := range out { + out[i] = Segment{ID: strconv.Itoa(i + 1), Text: fmt.Sprintf("line %d", i+1)} + } + return out +} + +func TestTranslateBatchesAndPreservesIDs(t *testing.T) { + var batches []int + out, err := Translate(context.Background(), echoChat(t), Request{ + Segments: segments(7), + BatchSize: 3, + }, func(batch []Segment, done, total int) { + batches = append(batches, len(batch)) + if total != 7 { + t.Errorf("total = %d, want 7", total) + } + }) + if err != nil { + t.Fatalf("Translate: %v", err) + } + if len(out) != 7 { + t.Fatalf("len(out) = %d", len(out)) + } + for i, s := range out { + if s.ID != strconv.Itoa(i+1) { + t.Errorf("ID[%d] = %q, want %d", i, s.ID, i+1) + } + if want := strings.ToUpper(fmt.Sprintf("line %d", i+1)); s.Text != want { + t.Errorf("Text[%d] = %q, want %q", i, s.Text, want) + } + } + if fmt.Sprint(batches) != "[3 3 1]" { + t.Errorf("batch sizes = %v, want [3 3 1]", batches) + } +} + +func TestTranslateSendsContextNeighborsUntranslated(t *testing.T) { + var secondBatchMsg string + call := 0 + chat := func(ctx context.Context, system, user string) (string, error) { + call++ + if call == 2 { + secondBatchMsg = user + } + return echoChat(t)(ctx, system, user) + } + _, err := Translate(context.Background(), chat, Request{ + Segments: segments(4), + BatchSize: 2, + ContextNeighbors: 2, + TargetName: "French", + EntryNoun: "cues", + }, nil) + if err != nil { + t.Fatalf("Translate: %v", err) + } + if !strings.Contains(secondBatchMsg, "context only") || + !strings.Contains(secondBatchMsg, "line 1\nline 2") { + t.Errorf("second batch missing context block:\n%s", secondBatchMsg) + } + if !strings.Contains(secondBatchMsg, "Translate these 2 cues into French.") { + t.Errorf("second batch missing instruction:\n%s", secondBatchMsg) + } +} + +func TestTranslateToleratesCodeFences(t *testing.T) { + chat := func(_ context.Context, _, _ string) (string, error) { + return "Sure! Here you go:\n```json\n{\"1\":\"ok\"}\n```", nil + } + out, err := Translate(context.Background(), chat, Request{ + Segments: segments(1), + BatchSize: 5, + }, nil) + if err != nil { + t.Fatalf("Translate: %v", err) + } + if out[0].Text != "ok" { + t.Errorf("Text = %q", out[0].Text) + } +} + +func TestTranslateRetriesOmittedKeysThenFails(t *testing.T) { + calls := 0 + chat := func(_ context.Context, _, _ string) (string, error) { + calls++ + return `{"1":"only one"}`, nil // always omits key "2" + } + _, err := Translate(context.Background(), chat, Request{ + Segments: segments(2), + BatchSize: 5, + }, nil) + if err == nil || !strings.Contains(err.Error(), "omitted") { + t.Fatalf("err = %v, want omission failure", err) + } + if calls != maxFormatRetries+1 { + t.Errorf("calls = %d, want %d", calls, maxFormatRetries+1) + } +} + +func TestTranslateStopsOnCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + chat := func(_ context.Context, _, _ string) (string, error) { + calls++ + cancel() // cancel after the first batch completes + return `{"1":"a"}`, nil + } + _, err := Translate(ctx, chat, Request{ + Segments: segments(3), + BatchSize: 1, + }, nil) + if err == nil || !strings.Contains(err.Error(), "context canceled") { + t.Fatalf("err = %v, want context cancellation", err) + } + if calls != 1 { + t.Errorf("calls = %d, want 1", calls) + } +} diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 48b33281..bf453046 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -20,6 +20,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/Silo-Server/silo-server/internal/access" + "github.com/Silo-Server/silo-server/internal/ai/llm" "github.com/Silo-Server/silo-server/internal/adminjob" apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/auth" @@ -2101,6 +2102,21 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques } else { req.Value = normalized } + case "ai.asr_base_url": + if llm.IsChatOnlyGateway(req.Value) { + writeError(w, http.StatusBadRequest, "bad_request", + "This endpoint cannot produce timestamped transcriptions (chat-only gateway). "+ + "Use a self-hosted Whisper server (faster-whisper/speaches), api.groq.com/openai, or api.openai.com.") + return + } + case "metadata_ai.on_view": + switch req.Value { + case "off", "button", "auto": + default: + writeError(w, http.StatusBadRequest, "bad_request", + "metadata_ai.on_view must be off, button, or auto") + return + } } if err := h.SettingsRepo.Set(r.Context(), key, req.Value); err != nil { diff --git a/internal/api/handlers/items.go b/internal/api/handlers/items.go index 7697073c..4960b820 100644 --- a/internal/api/handlers/items.go +++ b/internal/api/handlers/items.go @@ -1497,14 +1497,15 @@ func (h *ItemsHandler) accessFilter(r *http.Request) catalog.AccessFilter { if scope, ok := access.GetScope(r.Context()); ok { return catalog.AccessFilter{ - AllowedLibraryIDs: scope.AllowedLibraryIDs, - DisabledLibraryIDs: scope.DisabledLibraryIDs, - MaxContentRating: scope.MaxContentRating, - MaxPlaybackQuality: scope.MaxPlaybackQuality, - PresentationLibraryID: presentationLibraryID, - SelectedFileID: selectedFileID, - UserID: apimw.GetUserID(r.Context()), - ProfileID: apimw.GetProfileID(r.Context()), + AllowedLibraryIDs: scope.AllowedLibraryIDs, + DisabledLibraryIDs: scope.DisabledLibraryIDs, + MaxContentRating: scope.MaxContentRating, + MaxPlaybackQuality: scope.MaxPlaybackQuality, + PresentationLibraryID: presentationLibraryID, + ProfilePreferredLanguage: scope.PreferredMetadataLanguage, + SelectedFileID: selectedFileID, + UserID: apimw.GetUserID(r.Context()), + ProfileID: apimw.GetProfileID(r.Context()), } } diff --git a/internal/api/handlers/libraries.go b/internal/api/handlers/libraries.go index fae7be03..40c852e2 100644 --- a/internal/api/handlers/libraries.go +++ b/internal/api/handlers/libraries.go @@ -192,6 +192,7 @@ type updateLibraryRequest struct { Name *string `json:"name,omitempty"` Enabled *bool `json:"enabled,omitempty"` MetadataLanguage *string `json:"metadata_language,omitempty"` + AutoTranslateMetadata *bool `json:"auto_translate_metadata,omitempty"` ChapterThumbnailsEnabled *bool `json:"chapter_thumbnails_enabled,omitempty"` IntroDetectionEnabled *bool `json:"intro_detection_enabled,omitempty"` } @@ -226,6 +227,7 @@ type libraryResponse struct { Name string `json:"name"` Enabled bool `json:"enabled"` MetadataLanguage string `json:"metadata_language"` + AutoTranslateMetadata bool `json:"auto_translate_metadata"` ChapterThumbnailsEnabled bool `json:"chapter_thumbnails_enabled"` ChapterThumbnailsSupported bool `json:"chapter_thumbnails_supported"` IntroDetectionEnabled bool `json:"intro_detection_enabled"` @@ -348,6 +350,7 @@ func toLibraryResponse(f *models.MediaFolder) libraryResponse { Name: f.Name, Enabled: f.Enabled, MetadataLanguage: f.MetadataLanguage, + AutoTranslateMetadata: f.AutoTranslateMetadata, ChapterThumbnailsEnabled: f.ChapterThumbnailsEnabled, ChapterThumbnailsSupported: false, IntroDetectionEnabled: f.IntroDetectionEnabled, @@ -652,6 +655,7 @@ func (h *LibraryHandler) HandleUpdateLibrary(w http.ResponseWriter, r *http.Requ Name: req.Name, Enabled: req.Enabled, MetadataLanguage: req.MetadataLanguage, + AutoTranslateMetadata: req.AutoTranslateMetadata, ChapterThumbnailsEnabled: req.ChapterThumbnailsEnabled, IntroDetectionEnabled: req.IntroDetectionEnabled, }) diff --git a/internal/api/handlers/metadata_ai.go b/internal/api/handlers/metadata_ai.go new file mode 100644 index 00000000..16848b03 --- /dev/null +++ b/internal/api/handlers/metadata_ai.go @@ -0,0 +1,212 @@ +package handlers + +import ( + "encoding/json" + "errors" + "log/slog" + "net/http" + "strconv" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/access" + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/metadata/translation" +) + +// MetadataAIHandler exposes AI translation of catalog descriptions into the +// localization tables. The admin routes are mounted under the per-item +// metadata curation guard; the on-view route is viewer-facing and enforces +// item access itself. +type MetadataAIHandler struct { + service *translation.Service + // ItemAccess authorizes the viewer-facing on-view route; nil disables it. + ItemAccess *catalog.ItemRepository +} + +// NewMetadataAIHandler creates a handler backed by the given service. +func NewMetadataAIHandler(service *translation.Service) *MetadataAIHandler { + return &MetadataAIHandler{service: service} +} + +// HandleStatus reports whether metadata AI translation is available and the +// viewer-facing on-view mode, so the metadata editor and detail pages can +// show or hide their entry points. +// GET /api/v1/metadata/ai/status +func (h *MetadataAIHandler) HandleStatus(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{ + "enabled": h.service.Enabled(), + "on_view": h.service.OnViewMode(), + }) +} + +// WriteMetadataAIDisabledStatus answers the status probe with a clean negative +// when no metadata AI handler is wired. +func WriteMetadataAIDisabledStatus(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]any{"enabled": false, "on_view": "off"}) +} + +type translateDescriptionRequest struct { + // TargetLanguage echoes the detail response's pending_translation_language. + TargetLanguage string `json:"target_language"` +} + +// HandleTranslateOnView is the viewer-facing on-demand description +// translation: any profile that can access the item may request its +// descriptions in the language the detail response reported missing. Gated by +// metadata_ai.on_view; duplicate viewers collapse onto one job and recently +// failed targets are not retried (cooldown in the service). +// POST /api/v1/items/{id}/translate-description +func (h *MetadataAIHandler) HandleTranslateOnView(w http.ResponseWriter, r *http.Request) { + contentID := chi.URLParam(r, "id") + var req translateDescriptionRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Invalid request body") + return + } + if req.TargetLanguage == "" { + writeError(w, http.StatusBadRequest, "bad_request", "target_language is required") + return + } + + scope, ok := access.GetScope(r.Context()) + if !ok || h.ItemAccess == nil { + writeError(w, http.StatusForbidden, "forbidden", "Viewer access is required") + return + } + filter := catalog.AccessFilter{ + AllowedLibraryIDs: scope.AllowedLibraryIDs, + DisabledLibraryIDs: scope.DisabledLibraryIDs, + MaxContentRating: scope.MaxContentRating, + UserID: scope.UserID, + ProfileID: scope.ProfileID, + } + if err := h.ItemAccess.EnsureAccessible(r.Context(), contentID, filter); err != nil { + if errors.Is(err, catalog.ErrItemNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Item not found") + return + } + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize item") + return + } + + var requestedBy *int + if userID := apimw.GetUserID(r.Context()); userID != 0 { + requestedBy = &userID + } + + job, err := h.service.RequestOnView(r.Context(), contentID, req.TargetLanguage, requestedBy) + if err != nil { + switch { + case errors.Is(err, translation.ErrNotConfigured): + writeError(w, http.StatusServiceUnavailable, "not_configured", + "On-view translation is not enabled on this server") + case errors.Is(err, translation.ErrInvalidRequest): + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + default: + slog.Error("failed to request on-view translation", + "content_id", contentID, "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to start translation") + } + return + } + + writeJSON(w, http.StatusAccepted, map[string]any{"job": job}) +} + +type translateMetadataRequest struct { + TargetLanguage string `json:"target_language"` + IncludeChildren *bool `json:"include_children"` // default true + Force bool `json:"force"` +} + +// HandleTranslate enqueues a translation job for an item. +// POST /api/v1/admin/items/{id}/metadata-translation +func (h *MetadataAIHandler) HandleTranslate(w http.ResponseWriter, r *http.Request) { + contentID := chi.URLParam(r, "id") + var req translateMetadataRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "invalid_request", "Invalid request body") + return + } + if req.TargetLanguage == "" { + writeError(w, http.StatusBadRequest, "bad_request", "target_language is required") + return + } + includeChildren := true + if req.IncludeChildren != nil { + includeChildren = *req.IncludeChildren + } + + var requestedBy *int + if userID := apimw.GetUserID(r.Context()); userID != 0 { + requestedBy = &userID + } + + job, err := h.service.Enqueue(r.Context(), translation.JobRequest{ + TargetKind: translation.TargetItem, + ContentID: contentID, + TargetLanguage: req.TargetLanguage, + IncludeChildren: includeChildren, + Force: req.Force, + RequestedBy: requestedBy, + }) + if err != nil { + switch { + case errors.Is(err, translation.ErrNotConfigured): + writeError(w, http.StatusServiceUnavailable, "not_configured", + "Metadata AI translation is not configured on this server") + case errors.Is(err, translation.ErrInvalidRequest): + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + default: + slog.Error("failed to enqueue metadata translation", + "content_id", contentID, "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to start translation") + } + return + } + + writeJSON(w, http.StatusAccepted, map[string]any{"job": job}) +} + +// HandleListJobs lists recent translation jobs for an item; the metadata +// editor polls this for progress. +// GET /api/v1/admin/items/{id}/metadata-translation/jobs +func (h *MetadataAIHandler) HandleListJobs(w http.ResponseWriter, r *http.Request) { + jobs, err := h.service.ListJobs(r.Context(), chi.URLParam(r, "id")) + if err != nil { + writeError(w, http.StatusInternalServerError, "list_error", "Failed to list jobs") + return + } + writeJSON(w, http.StatusOK, map[string]any{"jobs": jobs}) +} + +// HandleCancelJob cancels a job belonging to the item in the URL. +// POST /api/v1/admin/items/{id}/metadata-translation/jobs/{job_id}/cancel +func (h *MetadataAIHandler) HandleCancelJob(w http.ResponseWriter, r *http.Request) { + jobID, err := strconv.ParseInt(chi.URLParam(r, "job_id"), 10, 64) + if err != nil { + writeError(w, http.StatusBadRequest, "invalid_id", "Invalid job ID") + return + } + job, err := h.service.GetJob(r.Context(), jobID) + if err != nil { + if errors.Is(err, translation.ErrJobNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Job not found") + return + } + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load job") + return + } + // The curation guard authorized {id}; the job must belong to it. + if job.ContentID != chi.URLParam(r, "id") { + writeError(w, http.StatusNotFound, "not_found", "Job not found") + return + } + if err := h.service.Cancel(r.Context(), jobID); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to cancel job") + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/handlers/profiles.go b/internal/api/handlers/profiles.go index e073e92b..07caacd1 100644 --- a/internal/api/handlers/profiles.go +++ b/internal/api/handlers/profiles.go @@ -48,6 +48,7 @@ type createProfileRequest struct { MaxContentRating string `json:"max_content_rating,omitempty"` QualityPreference string `json:"quality_preference,omitempty"` Language string `json:"language,omitempty"` + PreferredMetadataLanguage string `json:"preferred_metadata_language,omitempty"` SubtitleLanguage string `json:"subtitle_language,omitempty"` SubtitleMode string `json:"subtitle_mode,omitempty"` AutoSkipIntro bool `json:"auto_skip_intro"` @@ -68,6 +69,7 @@ type updateProfileRequest struct { MaxContentRating *string `json:"max_content_rating,omitempty"` QualityPreference *string `json:"quality_preference,omitempty"` Language *string `json:"language,omitempty"` + PreferredMetadataLanguage *string `json:"preferred_metadata_language,omitempty"` SubtitleLanguage *string `json:"subtitle_language,omitempty"` SubtitleMode *string `json:"subtitle_mode,omitempty"` AutoSkipIntro *bool `json:"auto_skip_intro,omitempty"` @@ -96,6 +98,7 @@ type profileResponse struct { MaxContentRating string `json:"max_content_rating,omitempty"` QualityPreference string `json:"quality_preference,omitempty"` Language string `json:"language,omitempty"` + PreferredMetadataLanguage string `json:"preferred_metadata_language,omitempty"` SubtitleLanguage string `json:"subtitle_language,omitempty"` SubtitleMode string `json:"subtitle_mode,omitempty"` AutoSkipIntro bool `json:"auto_skip_intro"` @@ -356,6 +359,7 @@ func (h *ProfileHandler) HandleCreateProfile(w http.ResponseWriter, r *http.Requ MaxContentRating: req.MaxContentRating, QualityPreference: req.QualityPreference, Language: req.Language, + PreferredMetadataLanguage: req.PreferredMetadataLanguage, SubtitleLanguage: req.SubtitleLanguage, SubtitleMode: req.SubtitleMode, AutoSkipIntro: req.AutoSkipIntro, @@ -505,6 +509,7 @@ func (h *ProfileHandler) HandleUpdateProfile(w http.ResponseWriter, r *http.Requ MaxContentRating: req.MaxContentRating, QualityPreference: req.QualityPreference, Language: req.Language, + PreferredMetadataLanguage: req.PreferredMetadataLanguage, SubtitleLanguage: req.SubtitleLanguage, SubtitleMode: req.SubtitleMode, AutoSkipIntro: req.AutoSkipIntro, @@ -684,6 +689,7 @@ func (h *ProfileHandler) toProfileResponse(ctx context.Context, p userstore.Prof MaxContentRating: p.MaxContentRating, QualityPreference: p.QualityPreference, Language: p.Language, + PreferredMetadataLanguage: p.PreferredMetadataLanguage, SubtitleLanguage: p.SubtitleLanguage, SubtitleMode: p.SubtitleMode, AutoSkipIntro: p.AutoSkipIntro, diff --git a/internal/api/handlers/subtitle_ai.go b/internal/api/handlers/subtitle_ai.go index 72d0bf4e..90a8e021 100644 --- a/internal/api/handlers/subtitle_ai.go +++ b/internal/api/handlers/subtitle_ai.go @@ -48,7 +48,12 @@ func authorizeMediaFileAccess(w http.ResponseWriter, r *http.Request, authorizer } type translateSubtitleRequest struct { - MediaFileID int `json:"media_file_id"` + MediaFileID int `json:"media_file_id"` + // Kind selects the job: "translate" (default), "transcribe", or + // "transcribe_translate". For ASR kinds, source_index is the audio track + // index (-1 = default track) and target_language is optional for plain + // transcribe (acts as a language hint). + Kind string `json:"kind"` SourceIndex int `json:"source_index"` SourceLanguage string `json:"source_language"` TargetLanguage string `json:"target_language"` @@ -56,10 +61,14 @@ type translateSubtitleRequest struct { StartPosition float64 `json:"start_position"` } -// HandleStatus reports whether AI subtitle translation is available, so the -// player can show or hide the entry point. GET /api/v1/subtitles/ai/status +// HandleStatus reports whether AI subtitle translation / ASR generation are +// available, so the player can show or hide the entry points. +// GET /api/v1/subtitles/ai/status func (h *SubtitleAIHandler) HandleStatus(w http.ResponseWriter, r *http.Request) { - writeJSON(w, http.StatusOK, map[string]any{"enabled": h.service.Enabled()}) + writeJSON(w, http.StatusOK, map[string]any{ + "enabled": h.service.Enabled(), + "transcribe_enabled": h.service.TranscribeEnabled(), + }) } // WriteSubtitleAIDisabledStatus answers the AI status capability probe with a @@ -67,7 +76,7 @@ func (h *SubtitleAIHandler) HandleStatus(w http.ResponseWriter, r *http.Request) // negative instead of a 404 (the 2-segment /ai/status path is not shadowed by the // 1-segment /{media_file_id} route — they never compete in chi's router). func WriteSubtitleAIDisabledStatus(w http.ResponseWriter, _ *http.Request) { - writeJSON(w, http.StatusOK, map[string]any{"enabled": false}) + writeJSON(w, http.StatusOK, map[string]any{"enabled": false, "transcribe_enabled": false}) } // HandleTranslate enqueues a translation job. POST /api/v1/subtitles/ai/translate @@ -81,7 +90,13 @@ func (h *SubtitleAIHandler) HandleTranslate(w http.ResponseWriter, r *http.Reque writeError(w, http.StatusBadRequest, "bad_request", "media_file_id is required") return } - if req.TargetLanguage == "" { + kind := ai.JobKind(req.Kind) + if kind == "" { + kind = ai.JobKindTranslate + } + // A plain transcribe produces a track in the spoken language, so the + // target is optional; every other kind needs one. + if req.TargetLanguage == "" && kind != ai.JobKindTranscribe { writeError(w, http.StatusBadRequest, "bad_request", "target_language is required") return } @@ -97,7 +112,7 @@ func (h *SubtitleAIHandler) HandleTranslate(w http.ResponseWriter, r *http.Reque job, err := h.service.Enqueue(r.Context(), ai.JobRequest{ MediaFileID: req.MediaFileID, - Kind: ai.JobKindTranslate, + Kind: kind, SourceIndex: req.SourceIndex, SourceLanguage: req.SourceLanguage, TargetLanguage: req.TargetLanguage, diff --git a/internal/api/router.go b/internal/api/router.go index e14c0bea..f1906fe2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -20,6 +20,8 @@ import ( "github.com/Silo-Server/silo-server/internal/access" "github.com/Silo-Server/silo-server/internal/activitylog" "github.com/Silo-Server/silo-server/internal/adminjob" + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" + "github.com/Silo-Server/silo-server/internal/ai/llm" "github.com/Silo-Server/silo-server/internal/api/handlers" apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/auth" @@ -40,6 +42,7 @@ import ( "github.com/Silo-Server/silo-server/internal/metadata" "github.com/Silo-Server/silo-server/internal/metadata/tmdb" metatrakt "github.com/Silo-Server/silo-server/internal/metadata/trakt" + metadatatranslation "github.com/Silo-Server/silo-server/internal/metadata/translation" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/notifications" "github.com/Silo-Server/silo-server/internal/opslog" @@ -907,14 +910,45 @@ func NewRouter(deps Dependencies) chi.Router { // Build the AI subtitle handler (on-demand translation). Generated tracks are // stored as ordinary downloaded subtitles, so they reach every client through // the existing subtitle pipeline with no client changes. + // Shared AI endpoint client + dispatch semaphore: subtitle translation/ASR + // and metadata translation draw from one client and one concurrency bound. + var aiClient *llm.Client + var aiSem chan struct{} + if deps.Config != nil { + aiClient = llm.NewClient(llm.Config{ + BaseURL: deps.Config.AI.BaseURL, + APIKey: deps.Config.AI.APIKey, + ChatModel: deps.Config.AI.ChatModel, + ASRBaseURL: deps.Config.AI.ASRBaseURL, + ASRAPIKey: deps.Config.AI.ASRAPIKey, + ASRModel: deps.Config.AI.ASRModel, + }) + aiSem = jobrunner.NewSemaphore(deps.Config.AI.MaxConcurrentJobs) + } + var subtitleAIHandler *handlers.SubtitleAIHandler if subtitleManager != nil && subtitleRepo != nil && deps.FileRepo != nil && deps.DB != nil && deps.Config != nil { + // A chat-only gateway (e.g. OpenRouter) cannot produce timestamped + // transcriptions; disable ASR rather than let every job fail. The + // settings API rejects such values for the ASR URL, but the chat base + // URL legitimately may be one — this catches the blank-ASR-URL + // fallback case. + transcribeEnabled := deps.Config.SubtitleAI.TranscribeEnabled + effectiveASRBase := deps.Config.AI.ASRBaseURL + if effectiveASRBase == "" { + effectiveASRBase = deps.Config.AI.BaseURL + } + if transcribeEnabled && llm.IsChatOnlyGateway(effectiveASRBase) { + slog.Warn("subtitle transcription disabled: the effective transcription endpoint is a chat-only gateway; "+ + "set a Whisper-compatible Transcription base URL in AI Services", "endpoint", effectiveASRBase) + transcribeEnabled = false + } aiCfg := subtitleai.Config{ - Enabled: deps.Config.SubtitleAI.Enabled, - BaseURL: deps.Config.SubtitleAI.BaseURL, - APIKey: deps.Config.SubtitleAI.APIKey, - ChatModel: deps.Config.SubtitleAI.ChatModel, - MaxConcurrentJobs: deps.Config.SubtitleAI.MaxConcurrentJobs, + Configured: deps.Config.AI.BaseURL != "", + TranslateEnabled: deps.Config.SubtitleAI.Enabled, + TranscribeEnabled: transcribeEnabled, + ChatModel: deps.Config.AI.ChatModel, + ASRModel: deps.Config.AI.ASRModel, BatchSize: deps.Config.SubtitleAI.BatchSize, ContextNeighbors: deps.Config.SubtitleAI.ContextNeighbors, } @@ -926,18 +960,54 @@ func NewRouter(deps Dependencies) chi.Router { deps.AppContext, aiCfg, subtitleai.NewPgJobRepository(deps.DB), - subtitleai.NewLLMTranslator(subtitleai.NewClient(aiCfg), aiCfg.BatchSize, aiCfg.ContextNeighbors), + subtitleai.NewLLMTranslator(aiClient, aiCfg.BatchSize, aiCfg.ContextNeighbors), + subtitleai.NewWhisperTranscriber(aiClient, deps.Config.Playback.FFmpegPath, deps.Config.SubtitleAI.ASRChunkSeconds), subtitleManager, subtitleRepo, deps.FileRepo, aiNotifier, deps.Config.Playback.FFmpegPath, slog.Default(), + aiSem, ) aiService.Recover() subtitleAIHandler = handlers.NewSubtitleAIHandler(aiService) } + // Metadata AI translation (descriptions into the localization tables). + var metadataAIHandler *handlers.MetadataAIHandler + if deps.DB != nil && deps.Config != nil && aiClient != nil { + mtRepo := metadatatranslation.NewPgRepository(deps.DB) + mtService := metadatatranslation.NewService( + deps.AppContext, + metadatatranslation.Config{ + Enabled: deps.Config.MetadataAI.Enabled, + Configured: deps.Config.AI.BaseURL != "", + ChatModel: deps.Config.AI.ChatModel, + OnView: deps.Config.MetadataAI.OnView, + }, + mtRepo, + mtRepo, + &metadatatranslation.CatalogLocalizationStore{ + Items: catalog.NewMediaItemLocalizationRepository(deps.DB), + Seasons: catalog.NewSeasonLocalizationRepository(deps.DB), + Episodes: catalog.NewEpisodeLocalizationRepository(deps.DB), + }, + aiClient.SystemUserChat, + aiSem, + slog.Default(), + ) + mtService.Recover() + metadataAIHandler = handlers.NewMetadataAIHandler(mtService) + // Wire the refresh fallback: libraries with auto_translate_metadata get + // missing localizations filled after each metadata refresh. + if mt, ok := deps.MetadataService.(interface { + SetAutoTranslator(metadata.AutoTranslator) + }); ok { + mt.SetAutoTranslator(mtService) + } + } + // Build section handler if DB is available. var sectionHandler *handlers.SectionHandler var sectionSettingsHandler *handlers.SectionSettingsHandler @@ -1759,6 +1829,19 @@ func NewRouter(deps Dependencies) chi.Router { }) } + // Metadata AI translation availability probe (the metadata editor + // and detail pages show or hide their translate actions based on + // this) plus the viewer-facing on-view translation trigger. + if metadataAIHandler != nil { + r.Get("/metadata/ai/status", metadataAIHandler.HandleStatus) + if itemRepo != nil { + metadataAIHandler.ItemAccess = itemRepo + r.Post("/items/{id}/translate-description", metadataAIHandler.HandleTranslateOnView) + } + } else { + r.Get("/metadata/ai/status", handlers.WriteMetadataAIDisabledStatus) + } + // Subtitle search + AI translation routes. if subtitleSearchHandler != nil { if deps.FileRepo != nil && itemRepo != nil { @@ -1929,6 +2012,11 @@ func NewRouter(deps Dependencies) chi.Router { r.Post("/items/{id}/match/search", adminMatchHandler.HandleSearchItemMatchCandidates) r.Post("/items/{id}/match/apply", adminMatchHandler.HandleApplyItemMatch) } + if metadataAIHandler != nil { + r.Post("/items/{id}/metadata-translation", metadataAIHandler.HandleTranslate) + r.Get("/items/{id}/metadata-translation/jobs", metadataAIHandler.HandleListJobs) + r.Post("/items/{id}/metadata-translation/jobs/{job_id}/cancel", metadataAIHandler.HandleCancelJob) + } }) if adminJobsHandler != nil { diff --git a/internal/catalog/access_filter.go b/internal/catalog/access_filter.go index 341b9c36..c9316d2b 100644 --- a/internal/catalog/access_filter.go +++ b/internal/catalog/access_filter.go @@ -15,7 +15,11 @@ type AccessFilter struct { DisabledLibraryIDs []int // user-disabled libraries (only set when AllowedLibraryIDs is nil) PresentationLibraryID *int PresentationLanguage string - MaxContentRating string + // ProfilePreferredLanguage is the viewer profile's preferred metadata + // language. Presentation language resolves: explicit PresentationLanguage + // → ProfilePreferredLanguage → the library's metadata_language. + ProfilePreferredLanguage string + MaxContentRating string MaxPlaybackQuality string SelectedFileID int UserID int diff --git a/internal/catalog/detail.go b/internal/catalog/detail.go index e46b016a..ddf56718 100644 --- a/internal/catalog/detail.go +++ b/internal/catalog/detail.go @@ -74,6 +74,10 @@ type ItemDetail struct { Year int `json:"year,omitempty"` Overview string `json:"overview,omitempty"` Tagline string `json:"tagline,omitempty"` + // PendingTranslationLanguage, when set, is the viewer's presentation + // language that the description is missing — the on-view AI translation + // affordance keys off it. + PendingTranslationLanguage string `json:"pending_translation_language,omitempty"` Runtime int `json:"runtime,omitempty"` ContentRating string `json:"content_rating,omitempty"` Genres []string `json:"genres"` @@ -576,10 +580,16 @@ func cloneEpisode(ep *models.Episode) *models.Episode { return &cp } +// resolvePresentationLanguage picks the display language for a request: +// explicit request language → viewer profile preference → the presentation +// library's metadata language. func (s *DetailService) resolvePresentationLanguage(ctx context.Context, filter AccessFilter) (string, error) { if strings.TrimSpace(filter.PresentationLanguage) != "" { return strings.TrimSpace(filter.PresentationLanguage), nil } + if strings.TrimSpace(filter.ProfilePreferredLanguage) != "" { + return strings.TrimSpace(filter.ProfilePreferredLanguage), nil + } if filter.PresentationLibraryID == nil || s.folderRepo == nil { return "", nil } @@ -593,6 +603,27 @@ func (s *DetailService) resolvePresentationLanguage(ctx context.Context, filter return strings.TrimSpace(folder.MetadataLanguage), nil } +// PendingTranslationLanguage reports the presentation language the item's +// description is missing: non-empty when the resolved language differs from +// the item's base metadata language, the base overview has text, and no +// localized overview exists yet. Clients use it to offer (or auto-run) +// on-view AI translation; it is pure data, independent of whether the AI +// feature is enabled. +func (s *DetailService) PendingTranslationLanguage(ctx context.Context, item *models.MediaItem, filter AccessFilter) string { + if item == nil || strings.TrimSpace(item.Overview) == "" || s.itemLocRepo == nil { + return "" + } + language, err := s.resolvePresentationLanguage(ctx, filter) + if err != nil || language == "" || sameMetadataLanguage(item.DefaultMetadataLanguage, language) { + return "" + } + loc, err := s.itemLocRepo.Get(ctx, item.ContentID, language) + if err != nil || (loc != nil && loc.Overview != "") { + return "" + } + return language +} + func (s *DetailService) validatePresentationItemAccess(ctx context.Context, filter AccessFilter, contentID string) error { if filter.PresentationLibraryID == nil { return nil @@ -623,17 +654,7 @@ func (s *DetailService) LocalizeItemModel(ctx context.Context, item *models.Medi if err != nil || loc == nil { return cloneMediaItem(item), err } - localized := cloneMediaItem(item) - localized.Title = loc.Title - localized.SortTitle = loc.SortTitle - localized.Overview = loc.Overview - localized.Tagline = loc.Tagline - localized.PosterPath = loc.PosterPath - localized.PosterThumbhash = loc.PosterThumbhash - localized.BackdropPath = loc.BackdropPath - localized.BackdropThumbhash = loc.BackdropThumbhash - localized.LogoPath = loc.LogoPath - return localized, nil + return applyItemLocalization(item, loc), nil } func (s *DetailService) LocalizeSeasonModel(ctx context.Context, season *models.Season, filter AccessFilter) (*models.Season, error) { @@ -648,12 +669,7 @@ func (s *DetailService) LocalizeSeasonModel(ctx context.Context, season *models. if err != nil || loc == nil { return cloneSeason(season), err } - localized := cloneSeason(season) - localized.Title = loc.Title - localized.Overview = loc.Overview - localized.PosterPath = loc.PosterPath - localized.PosterThumbhash = loc.PosterThumbhash - return localized, nil + return applySeasonLocalization(season, loc), nil } func (s *DetailService) LocalizeEpisodeModel(ctx context.Context, episode *models.Episode, filter AccessFilter) (*models.Episode, error) { @@ -668,10 +684,7 @@ func (s *DetailService) LocalizeEpisodeModel(ctx context.Context, episode *model if err != nil || loc == nil { return cloneEpisode(episode), err } - localized := cloneEpisode(episode) - localized.Title = loc.Title - localized.Overview = loc.Overview - return localized, nil + return applyEpisodeLocalization(episode, loc), nil } // GetItemDetail retrieves a full item detail with presigned URLs and file versions. @@ -828,6 +841,7 @@ func (s *DetailService) fetchCredits(ctx context.Context, contentID string) ([]C } func (s *DetailService) buildMediaItemDetail(ctx context.Context, item *models.MediaItem, contentID string, filter AccessFilter) (*ItemDetail, error) { + pendingTranslation := s.PendingTranslationLanguage(ctx, item, filter) localizedItem, err := s.LocalizeItemModel(ctx, item, filter) if err != nil { return nil, fmt.Errorf("localizing item detail: %w", err) @@ -843,6 +857,7 @@ func (s *DetailService) buildMediaItemDetail(ctx context.Context, item *models.M Year: item.Year, Overview: item.Overview, Tagline: item.Tagline, + PendingTranslationLanguage: pendingTranslation, Runtime: item.Runtime, ContentRating: item.ContentRating, Genres: item.Genres, diff --git a/internal/catalog/encrypted_settings_repo.go b/internal/catalog/encrypted_settings_repo.go index 7ce1e46a..30328d48 100644 --- a/internal/catalog/encrypted_settings_repo.go +++ b/internal/catalog/encrypted_settings_repo.go @@ -69,7 +69,11 @@ var SensitiveSettingKeys = map[string]bool{ "mdblist.api_key": true, "introdb.api_key": true, - // Subtitle AI (on-demand translation) API key. + // Shared AI endpoint API keys (+ legacy subtitle_ai alias the loader still + // falls back to; the legacy row is never renamed because ciphertext is + // GCM-bound to its key). + "ai.api_key": true, + "ai.asr_api_key": true, "subtitle_ai.api_key": true, // Recommendations embedding auth (+ legacy openai alias at db_loader L409). diff --git a/internal/catalog/folder_repo.go b/internal/catalog/folder_repo.go index 4ef8716f..c8e6fdbe 100644 --- a/internal/catalog/folder_repo.go +++ b/internal/catalog/folder_repo.go @@ -112,6 +112,7 @@ type UpdateFolderInput struct { Name *string Enabled *bool MetadataLanguage *string + AutoTranslateMetadata *bool ChapterThumbnailsEnabled *bool IntroDetectionEnabled *bool } @@ -142,7 +143,7 @@ func (r *FolderRepository) Pool() *pgxpool.Pool { // folderColumns is the list of columns returned by all SELECT queries. // Kept in one place so scanFolder stays in sync. -const folderColumns = `id, type, name, enabled, metadata_language, chapter_thumbnails_enabled, intro_detection_enabled, poster_path, last_scanned_at, +const folderColumns = `id, type, name, enabled, metadata_language, auto_translate_metadata, chapter_thumbnails_enabled, intro_detection_enabled, poster_path, last_scanned_at, scan_warning_code, scan_warning_message, scan_warning_at, allow_empty_cleanup_once, sort_order` // scanFolder scans a single row into a *models.MediaFolder. @@ -155,6 +156,7 @@ func scanFolder(row pgx.Row) (*models.MediaFolder, error) { &f.Name, &f.Enabled, &f.MetadataLanguage, + &f.AutoTranslateMetadata, &f.ChapterThumbnailsEnabled, &f.IntroDetectionEnabled, &f.PosterPath, @@ -187,6 +189,7 @@ func scanFolders(rows pgx.Rows) ([]*models.MediaFolder, error) { &f.Name, &f.Enabled, &f.MetadataLanguage, + &f.AutoTranslateMetadata, &f.ChapterThumbnailsEnabled, &f.IntroDetectionEnabled, &f.PosterPath, @@ -400,6 +403,11 @@ func (r *FolderRepository) Update(ctx context.Context, id int, input UpdateFolde args = append(args, *input.MetadataLanguage) argIndex++ } + if input.AutoTranslateMetadata != nil { + setClauses = append(setClauses, fmt.Sprintf("auto_translate_metadata = $%d", argIndex)) + args = append(args, *input.AutoTranslateMetadata) + argIndex++ + } if input.ChapterThumbnailsEnabled != nil { setClauses = append(setClauses, fmt.Sprintf("chapter_thumbnails_enabled = $%d", argIndex)) args = append(args, *input.ChapterThumbnailsEnabled) diff --git a/internal/catalog/localization_repo.go b/internal/catalog/localization_repo.go index c35afce8..9e5a3328 100644 --- a/internal/catalog/localization_repo.go +++ b/internal/catalog/localization_repo.go @@ -10,6 +10,13 @@ import ( "github.com/Silo-Server/silo-server/internal/models" ) +// The localization tables carry per-field provenance (provider | ai | manual) +// on their AI-writable fields (overview, tagline) so writes from different +// origins can never regress quality: manual beats provider beats ai. The +// precedence is enforced here, in single-statement upserts, so concurrent +// writers (a metadata refresh racing an AI translation job) cannot interleave +// a read-modify-write. + type MediaItemLocalizationRepository struct { pool *pgxpool.Pool } @@ -18,6 +25,11 @@ func NewMediaItemLocalizationRepository(pool *pgxpool.Pool) *MediaItemLocalizati return &MediaItemLocalizationRepository{pool: pool} } +// Upsert writes a provider localization. Title/sort-title/artwork fields are +// taken wholesale (provider data is authoritative for them); overview and +// tagline respect provenance — a manual value is never overwritten, and an +// empty incoming value never blanks an existing one (so a provider with no +// translated overview does not erase an AI or manual translation). func (r *MediaItemLocalizationRepository) Upsert(ctx context.Context, loc *models.MediaItemLocalization) error { if loc == nil || loc.ContentID == "" || loc.Language == "" { return fmt.Errorf("invalid media item localization") @@ -34,8 +46,22 @@ func (r *MediaItemLocalizationRepository) Upsert(ctx context.Context, loc *model ON CONFLICT (content_id, language) DO UPDATE SET title = EXCLUDED.title, sort_title = EXCLUDED.sort_title, - overview = EXCLUDED.overview, - tagline = EXCLUDED.tagline, + overview = CASE + WHEN media_item_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN media_item_localizations.overview + ELSE EXCLUDED.overview END, + overview_source = CASE + WHEN media_item_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN media_item_localizations.overview_source + ELSE 'provider' END, + tagline = CASE + WHEN media_item_localizations.tagline_source = 'manual' OR EXCLUDED.tagline = '' + THEN media_item_localizations.tagline + ELSE EXCLUDED.tagline END, + tagline_source = CASE + WHEN media_item_localizations.tagline_source = 'manual' OR EXCLUDED.tagline = '' + THEN media_item_localizations.tagline_source + ELSE 'provider' END, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, backdrop_path = EXCLUDED.backdrop_path, @@ -50,10 +76,69 @@ func (r *MediaItemLocalizationRepository) Upsert(ctx context.Context, loc *model return nil } +// UpsertAITranslation writes AI-translated overview/tagline values. A nil +// pointer leaves that field untouched. Per field, the write lands when the +// existing value is empty or already AI-sourced; force additionally overwrites +// provider values (the admin explicitly asked to re-translate). Manual values +// are never overwritten. Rows created here carry empty titles/artwork — the +// serving layer falls back to the base item for empty localized fields. +func (r *MediaItemLocalizationRepository) UpsertAITranslation(ctx context.Context, contentID, language string, overview, tagline *string, force bool) error { + if contentID == "" || language == "" { + return fmt.Errorf("invalid media item AI localization") + } + if overview == nil && tagline == nil { + return nil + } + + _, err := r.pool.Exec(ctx, ` + INSERT INTO media_item_localizations ( + content_id, language, title, sort_title, overview, tagline, + poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + overview_source, tagline_source + ) VALUES ( + $1, $2, '', '', COALESCE($3, ''), COALESCE($4, ''), + '', '', '', '', '', + CASE WHEN $3::text IS NULL THEN 'provider' ELSE 'ai' END, + CASE WHEN $4::text IS NULL THEN 'provider' ELSE 'ai' END + ) + ON CONFLICT (content_id, language) DO UPDATE SET + overview = CASE + WHEN $3::text IS NULL OR media_item_localizations.overview_source = 'manual' + THEN media_item_localizations.overview + WHEN $5 OR media_item_localizations.overview_source = 'ai' OR media_item_localizations.overview = '' + THEN EXCLUDED.overview + ELSE media_item_localizations.overview END, + overview_source = CASE + WHEN $3::text IS NULL OR media_item_localizations.overview_source = 'manual' + THEN media_item_localizations.overview_source + WHEN $5 OR media_item_localizations.overview_source = 'ai' OR media_item_localizations.overview = '' + THEN 'ai' + ELSE media_item_localizations.overview_source END, + tagline = CASE + WHEN $4::text IS NULL OR media_item_localizations.tagline_source = 'manual' + THEN media_item_localizations.tagline + WHEN $5 OR media_item_localizations.tagline_source = 'ai' OR media_item_localizations.tagline = '' + THEN EXCLUDED.tagline + ELSE media_item_localizations.tagline END, + tagline_source = CASE + WHEN $4::text IS NULL OR media_item_localizations.tagline_source = 'manual' + THEN media_item_localizations.tagline_source + WHEN $5 OR media_item_localizations.tagline_source = 'ai' OR media_item_localizations.tagline = '' + THEN 'ai' + ELSE media_item_localizations.tagline_source END, + updated_at = NOW() + `, contentID, language, overview, tagline, force) + if err != nil { + return fmt.Errorf("upserting media item AI localization: %w", err) + } + return nil +} + func (r *MediaItemLocalizationRepository) Get(ctx context.Context, contentID, language string) (*models.MediaItemLocalization, error) { row := r.pool.QueryRow(ctx, ` SELECT content_id, language, title, sort_title, overview, tagline, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + overview_source, tagline_source, created_at, updated_at FROM media_item_localizations WHERE content_id = $1 AND language = $2 @@ -69,6 +154,7 @@ func (r *MediaItemLocalizationRepository) GetByContentIDs(ctx context.Context, c rows, err := r.pool.Query(ctx, ` SELECT content_id, language, title, sort_title, overview, tagline, poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + overview_source, tagline_source, created_at, updated_at FROM media_item_localizations WHERE language = $1 AND content_id = ANY($2) @@ -98,6 +184,8 @@ func NewSeasonLocalizationRepository(pool *pgxpool.Pool) *SeasonLocalizationRepo return &SeasonLocalizationRepository{pool: pool} } +// Upsert writes a provider localization; overview respects provenance (see +// MediaItemLocalizationRepository.Upsert). func (r *SeasonLocalizationRepository) Upsert(ctx context.Context, loc *models.SeasonLocalization) error { if loc == nil || loc.SeasonContentID == "" || loc.Language == "" { return fmt.Errorf("invalid season localization") @@ -108,7 +196,14 @@ func (r *SeasonLocalizationRepository) Upsert(ctx context.Context, loc *models.S ) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (season_content_id, language) DO UPDATE SET title = EXCLUDED.title, - overview = EXCLUDED.overview, + overview = CASE + WHEN season_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN season_localizations.overview + ELSE EXCLUDED.overview END, + overview_source = CASE + WHEN season_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN season_localizations.overview_source + ELSE 'provider' END, poster_path = EXCLUDED.poster_path, poster_thumbhash = EXCLUDED.poster_thumbhash, updated_at = NOW() @@ -119,9 +214,41 @@ func (r *SeasonLocalizationRepository) Upsert(ctx context.Context, loc *models.S return nil } +// UpsertAIOverview writes an AI-translated season overview (see +// MediaItemLocalizationRepository.UpsertAITranslation for the precedence). +func (r *SeasonLocalizationRepository) UpsertAIOverview(ctx context.Context, seasonContentID, language, overview string, force bool) error { + if seasonContentID == "" || language == "" { + return fmt.Errorf("invalid season AI localization") + } + _, err := r.pool.Exec(ctx, ` + INSERT INTO season_localizations ( + season_content_id, language, title, overview, poster_path, poster_thumbhash, overview_source + ) VALUES ($1, $2, '', $3, '', '', 'ai') + ON CONFLICT (season_content_id, language) DO UPDATE SET + overview = CASE + WHEN season_localizations.overview_source = 'manual' + THEN season_localizations.overview + WHEN $4 OR season_localizations.overview_source = 'ai' OR season_localizations.overview = '' + THEN EXCLUDED.overview + ELSE season_localizations.overview END, + overview_source = CASE + WHEN season_localizations.overview_source = 'manual' + THEN season_localizations.overview_source + WHEN $4 OR season_localizations.overview_source = 'ai' OR season_localizations.overview = '' + THEN 'ai' + ELSE season_localizations.overview_source END, + updated_at = NOW() + `, seasonContentID, language, overview, force) + if err != nil { + return fmt.Errorf("upserting season AI localization: %w", err) + } + return nil +} + func (r *SeasonLocalizationRepository) Get(ctx context.Context, seasonContentID, language string) (*models.SeasonLocalization, error) { row := r.pool.QueryRow(ctx, ` - SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, created_at, updated_at + SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, + overview_source, created_at, updated_at FROM season_localizations WHERE season_content_id = $1 AND language = $2 `, seasonContentID, language) @@ -134,7 +261,8 @@ func (r *SeasonLocalizationRepository) GetBySeasonIDs(ctx context.Context, seaso return result, nil } rows, err := r.pool.Query(ctx, ` - SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, created_at, updated_at + SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, + overview_source, created_at, updated_at FROM season_localizations WHERE language = $1 AND season_content_id = ANY($2) `, language, seasonIDs) @@ -163,6 +291,8 @@ func NewEpisodeLocalizationRepository(pool *pgxpool.Pool) *EpisodeLocalizationRe return &EpisodeLocalizationRepository{pool: pool} } +// Upsert writes a provider localization; overview respects provenance (see +// MediaItemLocalizationRepository.Upsert). func (r *EpisodeLocalizationRepository) Upsert(ctx context.Context, loc *models.EpisodeLocalization) error { if loc == nil || loc.EpisodeContentID == "" || loc.Language == "" { return fmt.Errorf("invalid episode localization") @@ -172,7 +302,14 @@ func (r *EpisodeLocalizationRepository) Upsert(ctx context.Context, loc *models. VALUES ($1, $2, $3, $4) ON CONFLICT (episode_content_id, language) DO UPDATE SET title = EXCLUDED.title, - overview = EXCLUDED.overview, + overview = CASE + WHEN episode_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN episode_localizations.overview + ELSE EXCLUDED.overview END, + overview_source = CASE + WHEN episode_localizations.overview_source = 'manual' OR EXCLUDED.overview = '' + THEN episode_localizations.overview_source + ELSE 'provider' END, updated_at = NOW() `, loc.EpisodeContentID, loc.Language, loc.Title, loc.Overview) if err != nil { @@ -181,9 +318,39 @@ func (r *EpisodeLocalizationRepository) Upsert(ctx context.Context, loc *models. return nil } +// UpsertAIOverview writes an AI-translated episode overview (see +// MediaItemLocalizationRepository.UpsertAITranslation for the precedence). +func (r *EpisodeLocalizationRepository) UpsertAIOverview(ctx context.Context, episodeContentID, language, overview string, force bool) error { + if episodeContentID == "" || language == "" { + return fmt.Errorf("invalid episode AI localization") + } + _, err := r.pool.Exec(ctx, ` + INSERT INTO episode_localizations (episode_content_id, language, title, overview, overview_source) + VALUES ($1, $2, '', $3, 'ai') + ON CONFLICT (episode_content_id, language) DO UPDATE SET + overview = CASE + WHEN episode_localizations.overview_source = 'manual' + THEN episode_localizations.overview + WHEN $4 OR episode_localizations.overview_source = 'ai' OR episode_localizations.overview = '' + THEN EXCLUDED.overview + ELSE episode_localizations.overview END, + overview_source = CASE + WHEN episode_localizations.overview_source = 'manual' + THEN episode_localizations.overview_source + WHEN $4 OR episode_localizations.overview_source = 'ai' OR episode_localizations.overview = '' + THEN 'ai' + ELSE episode_localizations.overview_source END, + updated_at = NOW() + `, episodeContentID, language, overview, force) + if err != nil { + return fmt.Errorf("upserting episode AI localization: %w", err) + } + return nil +} + func (r *EpisodeLocalizationRepository) Get(ctx context.Context, episodeContentID, language string) (*models.EpisodeLocalization, error) { row := r.pool.QueryRow(ctx, ` - SELECT episode_content_id, language, title, overview, created_at, updated_at + SELECT episode_content_id, language, title, overview, overview_source, created_at, updated_at FROM episode_localizations WHERE episode_content_id = $1 AND language = $2 `, episodeContentID, language) @@ -196,7 +363,7 @@ func (r *EpisodeLocalizationRepository) GetByEpisodeIDs(ctx context.Context, epi return result, nil } rows, err := r.pool.Query(ctx, ` - SELECT episode_content_id, language, title, overview, created_at, updated_at + SELECT episode_content_id, language, title, overview, overview_source, created_at, updated_at FROM episode_localizations WHERE language = $1 AND episode_content_id = ANY($2) `, language, episodeIDs) @@ -231,6 +398,8 @@ func scanMediaItemLocalization(row pgx.Row) (*models.MediaItemLocalization, erro &loc.BackdropPath, &loc.BackdropThumbhash, &loc.LogoPath, + &loc.OverviewSource, + &loc.TaglineSource, &loc.CreatedAt, &loc.UpdatedAt, ); err != nil { @@ -251,6 +420,7 @@ func scanSeasonLocalization(row pgx.Row) (*models.SeasonLocalization, error) { &loc.Overview, &loc.PosterPath, &loc.PosterThumbhash, + &loc.OverviewSource, &loc.CreatedAt, &loc.UpdatedAt, ); err != nil { @@ -269,6 +439,7 @@ func scanEpisodeLocalization(row pgx.Row) (*models.EpisodeLocalization, error) { &loc.Language, &loc.Title, &loc.Overview, + &loc.OverviewSource, &loc.CreatedAt, &loc.UpdatedAt, ); err != nil { diff --git a/internal/catalog/localize.go b/internal/catalog/localize.go new file mode 100644 index 00000000..ad04babe --- /dev/null +++ b/internal/catalog/localize.go @@ -0,0 +1,75 @@ +package catalog + +import "github.com/Silo-Server/silo-server/internal/models" + +// applyItemLocalization merges a localization onto a clone of item. Only +// non-empty localized fields override the base — localization rows are +// legitimately partial (an AI translation carries only overview/tagline; a +// provider row may lack a tagline or logo), and an empty field must fall back +// to the base value rather than blank it. +func applyItemLocalization(item *models.MediaItem, loc *models.MediaItemLocalization) *models.MediaItem { + localized := cloneMediaItem(item) + if localized == nil || loc == nil { + return localized + } + if loc.Title != "" { + localized.Title = loc.Title + } + if loc.SortTitle != "" { + localized.SortTitle = loc.SortTitle + } + if loc.Overview != "" { + localized.Overview = loc.Overview + } + if loc.Tagline != "" { + localized.Tagline = loc.Tagline + } + if loc.PosterPath != "" { + localized.PosterPath = loc.PosterPath + localized.PosterThumbhash = loc.PosterThumbhash + } + if loc.BackdropPath != "" { + localized.BackdropPath = loc.BackdropPath + localized.BackdropThumbhash = loc.BackdropThumbhash + } + if loc.LogoPath != "" { + localized.LogoPath = loc.LogoPath + } + return localized +} + +// applySeasonLocalization merges a localization onto a clone of season; see +// applyItemLocalization for the empty-field semantics. +func applySeasonLocalization(season *models.Season, loc *models.SeasonLocalization) *models.Season { + localized := cloneSeason(season) + if localized == nil || loc == nil { + return localized + } + if loc.Title != "" { + localized.Title = loc.Title + } + if loc.Overview != "" { + localized.Overview = loc.Overview + } + if loc.PosterPath != "" { + localized.PosterPath = loc.PosterPath + localized.PosterThumbhash = loc.PosterThumbhash + } + return localized +} + +// applyEpisodeLocalization merges a localization onto a clone of episode; see +// applyItemLocalization for the empty-field semantics. +func applyEpisodeLocalization(episode *models.Episode, loc *models.EpisodeLocalization) *models.Episode { + localized := cloneEpisode(episode) + if localized == nil || loc == nil { + return localized + } + if loc.Title != "" { + localized.Title = loc.Title + } + if loc.Overview != "" { + localized.Overview = loc.Overview + } + return localized +} diff --git a/internal/catalog/localize_test.go b/internal/catalog/localize_test.go new file mode 100644 index 00000000..b4e9b123 --- /dev/null +++ b/internal/catalog/localize_test.go @@ -0,0 +1,109 @@ +package catalog + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func baseItem() *models.MediaItem { + return &models.MediaItem{ + ContentID: "c1", + Title: "Base Title", + SortTitle: "Base Title, The", + Overview: "Base overview.", + Tagline: "Base tagline.", + PosterPath: "posters/base.jpg", + PosterThumbhash: "hash-poster", + BackdropPath: "backdrops/base.jpg", + BackdropThumbhash: "hash-backdrop", + LogoPath: "logos/base.png", + } +} + +// An AI-created localization row carries only overview/tagline; everything +// else must fall back to the base item rather than be blanked. +func TestApplyItemLocalizationPartialRowFallsBackToBase(t *testing.T) { + loc := &models.MediaItemLocalization{ + ContentID: "c1", + Language: "fr", + Overview: "Résumé traduit.", + Tagline: "Slogan traduit.", + } + got := applyItemLocalization(baseItem(), loc) + + if got.Overview != "Résumé traduit." || got.Tagline != "Slogan traduit." { + t.Errorf("translated fields not applied: %q / %q", got.Overview, got.Tagline) + } + if got.Title != "Base Title" || got.SortTitle != "Base Title, The" { + t.Errorf("empty localized titles blanked the base: %q / %q", got.Title, got.SortTitle) + } + if got.PosterPath != "posters/base.jpg" || got.PosterThumbhash != "hash-poster" { + t.Errorf("empty localized poster blanked the base: %q / %q", got.PosterPath, got.PosterThumbhash) + } + if got.BackdropPath != "backdrops/base.jpg" || got.LogoPath != "logos/base.png" { + t.Errorf("empty localized artwork blanked the base: %q / %q", got.BackdropPath, got.LogoPath) + } +} + +func TestApplyItemLocalizationFullRowOverridesEverything(t *testing.T) { + loc := &models.MediaItemLocalization{ + ContentID: "c1", + Language: "fr", + Title: "Titre", + SortTitle: "Titre, Le", + Overview: "Résumé.", + Tagline: "Slogan.", + PosterPath: "posters/fr.jpg", + PosterThumbhash: "hash-fr", + BackdropPath: "backdrops/fr.jpg", + BackdropThumbhash: "hash-fr-bd", + LogoPath: "logos/fr.png", + } + got := applyItemLocalization(baseItem(), loc) + if got.Title != "Titre" || got.Overview != "Résumé." || got.PosterPath != "posters/fr.jpg" || + got.PosterThumbhash != "hash-fr" || got.LogoPath != "logos/fr.png" { + t.Errorf("full localization not applied: %+v", got) + } +} + +func TestApplyItemLocalizationDoesNotMutateBase(t *testing.T) { + item := baseItem() + _ = applyItemLocalization(item, &models.MediaItemLocalization{Title: "Titre"}) + if item.Title != "Base Title" { + t.Errorf("base item mutated: %q", item.Title) + } +} + +func TestApplyItemLocalizationNilLocalizationClones(t *testing.T) { + item := baseItem() + got := applyItemLocalization(item, nil) + if got == item { + t.Fatal("expected a clone, got the same pointer") + } + if got.Title != item.Title { + t.Errorf("clone differs: %q", got.Title) + } +} + +func TestApplySeasonLocalizationPartialRow(t *testing.T) { + season := &models.Season{ContentID: "s1", Title: "Season 1", Overview: "Base.", PosterPath: "p.jpg", PosterThumbhash: "h"} + got := applySeasonLocalization(season, &models.SeasonLocalization{Overview: "Saison résumé."}) + if got.Overview != "Saison résumé." { + t.Errorf("overview not applied: %q", got.Overview) + } + if got.Title != "Season 1" || got.PosterPath != "p.jpg" || got.PosterThumbhash != "h" { + t.Errorf("empty fields blanked the base: %+v", got) + } +} + +func TestApplyEpisodeLocalizationPartialRow(t *testing.T) { + ep := &models.Episode{ContentID: "e1", Title: "Pilot", Overview: "Base."} + got := applyEpisodeLocalization(ep, &models.EpisodeLocalization{Overview: "Épisode résumé."}) + if got.Overview != "Épisode résumé." { + t.Errorf("overview not applied: %q", got.Overview) + } + if got.Title != "Pilot" { + t.Errorf("empty localized title blanked the base: %q", got.Title) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 6847c133..f7824808 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -246,18 +246,43 @@ type RecommendationsConfig struct { CowatchCron string `yaml:"-"` } -// SubtitleAIConfig holds settings for on-demand AI subtitle translation (and, -// in a follow-up, Whisper ASR generation) via a single OpenAI-compatible -// endpoint — the operator can point it at OpenAI, Groq, a local Ollama server, -// etc. Mirrors the recommendations embedding client's configuration style. -type SubtitleAIConfig struct { - Enabled bool `yaml:"-"` +// AIConfig holds the shared connection settings for Silo's AI features +// (subtitle translation, metadata translation, Whisper ASR): one +// OpenAI-compatible endpoint the operator can point at OpenAI, Groq, a local +// Ollama/llama.cpp server, etc., with an optional separate endpoint for audio +// transcription (operators often run a dedicated Whisper server). Settings +// load from the ai.* keys, falling back to the legacy subtitle_ai.* rows. +type AIConfig struct { BaseURL string `yaml:"-"` APIKey string `yaml:"-"` ChatModel string `yaml:"-"` - MaxConcurrentJobs int `yaml:"-"` - BatchSize int `yaml:"-"` - ContextNeighbors int `yaml:"-"` + ASRBaseURL string `yaml:"-"` // empty = BaseURL + ASRAPIKey string `yaml:"-"` // empty = APIKey + ASRModel string `yaml:"-"` + MaxConcurrentJobs int `yaml:"-"` // one shared bound across all AI job services +} + +// SubtitleAIConfig holds the subtitle-AI feature toggles and tuning. The +// endpoint connection lives in AIConfig. +type SubtitleAIConfig struct { + Enabled bool `yaml:"-"` // on-demand translation + TranscribeEnabled bool `yaml:"-"` // Whisper ASR generation + BatchSize int `yaml:"-"` + ContextNeighbors int `yaml:"-"` + // ASRChunkSeconds is the audio chunk length per transcription request + // (60..600). Shorter chunks bound Whisper timestamp drift on long files; + // longer chunks mean fewer requests and fewer boundary word-clips. + ASRChunkSeconds int `yaml:"-"` +} + +// MetadataAIConfig holds the metadata translation feature toggles. The +// endpoint connection lives in AIConfig. +type MetadataAIConfig struct { + Enabled bool `yaml:"-"` + // OnView controls viewer-triggered description translation on detail + // pages: "off" (default), "button" (explicit chip), or "auto" (translate + // on view with a loading animation). + OnView string `yaml:"-"` } // DownloadConfig holds server-wide download policy settings. @@ -291,7 +316,9 @@ type Config struct { JellyfinCompat JellyfinCompatConfig `yaml:"-"` AudiobookshelfCompat AudiobookshelfCompatConfig `yaml:"-"` Recommendations RecommendationsConfig `yaml:"-"` + AI AIConfig `yaml:"-"` SubtitleAI SubtitleAIConfig `yaml:"-"` + MetadataAI MetadataAIConfig `yaml:"-"` Download DownloadConfig `yaml:"-"` TMDBAPIKey string `yaml:"-"` MDBListAPIKey string `yaml:"-"` diff --git a/internal/config/db_loader.go b/internal/config/db_loader.go index 9b16b074..fcbd682c 100644 --- a/internal/config/db_loader.go +++ b/internal/config/db_loader.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "log/slog" "strconv" "time" @@ -422,20 +423,38 @@ func LoadFromDB(m map[string]string) (*Config, error) { cfg.Recommendations.DiversityLambda = diversityLambda cfg.Recommendations.CowatchCron = stringOr(m, "recommendations.cowatch_cron", "30 4 * * *") - // Subtitle AI (on-demand translation; Whisper ASR generation in a follow-up) + // Shared AI endpoint (subtitle translation, metadata translation, Whisper + // ASR). The connection settings read ai.* with a fallback to the legacy + // subtitle_ai.* rows; the legacy rows are never renamed in SQL because + // encrypted values are GCM-bound to their setting key. + cfg.AI.BaseURL = stringOr(m, "ai.base_url", stringOr(m, "subtitle_ai.base_url", "https://api.openai.com")) + cfg.AI.APIKey = stringOr(m, "ai.api_key", stringOr(m, "subtitle_ai.api_key", "")) + cfg.AI.ChatModel = stringOr(m, "ai.chat_model", stringOr(m, "subtitle_ai.chat_model", "gpt-4o-mini")) + cfg.AI.ASRBaseURL = stringOr(m, "ai.asr_base_url", "") + cfg.AI.ASRAPIKey = stringOr(m, "ai.asr_api_key", "") + cfg.AI.ASRModel = stringOr(m, "ai.asr_model", "whisper-1") + aiMaxConcurrent, err := intOr(m, "ai.max_concurrent_jobs", 0) + if err != nil { + return nil, err + } + if aiMaxConcurrent <= 0 { + if aiMaxConcurrent, err = intOr(m, "subtitle_ai.max_concurrent_jobs", 2); err != nil { + return nil, err + } + } + cfg.AI.MaxConcurrentJobs = aiMaxConcurrent + + // Subtitle AI feature toggles and tuning. subtitleAIEnabled, err := boolOr(m, "subtitle_ai.enabled", false) if err != nil { return nil, err } cfg.SubtitleAI.Enabled = subtitleAIEnabled - cfg.SubtitleAI.BaseURL = stringOr(m, "subtitle_ai.base_url", "https://api.openai.com") - cfg.SubtitleAI.APIKey = stringOr(m, "subtitle_ai.api_key", "") - cfg.SubtitleAI.ChatModel = stringOr(m, "subtitle_ai.chat_model", "gpt-4o-mini") - subtitleAIMaxConcurrent, err := intOr(m, "subtitle_ai.max_concurrent_jobs", 2) + subtitleAITranscribe, err := boolOr(m, "subtitle_ai.transcribe_enabled", false) if err != nil { return nil, err } - cfg.SubtitleAI.MaxConcurrentJobs = subtitleAIMaxConcurrent + cfg.SubtitleAI.TranscribeEnabled = subtitleAITranscribe subtitleAIBatchSize, err := intOr(m, "subtitle_ai.batch_size", 40) if err != nil { return nil, err @@ -446,6 +465,26 @@ func LoadFromDB(m map[string]string) (*Config, error) { return nil, err } cfg.SubtitleAI.ContextNeighbors = subtitleAIContextNeighbors + subtitleAIChunkSeconds, err := intOr(m, "subtitle_ai.asr_chunk_seconds", 600) + if err != nil { + return nil, err + } + cfg.SubtitleAI.ASRChunkSeconds = subtitleAIChunkSeconds + + // Metadata AI translation feature toggles. + metadataAIEnabled, err := boolOr(m, "metadata_ai.enabled", false) + if err != nil { + return nil, err + } + cfg.MetadataAI.Enabled = metadataAIEnabled + switch onView := stringOr(m, "metadata_ai.on_view", "off"); onView { + case "off", "button", "auto": + cfg.MetadataAI.OnView = onView + default: + // A bad row must not block startup; the feature just stays off. + slog.Warn("invalid metadata_ai.on_view setting; using off", "value", onView) + cfg.MetadataAI.OnView = "off" + } // Download downloadEnabled, err := boolOr(m, "download.enabled", false) diff --git a/internal/jellycompat/access_filter.go b/internal/jellycompat/access_filter.go index ec71c2c9..b26dd390 100644 --- a/internal/jellycompat/access_filter.go +++ b/internal/jellycompat/access_filter.go @@ -43,12 +43,13 @@ func NewScopeAccessFilter(resolver ScopeResolver) AccessFilterResolver { } } return catalog.AccessFilter{ - AllowedLibraryIDs: scope.AllowedLibraryIDs, - DisabledLibraryIDs: scope.DisabledLibraryIDs, - MaxContentRating: scope.MaxContentRating, - MaxPlaybackQuality: scope.MaxPlaybackQuality, - UserID: userID, - ProfileID: profileID, + AllowedLibraryIDs: scope.AllowedLibraryIDs, + DisabledLibraryIDs: scope.DisabledLibraryIDs, + MaxContentRating: scope.MaxContentRating, + MaxPlaybackQuality: scope.MaxPlaybackQuality, + ProfilePreferredLanguage: scope.PreferredMetadataLanguage, + UserID: userID, + ProfileID: profileID, } } } diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 754db16c..2f754e31 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -174,6 +174,14 @@ type metadataFolderRepo interface { GetByID(ctx context.Context, id int) (*models.MediaFolder, error) } +// AutoTranslator is the seam to the metadata AI translation service: after a +// refresh, libraries that opted in get missing localizations filled by AI. +// Implemented by *translation.Service; AutoEnqueue must be cheap and must +// never fail the refresh (it logs its own errors). +type AutoTranslator interface { + AutoEnqueue(ctx context.Context, itemContentID, language string) +} + type metadataContentFileLister interface { GetByContentID(ctx context.Context, contentID string) ([]*models.MediaFile, error) } @@ -280,6 +288,7 @@ type MetadataService struct { itemLocalizationRepo *catalog.MediaItemLocalizationRepository seasonLocalizationRepo *catalog.SeasonLocalizationRepository episodeLocalizationRepo *catalog.EpisodeLocalizationRepository + autoTranslator AutoTranslator // optional; set via SetAutoTranslator personRepo *catalog.PersonRepository fileRepo FileContentUpdater skippedRootRepo metadataSkippedRootRepo @@ -414,6 +423,12 @@ func NewMetadataService( } // SetImageCacher enables S3 image caching during metadata persistence. +// SetAutoTranslator wires the metadata AI translation fallback. Optional; +// without it, refreshes simply skip the auto-translate hook. +func (s *MetadataService) SetAutoTranslator(t AutoTranslator) { + s.autoTranslator = t +} + func (s *MetadataService) SetImageCacher(c ImageCacher) { s.imageCacher = c } @@ -514,9 +529,38 @@ func (s *MetadataService) Process(ctx context.Context, req ProcessRequest) (*Pro final.Updated = final.Updated || result.Updated } + s.maybeAutoTranslate(ctx, folderID, final.ContentID) + return &final, nil } +// maybeAutoTranslate queues an AI translation for libraries that opted in, +// when the item's default metadata language differs from the library's. The +// translation service itself checks whether anything is actually missing, so +// this fires cheaply on every refresh. Runs in the background — a refresh +// never waits on (or fails because of) the translation seam. +func (s *MetadataService) maybeAutoTranslate(ctx context.Context, folderID int, contentID string) { + if s.autoTranslator == nil || contentID == "" || folderID <= 0 || s.folderRepo == nil || s.itemRepo == nil { + return + } + folder, err := s.folderRepo.GetByID(ctx, folderID) + if err != nil || folder == nil || !folder.AutoTranslateMetadata { + return + } + library := strings.TrimSpace(folder.MetadataLanguage) + if library == "" { + return + } + item, err := s.itemRepo.GetByID(ctx, contentID) + if err != nil || item == nil { + return + } + if strings.EqualFold(strings.TrimSpace(item.DefaultMetadataLanguage), library) { + return + } + go s.autoTranslator.AutoEnqueue(context.WithoutCancel(ctx), contentID, library) +} + func parseProcessFolderID(raw string) int { folderID := 0 if raw != "" { diff --git a/internal/metadata/translation/job.go b/internal/metadata/translation/job.go new file mode 100644 index 00000000..39b68f4c --- /dev/null +++ b/internal/metadata/translation/job.go @@ -0,0 +1,139 @@ +// Package translation translates catalog descriptions (overviews, taglines) +// into the localization tables with an OpenAI-compatible chat model, via the +// shared AI core. Jobs are triggered manually from the metadata editor or +// automatically after a metadata refresh when the library opts in and the +// providers had no localization for its language. Results are ordinary +// localization rows (marked source='ai'), so every client receives them +// through the existing localized detail responses with no client changes. +package translation + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "time" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" +) + +var ( + // ErrNotConfigured is returned when metadata translation is requested but + // the feature is disabled or the AI endpoint is missing required settings. + ErrNotConfigured = errors.New("metadata AI translation is not configured") + // ErrInvalidRequest wraps caller-input validation failures so handlers can + // map them to 400 rather than 500. + ErrInvalidRequest = errors.New("invalid metadata translation request") + // ErrJobNotFound is returned for unknown job IDs. + ErrJobNotFound = errors.New("metadata translation job not found") +) + +// TargetKind identifies what a job translates. +type TargetKind string + +const ( + // TargetItem covers a movie or series; for series, IncludeChildren expands + // the job to every season and episode overview. + TargetItem TargetKind = "item" + TargetSeason TargetKind = "season" + TargetEpisode TargetKind = "episode" +) + +// JobStatus is the lifecycle state of a job, shared with the other AI job +// services via jobrunner. +type JobStatus = jobrunner.Status + +// Job is a persisted metadata translation job. It is serialized to the API +// as-is. +type Job struct { + ID int64 `json:"id"` + TargetKind TargetKind `json:"target_kind"` + ContentID string `json:"content_id"` + IncludeChildren bool `json:"include_children"` + SourceLanguage string `json:"source_language"` + TargetLanguage string `json:"target_language"` + Engine string `json:"engine"` + Model string `json:"model"` + Status JobStatus `json:"status"` + Progress float64 `json:"progress"` + ProgressMessage string `json:"progress_message"` + FieldsDone int `json:"fields_done"` + FieldsTotal int `json:"fields_total"` + Force bool `json:"force"` + ErrorMessage string `json:"error_message,omitempty"` + IdempotencyKey string `json:"-"` + RequestedBy *int `json:"-"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + HeartbeatAt time.Time `json:"-"` +} + +// JobRequest is the input to Service.Enqueue. +type JobRequest struct { + TargetKind TargetKind + ContentID string + TargetLanguage string + IncludeChildren bool + // Force re-translates fields that already have a provider or AI value. + // Manual values are never overwritten regardless. + Force bool + RequestedBy *int +} + +// JobRepository persists metadata translation jobs. +type JobRepository interface { + InsertJob(ctx context.Context, job *Job) error + GetJob(ctx context.Context, id int64) (*Job, error) + // GetActiveJobByIdempotencyKey returns a pending/running job with the given + // key, or nil if none exists. + GetActiveJobByIdempotencyKey(ctx context.Context, key string) (*Job, error) + ListJobsByContent(ctx context.Context, contentID string) ([]Job, error) + UpdateProgress(ctx context.Context, id int64, status JobStatus, progress float64, message string, fieldsDone, fieldsTotal int) error + CompleteJob(ctx context.Context, id int64, message string, fieldsDone, fieldsTotal int) error + FailJob(ctx context.Context, id int64, status JobStatus, message string) error + Heartbeat(ctx context.Context, id int64) error + ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error) +} + +// ItemText is the translatable base-row text of a movie or series. +type ItemText struct { + ContentID string + Type string + Title string + Year int + Overview string + Tagline string + DefaultLanguage string +} + +// ChildText is the translatable base-row text of a season or episode. +// EpisodeNumber is zero for seasons. +type ChildText struct { + ContentID string + SeasonNumber int + EpisodeNumber int + Overview string +} + +// ContentReader loads the base-row text a job translates. Implementations +// return nil (not an error) for missing rows. +type ContentReader interface { + ItemText(ctx context.Context, contentID string) (*ItemText, error) + SeasonTexts(ctx context.Context, seriesID string) ([]ChildText, error) + EpisodeTexts(ctx context.Context, seriesID string) ([]ChildText, error) + // SeasonByID / EpisodeByID also return the parent series content ID. + SeasonByID(ctx context.Context, contentID string) (*ChildText, string, error) + EpisodeByID(ctx context.Context, contentID string) (*ChildText, string, error) + // CountMissingFields counts base fields (item overview/tagline, season and + // episode overviews) that have source text but no localized value for the + // language — the auto-translate trigger condition. + CountMissingFields(ctx context.Context, itemContentID, language string) (int, error) +} + +// idempotencyKey derives the dedup key for a job. Two requests for the same +// target, language, and model collapse to one in-flight job. +func idempotencyKey(kind TargetKind, contentID, targetLang, model string) string { + sum := sha256.Sum256(fmt.Appendf(nil, "%s|%s|%s|%s", kind, contentID, targetLang, model)) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/metadata/translation/pgrepo.go b/internal/metadata/translation/pgrepo.go new file mode 100644 index 00000000..0f004ccd --- /dev/null +++ b/internal/metadata/translation/pgrepo.go @@ -0,0 +1,279 @@ +package translation + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" +) + +// PgRepository implements JobRepository and ContentReader on PostgreSQL. +type PgRepository struct { + pool *pgxpool.Pool +} + +// NewPgRepository creates a Postgres-backed repository. +func NewPgRepository(pool *pgxpool.Pool) *PgRepository { + return &PgRepository{pool: pool} +} + +const jobColumns = `id, target_kind, content_id, include_children, source_language, target_language, + engine, model, status, progress, progress_message, fields_done, fields_total, force, + error_message, idempotency_key, requested_by, created_at, updated_at, heartbeat_at` + +func scanJob(row pgx.Row) (*Job, error) { + var j Job + err := row.Scan( + &j.ID, &j.TargetKind, &j.ContentID, &j.IncludeChildren, &j.SourceLanguage, &j.TargetLanguage, + &j.Engine, &j.Model, &j.Status, &j.Progress, &j.ProgressMessage, &j.FieldsDone, &j.FieldsTotal, &j.Force, + &j.ErrorMessage, &j.IdempotencyKey, &j.RequestedBy, &j.CreatedAt, &j.UpdatedAt, &j.HeartbeatAt, + ) + if err != nil { + return nil, err + } + return &j, nil +} + +func (r *PgRepository) InsertJob(ctx context.Context, job *Job) error { + if job.Engine == "" { + job.Engine = "openai" + } + if job.Status == "" { + job.Status = jobrunner.StatusPending + } + return r.pool.QueryRow(ctx, + `INSERT INTO metadata_translation_jobs + (target_kind, content_id, include_children, source_language, target_language, + engine, model, status, progress, progress_message, force, idempotency_key, requested_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) + RETURNING id, created_at, updated_at, heartbeat_at`, + job.TargetKind, job.ContentID, job.IncludeChildren, job.SourceLanguage, job.TargetLanguage, + job.Engine, job.Model, job.Status, job.Progress, job.ProgressMessage, job.Force, job.IdempotencyKey, job.RequestedBy, + ).Scan(&job.ID, &job.CreatedAt, &job.UpdatedAt, &job.HeartbeatAt) +} + +func (r *PgRepository) GetJob(ctx context.Context, id int64) (*Job, error) { + job, err := scanJob(r.pool.QueryRow(ctx, + `SELECT `+jobColumns+` FROM metadata_translation_jobs WHERE id = $1`, id)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get metadata translation job: %w", err) + } + return job, nil +} + +func (r *PgRepository) GetActiveJobByIdempotencyKey(ctx context.Context, key string) (*Job, error) { + job, err := scanJob(r.pool.QueryRow(ctx, + `SELECT `+jobColumns+` FROM metadata_translation_jobs + WHERE idempotency_key = $1 AND status IN ('pending', 'running') + ORDER BY created_at DESC LIMIT 1`, key)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("get active metadata translation job: %w", err) + } + return job, nil +} + +func (r *PgRepository) ListJobsByContent(ctx context.Context, contentID string) ([]Job, error) { + rows, err := r.pool.Query(ctx, + `SELECT `+jobColumns+` FROM metadata_translation_jobs + WHERE content_id = $1 ORDER BY created_at DESC LIMIT 50`, contentID) + if err != nil { + return nil, fmt.Errorf("list metadata translation jobs: %w", err) + } + defer rows.Close() + + var jobs []Job + for rows.Next() { + job, err := scanJob(rows) + if err != nil { + return nil, fmt.Errorf("scan metadata translation job: %w", err) + } + jobs = append(jobs, *job) + } + return jobs, rows.Err() +} + +// UpdateProgress, CompleteJob, and FailJob only transition a job that is still +// active ("pending"/"running"). The guard makes them no-ops on an already +// terminal row, so a job that was cancelled or reaped as stale can never be +// resurrected by a late write from its own worker goroutine. +func (r *PgRepository) UpdateProgress(ctx context.Context, id int64, status JobStatus, progress float64, message string, fieldsDone, fieldsTotal int) error { + _, err := r.pool.Exec(ctx, + `UPDATE metadata_translation_jobs + SET status = $2, progress = $3, progress_message = $4, fields_done = $5, fields_total = $6, + updated_at = now(), heartbeat_at = now() + WHERE id = $1 AND status IN ('pending', 'running')`, id, status, progress, message, fieldsDone, fieldsTotal) + if err != nil { + return fmt.Errorf("update metadata translation job progress: %w", err) + } + return nil +} + +func (r *PgRepository) CompleteJob(ctx context.Context, id int64, message string, fieldsDone, fieldsTotal int) error { + _, err := r.pool.Exec(ctx, + `UPDATE metadata_translation_jobs + SET status = 'completed', progress = 1, progress_message = $2, fields_done = $3, fields_total = $4, + error_message = '', updated_at = now(), heartbeat_at = now() + WHERE id = $1 AND status IN ('pending', 'running')`, id, message, fieldsDone, fieldsTotal) + if err != nil { + return fmt.Errorf("complete metadata translation job: %w", err) + } + return nil +} + +func (r *PgRepository) FailJob(ctx context.Context, id int64, status JobStatus, message string) error { + _, err := r.pool.Exec(ctx, + `UPDATE metadata_translation_jobs + SET status = $2, error_message = $3, updated_at = now(), heartbeat_at = now() + WHERE id = $1 AND status IN ('pending', 'running')`, id, status, message) + if err != nil { + return fmt.Errorf("fail metadata translation job: %w", err) + } + return nil +} + +func (r *PgRepository) Heartbeat(ctx context.Context, id int64) error { + _, err := r.pool.Exec(ctx, + `UPDATE metadata_translation_jobs SET heartbeat_at = now() WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("heartbeat metadata translation job: %w", err) + } + return nil +} + +func (r *PgRepository) ResetStaleJobs(ctx context.Context, before time.Time, message string) (int64, error) { + tag, err := r.pool.Exec(ctx, + `UPDATE metadata_translation_jobs + SET status = 'failed', error_message = $1, updated_at = now() + WHERE status IN ('pending', 'running') AND heartbeat_at < $2`, message, before) + if err != nil { + return 0, fmt.Errorf("reset stale metadata translation jobs: %w", err) + } + return tag.RowsAffected(), nil +} + +func (r *PgRepository) ItemText(ctx context.Context, contentID string) (*ItemText, error) { + var item ItemText + err := r.pool.QueryRow(ctx, + `SELECT content_id, type, title, COALESCE(year, 0), COALESCE(overview, ''), COALESCE(tagline, ''), + COALESCE(default_metadata_language, '') + FROM media_items WHERE content_id = $1`, contentID). + Scan(&item.ContentID, &item.Type, &item.Title, &item.Year, &item.Overview, &item.Tagline, &item.DefaultLanguage) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("load item text: %w", err) + } + return &item, nil +} + +func (r *PgRepository) SeasonTexts(ctx context.Context, seriesID string) ([]ChildText, error) { + rows, err := r.pool.Query(ctx, + `SELECT content_id, season_number, COALESCE(overview, '') + FROM seasons WHERE series_id = $1 ORDER BY season_number`, seriesID) + if err != nil { + return nil, fmt.Errorf("load season texts: %w", err) + } + defer rows.Close() + var out []ChildText + for rows.Next() { + var c ChildText + if err := rows.Scan(&c.ContentID, &c.SeasonNumber, &c.Overview); err != nil { + return nil, fmt.Errorf("scan season text: %w", err) + } + out = append(out, c) + } + return out, rows.Err() +} + +func (r *PgRepository) EpisodeTexts(ctx context.Context, seriesID string) ([]ChildText, error) { + rows, err := r.pool.Query(ctx, + `SELECT content_id, season_number, episode_number, COALESCE(overview, '') + FROM episodes WHERE series_id = $1 ORDER BY season_number, episode_number`, seriesID) + if err != nil { + return nil, fmt.Errorf("load episode texts: %w", err) + } + defer rows.Close() + var out []ChildText + for rows.Next() { + var c ChildText + if err := rows.Scan(&c.ContentID, &c.SeasonNumber, &c.EpisodeNumber, &c.Overview); err != nil { + return nil, fmt.Errorf("scan episode text: %w", err) + } + out = append(out, c) + } + return out, rows.Err() +} + +func (r *PgRepository) SeasonByID(ctx context.Context, contentID string) (*ChildText, string, error) { + var c ChildText + var seriesID string + err := r.pool.QueryRow(ctx, + `SELECT content_id, series_id, season_number, COALESCE(overview, '') + FROM seasons WHERE content_id = $1`, contentID). + Scan(&c.ContentID, &seriesID, &c.SeasonNumber, &c.Overview) + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", nil + } + if err != nil { + return nil, "", fmt.Errorf("load season: %w", err) + } + return &c, seriesID, nil +} + +func (r *PgRepository) EpisodeByID(ctx context.Context, contentID string) (*ChildText, string, error) { + var c ChildText + var seriesID string + err := r.pool.QueryRow(ctx, + `SELECT content_id, series_id, season_number, episode_number, COALESCE(overview, '') + FROM episodes WHERE content_id = $1`, contentID). + Scan(&c.ContentID, &seriesID, &c.SeasonNumber, &c.EpisodeNumber, &c.Overview) + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", nil + } + if err != nil { + return nil, "", fmt.Errorf("load episode: %w", err) + } + return &c, seriesID, nil +} + +// CountMissingFields counts base fields with source text but no localized +// value for the language. One round-trip; used to gate the auto-translate +// enqueue so repeat refreshes of fully translated items are free. +func (r *PgRepository) CountMissingFields(ctx context.Context, itemContentID, language string) (int, error) { + var missing int + err := r.pool.QueryRow(ctx, ` + SELECT + (SELECT count(*) FROM media_items mi + LEFT JOIN media_item_localizations l ON l.content_id = mi.content_id AND l.language = $2 + WHERE mi.content_id = $1 + AND COALESCE(mi.overview, '') <> '' AND COALESCE(l.overview, '') = '') + + (SELECT count(*) FROM media_items mi + LEFT JOIN media_item_localizations l ON l.content_id = mi.content_id AND l.language = $2 + WHERE mi.content_id = $1 + AND COALESCE(mi.tagline, '') <> '' AND COALESCE(l.tagline, '') = '') + + (SELECT count(*) FROM seasons s + LEFT JOIN season_localizations sl ON sl.season_content_id = s.content_id AND sl.language = $2 + WHERE s.series_id = $1 + AND COALESCE(s.overview, '') <> '' AND COALESCE(sl.overview, '') = '') + + (SELECT count(*) FROM episodes e + LEFT JOIN episode_localizations el ON el.episode_content_id = e.content_id AND el.language = $2 + WHERE e.series_id = $1 + AND COALESCE(e.overview, '') <> '' AND COALESCE(el.overview, '') = '') + `, itemContentID, language).Scan(&missing) + if err != nil { + return 0, fmt.Errorf("count missing localization fields: %w", err) + } + return missing, nil +} diff --git a/internal/metadata/translation/prompt.go b/internal/metadata/translation/prompt.go new file mode 100644 index 00000000..bc074f1a --- /dev/null +++ b/internal/metadata/translation/prompt.go @@ -0,0 +1,34 @@ +package translation + +import "fmt" + +// metadataBatchSize is how many fields are translated per chat request. +// Smaller than the subtitle batch size — these are paragraphs, not cue lines — +// while still batching a series' episode overviews together so the model keeps +// terminology consistent across them. +const metadataBatchSize = 10 + +// systemPrompt builds the translation system prompt. title/year ground the +// model in which work the descriptions belong to so names and references stay +// untranslated and consistent. +func systemPrompt(srcName, tgtName, title string, year int) string { + src := srcName + if src == "" { + src = "the source language" + } + work := title + if year > 0 { + work = fmt.Sprintf("%s (%d)", title, year) + } + return fmt.Sprintf( + "You are a professional translator for a media catalog. Translate plot summaries and taglines "+ + "for %q from %s into %s. Produce natural, idiomatic %s that preserves meaning and tone, keeps "+ + "proper nouns, character names, and place names as they are conventionally rendered in %s, and "+ + "adds no information that is not in the source. "+ + "You receive a JSON object whose keys are entry numbers and whose values are the source text. "+ + "Respond with ONLY a JSON object using the exact same keys, where each value is the translation "+ + "of that entry. Do not add, remove, merge, or renumber entries, and do not output anything "+ + "except the JSON object.", + work, src, tgtName, tgtName, tgtName, + ) +} diff --git a/internal/metadata/translation/service.go b/internal/metadata/translation/service.go new file mode 100644 index 00000000..0297b883 --- /dev/null +++ b/internal/metadata/translation/service.go @@ -0,0 +1,593 @@ +package translation + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" + "github.com/Silo-Server/silo-server/internal/ai/llm" + aitranslate "github.com/Silo-Server/silo-server/internal/ai/translate" + "github.com/Silo-Server/silo-server/internal/subtitles" +) + +// Config holds the runtime configuration for metadata translation. The shared +// endpoint connection lives in the llm client this service is wired with. +type Config struct { + Enabled bool + Configured bool // shared AI endpoint has a base URL + ChatModel string // for job provenance and idempotency + // OnView controls viewer-triggered translation: "off" | "button" | "auto". + OnView string +} + +// Ready reports whether metadata translation can currently run. +func (c Config) Ready() bool { return c.Enabled && c.Configured && c.ChatModel != "" } + +// OnViewMode reports the viewer-triggered translation mode ("off" when the +// feature itself is not ready). +func (c Config) OnViewMode() string { + if !c.Ready() || c.OnView == "" { + return "off" + } + return c.OnView +} + +// onViewFailureCooldown suppresses re-enqueueing after a failed job for the +// same target+language, so a broken endpoint is not retried on every page +// view. Admin-triggered jobs are unaffected. +const onViewFailureCooldown = 15 * time.Minute + +// Service owns the metadata translation job semantics: enqueue validation, +// field collection with skip-if-filled, batched translation, and +// provenance-aware persistence. Lifecycle mechanics are delegated to the +// shared jobrunner. +type Service struct { + cfg Config + repo JobRepository + content ContentReader + locs LocalizationStore + chat aitranslate.ChatFn + runner *jobrunner.Runner + logger *slog.Logger +} + +// NewService wires a metadata translation service. sem is the dispatch +// semaphore shared with the other AI job services. +func NewService( + appCtx context.Context, + cfg Config, + repo JobRepository, + content ContentReader, + locs LocalizationStore, + chat aitranslate.ChatFn, + sem chan struct{}, + logger *slog.Logger, +) *Service { + if logger == nil { + logger = slog.Default() + } + return &Service{ + cfg: cfg, + repo: repo, + content: content, + locs: locs, + chat: chat, + runner: jobrunner.New(appCtx, sem, repo, "metadata translation", logger), + logger: logger, + } +} + +// Enabled reports whether metadata translation can currently run. +func (s *Service) Enabled() bool { return s.cfg.Ready() } + +// Recover clears jobs orphaned by a crashed worker and starts a background +// reaper that keeps doing so. Call once at startup. +func (s *Service) Recover() { s.runner.Recover() } + +// Enqueue validates and queues a job, returning immediately. If an identical +// job is already pending or running, that job is returned instead of a new one. +func (s *Service) Enqueue(ctx context.Context, req JobRequest) (*Job, error) { + if !s.cfg.Ready() { + return nil, ErrNotConfigured + } + if req.TargetKind == "" { + req.TargetKind = TargetItem + } + switch req.TargetKind { + case TargetItem, TargetSeason, TargetEpisode: + default: + return nil, fmt.Errorf("%w: unsupported target kind %q", ErrInvalidRequest, req.TargetKind) + } + if strings.TrimSpace(req.ContentID) == "" { + return nil, fmt.Errorf("%w: content id is required", ErrInvalidRequest) + } + target, err := subtitles.NormalizeLanguageCode(req.TargetLanguage) + if err != nil { + return nil, fmt.Errorf("%w: invalid target language %q", ErrInvalidRequest, req.TargetLanguage) + } + req.TargetLanguage = target + + key := idempotencyKey(req.TargetKind, req.ContentID, req.TargetLanguage, s.cfg.ChatModel) + if existing, err := s.repo.GetActiveJobByIdempotencyKey(ctx, key); err != nil { + return nil, err + } else if existing != nil { + return existing, nil + } + + job := &Job{ + TargetKind: req.TargetKind, + ContentID: req.ContentID, + IncludeChildren: req.IncludeChildren, + TargetLanguage: req.TargetLanguage, + Engine: "openai", + Model: s.cfg.ChatModel, + Status: jobrunner.StatusPending, + ProgressMessage: "Queued", + Force: req.Force, + IdempotencyKey: key, + RequestedBy: req.RequestedBy, + } + if err := s.repo.InsertJob(ctx, job); err != nil { + // A racing duplicate trips the partial unique index; return the winner. + if existing, lookupErr := s.repo.GetActiveJobByIdempotencyKey(ctx, key); lookupErr == nil && existing != nil { + return existing, nil + } + return nil, err + } + + s.dispatch(*job) + return job, nil +} + +// AutoEnqueue is the metadata-refresh fallback hook: when the library opted +// in and the providers left translatable fields without a localization for +// language, it queues a non-force item job covering the children. Cheap on +// repeat refreshes — a missing-field count guards the enqueue, and the run +// loop re-checks per field, so fully translated items never reach the model. +// Errors are logged, never returned: a refresh must not fail on this. +func (s *Service) AutoEnqueue(ctx context.Context, itemContentID, language string) { + if !s.cfg.Ready() { + return + } + target, err := subtitles.NormalizeLanguageCode(language) + if err != nil { + return + } + missing, err := s.content.CountMissingFields(ctx, itemContentID, target) + if err != nil { + s.logger.Warn("metadata translation: missing-field count failed", + "content_id", itemContentID, "language", target, "error", err) + return + } + if missing == 0 { + return + } + if _, err := s.Enqueue(ctx, JobRequest{ + TargetKind: TargetItem, + ContentID: itemContentID, + TargetLanguage: target, + IncludeChildren: true, + }); err != nil { + s.logger.Warn("metadata translation: auto-enqueue failed", + "content_id", itemContentID, "language", target, "error", err) + } +} + +// OnViewMode exposes the viewer-triggered translation mode for status probes. +func (s *Service) OnViewMode() string { return s.cfg.OnViewMode() } + +// RequestOnView is the viewer-triggered enqueue path (detail-page button or +// auto-on-view). On top of the normal pipeline it suppresses retries for a +// cooldown window after a failed job for the same target+language: ordinary +// page views must never hammer a broken endpoint. Returns the resulting job +// (which may be the in-flight or recently failed one). +func (s *Service) RequestOnView(ctx context.Context, contentID, targetLanguage string, requestedBy *int) (*Job, error) { + if s.cfg.OnViewMode() == "off" { + return nil, ErrNotConfigured + } + target, err := subtitles.NormalizeLanguageCode(targetLanguage) + if err != nil { + return nil, fmt.Errorf("%w: invalid target language %q", ErrInvalidRequest, targetLanguage) + } + + jobs, err := s.repo.ListJobsByContent(ctx, contentID) + if err != nil { + return nil, err + } + for _, job := range jobs { + if job.TargetLanguage != target { + continue + } + if job.Status == jobrunner.StatusFailed && time.Since(job.UpdatedAt) < onViewFailureCooldown { + cooled := job + return &cooled, nil + } + break // most recent job for this language decides; older history is irrelevant + } + + return s.Enqueue(ctx, JobRequest{ + TargetKind: TargetItem, + ContentID: contentID, + TargetLanguage: target, + IncludeChildren: true, + RequestedBy: requestedBy, + }) +} + +// GetJob returns a job by ID. +func (s *Service) GetJob(ctx context.Context, id int64) (*Job, error) { + job, err := s.repo.GetJob(ctx, id) + if err != nil { + return nil, err + } + if job == nil { + return nil, ErrJobNotFound + } + return job, nil +} + +// ListJobs returns recent jobs for a content ID. +func (s *Service) ListJobs(ctx context.Context, contentID string) ([]Job, error) { + return s.repo.ListJobsByContent(ctx, contentID) +} + +// Cancel requests cancellation of a job. +func (s *Service) Cancel(ctx context.Context, id int64) error { + job, err := s.repo.GetJob(ctx, id) + if err != nil { + return err + } + if job == nil { + return ErrJobNotFound + } + if s.runner.Cancel(id) { + return nil + } + if !job.Status.Terminal() { + return s.repo.FailJob(ctx, id, jobrunner.StatusCancelled, "cancelled") + } + return nil +} + +func (s *Service) dispatch(job Job) { + s.runner.Dispatch(job.ID, func(ctx context.Context) { + s.run(ctx, &job) + }, func(ctx context.Context) { + _ = s.repo.FailJob(ctx, job.ID, jobrunner.StatusCancelled, "cancelled before start") + }) +} + +// field is one translatable unit of a job. +type field struct { + kind TargetKind + contentID string + isTagline bool + text string +} + +func (f field) segmentID() string { + if f.kind == TargetItem && f.isTagline { + return "item:tagline:" + f.contentID + } + return string(f.kind) + ":overview:" + f.contentID +} + +func (s *Service) run(ctx context.Context, job *Job) { + if err := s.repo.UpdateProgress(ctx, job.ID, jobrunner.StatusRunning, 0, "Loading content", 0, 0); err != nil { + s.logger.Warn("failed to mark metadata translation job running", "job", job.ID, "error", err) + } + + fields, meta, err := s.collectFields(ctx, job) + if err != nil { + s.finishWithError(ctx, job, err) + return + } + if len(fields) == 0 { + if err := s.repo.CompleteJob(context.WithoutCancel(ctx), job.ID, "Nothing to translate", 0, 0); err != nil { + s.logger.Warn("failed to complete metadata translation job", "job", job.ID, "error", err) + } + return + } + + total := len(fields) + if err := s.repo.UpdateProgress(ctx, job.ID, jobrunner.StatusRunning, 0.05, "Translating", 0, total); err != nil { + s.logger.Warn("failed to update metadata translation job", "job", job.ID, "error", err) + } + + segments := make([]aitranslate.Segment, total) + byID := make(map[string]field, total) + for i, f := range fields { + segments[i] = aitranslate.Segment{ID: f.segmentID(), Text: f.text} + byID[f.segmentID()] = f + } + + srcName := aitranslate.LanguageDisplayName(meta.srcLanguage) + tgtName := aitranslate.LanguageDisplayName(job.TargetLanguage) + + // Persist per batch so a cancellation keeps completed fields. A persist + // failure cancels the remaining batches via persistCtx instead of burning + // model calls whose output cannot be stored. + persistCtx, cancelPersist := context.WithCancel(ctx) + defer cancelPersist() + var persistMu sync.Mutex + var persistErr error + done := 0 + + _, translateErr := aitranslate.Translate(persistCtx, s.chat, aitranslate.Request{ + Segments: segments, + SystemPrompt: systemPrompt(srcName, tgtName, meta.title, meta.year), + TargetName: tgtName, + EntryNoun: "descriptions", + BatchSize: metadataBatchSize, + ContextNeighbors: 0, + }, func(batch []aitranslate.Segment, batchDone, batchTotal int) { + // Finished translations should not be thrown away by a late cancel. + storeCtx := context.WithoutCancel(ctx) + for _, seg := range batch { + f, ok := byID[seg.ID] + if !ok { + continue + } + if err := s.persistField(storeCtx, job, f, seg.Text); err != nil { + persistMu.Lock() + if persistErr == nil { + persistErr = err + } + persistMu.Unlock() + cancelPersist() + return + } + done++ + } + progress := 0.05 + 0.9*float64(batchDone)/float64(batchTotal) + _ = s.repo.UpdateProgress(storeCtx, job.ID, jobrunner.StatusRunning, progress, "Translating", done, total) + }) + + persistMu.Lock() + failure := persistErr + persistMu.Unlock() + if failure == nil && translateErr != nil { + failure = translateErr + } + if failure != nil { + s.finishWithError(ctx, job, failure) + return + } + + if err := s.repo.CompleteJob(context.WithoutCancel(ctx), job.ID, "", done, total); err != nil { + s.logger.Warn("failed to complete metadata translation job", "job", job.ID, "error", err) + } +} + +func (s *Service) persistField(ctx context.Context, job *Job, f field, text string) error { + text = strings.TrimSpace(text) + if text == "" { + return fmt.Errorf("model returned an empty translation for %s", f.segmentID()) + } + switch f.kind { + case TargetItem: + if f.isTagline { + return s.locs.UpsertItemAI(ctx, f.contentID, job.TargetLanguage, nil, &text, job.Force) + } + return s.locs.UpsertItemAI(ctx, f.contentID, job.TargetLanguage, &text, nil, job.Force) + case TargetSeason: + return s.locs.UpsertSeasonAI(ctx, f.contentID, job.TargetLanguage, text, job.Force) + case TargetEpisode: + return s.locs.UpsertEpisodeAI(ctx, f.contentID, job.TargetLanguage, text, job.Force) + default: + return fmt.Errorf("unknown field kind %q", f.kind) + } +} + +// jobMeta carries the prompt-grounding context for a job. +type jobMeta struct { + title string + year int + srcLanguage string +} + +// collectFields expands the job target into translatable fields, dropping any +// field whose localized value is already filled (unless force, which still +// respects manual provenance). +func (s *Service) collectFields(ctx context.Context, job *Job) ([]field, jobMeta, error) { + var meta jobMeta + + switch job.TargetKind { + case TargetItem: + item, err := s.content.ItemText(ctx, job.ContentID) + if err != nil { + return nil, meta, fmt.Errorf("load item: %w", err) + } + if item == nil { + return nil, meta, fmt.Errorf("%w: item not found", ErrInvalidRequest) + } + meta = jobMeta{title: item.Title, year: item.Year, srcLanguage: item.DefaultLanguage} + job.SourceLanguage = item.DefaultLanguage + + var fields []field + itemFields, err := s.itemFields(ctx, job, item) + if err != nil { + return nil, meta, err + } + fields = append(fields, itemFields...) + + if job.IncludeChildren { + childFields, err := s.childFields(ctx, job, job.ContentID) + if err != nil { + return nil, meta, err + } + fields = append(fields, childFields...) + } + return fields, meta, nil + + case TargetSeason: + season, seriesID, err := s.content.SeasonByID(ctx, job.ContentID) + if err != nil { + return nil, meta, fmt.Errorf("load season: %w", err) + } + if season == nil { + return nil, meta, fmt.Errorf("%w: season not found", ErrInvalidRequest) + } + meta, err = s.parentMeta(ctx, job, seriesID) + if err != nil { + return nil, meta, err + } + fields, err := s.filterSeasons(ctx, job, []ChildText{*season}) + return fields, meta, err + + case TargetEpisode: + episode, seriesID, err := s.content.EpisodeByID(ctx, job.ContentID) + if err != nil { + return nil, meta, fmt.Errorf("load episode: %w", err) + } + if episode == nil { + return nil, meta, fmt.Errorf("%w: episode not found", ErrInvalidRequest) + } + meta, err = s.parentMeta(ctx, job, seriesID) + if err != nil { + return nil, meta, err + } + fields, err := s.filterEpisodes(ctx, job, []ChildText{*episode}) + return fields, meta, err + + default: + return nil, meta, fmt.Errorf("%w: unsupported target kind %q", ErrInvalidRequest, job.TargetKind) + } +} + +func (s *Service) parentMeta(ctx context.Context, job *Job, seriesID string) (jobMeta, error) { + var meta jobMeta + item, err := s.content.ItemText(ctx, seriesID) + if err != nil { + return meta, fmt.Errorf("load parent series: %w", err) + } + if item != nil { + meta = jobMeta{title: item.Title, year: item.Year, srcLanguage: item.DefaultLanguage} + job.SourceLanguage = item.DefaultLanguage + } + return meta, nil +} + +// translatableField reports whether a field with base text and an existing +// localized value/source should be translated. Empty base text never +// translates; filled localizations are skipped unless force; manual values +// are skipped even with force (the SQL layer guards them too — skipping here +// saves the model call). +func translatableField(baseText, locValue, locSource string, force bool) bool { + if strings.TrimSpace(baseText) == "" { + return false + } + if locSource == "manual" { + return false + } + if locValue != "" && !force { + return false + } + return true +} + +func (s *Service) itemFields(ctx context.Context, job *Job, item *ItemText) ([]field, error) { + loc, err := s.locs.ItemLocalization(ctx, item.ContentID, job.TargetLanguage) + if err != nil { + return nil, fmt.Errorf("load item localization: %w", err) + } + var locOverview, locOverviewSrc, locTagline, locTaglineSrc string + if loc != nil { + locOverview, locOverviewSrc = loc.Overview, loc.OverviewSource + locTagline, locTaglineSrc = loc.Tagline, loc.TaglineSource + } + + var fields []field + if translatableField(item.Overview, locOverview, locOverviewSrc, job.Force) { + fields = append(fields, field{kind: TargetItem, contentID: item.ContentID, text: item.Overview}) + } + if translatableField(item.Tagline, locTagline, locTaglineSrc, job.Force) { + fields = append(fields, field{kind: TargetItem, contentID: item.ContentID, isTagline: true, text: item.Tagline}) + } + return fields, nil +} + +func (s *Service) childFields(ctx context.Context, job *Job, seriesID string) ([]field, error) { + seasons, err := s.content.SeasonTexts(ctx, seriesID) + if err != nil { + return nil, fmt.Errorf("load seasons: %w", err) + } + fields, err := s.filterSeasons(ctx, job, seasons) + if err != nil { + return nil, err + } + + episodes, err := s.content.EpisodeTexts(ctx, seriesID) + if err != nil { + return nil, fmt.Errorf("load episodes: %w", err) + } + episodeFields, err := s.filterEpisodes(ctx, job, episodes) + if err != nil { + return nil, err + } + return append(fields, episodeFields...), nil +} + +func (s *Service) filterSeasons(ctx context.Context, job *Job, seasons []ChildText) ([]field, error) { + ids := make([]string, 0, len(seasons)) + for _, season := range seasons { + ids = append(ids, season.ContentID) + } + locs, err := s.locs.SeasonLocalizations(ctx, ids, job.TargetLanguage) + if err != nil { + return nil, fmt.Errorf("load season localizations: %w", err) + } + var fields []field + for _, season := range seasons { + var locOverview, locSource string + if loc := locs[season.ContentID]; loc != nil { + locOverview, locSource = loc.Overview, loc.OverviewSource + } + if translatableField(season.Overview, locOverview, locSource, job.Force) { + fields = append(fields, field{kind: TargetSeason, contentID: season.ContentID, text: season.Overview}) + } + } + return fields, nil +} + +func (s *Service) filterEpisodes(ctx context.Context, job *Job, episodes []ChildText) ([]field, error) { + ids := make([]string, 0, len(episodes)) + for _, episode := range episodes { + ids = append(ids, episode.ContentID) + } + locs, err := s.locs.EpisodeLocalizations(ctx, ids, job.TargetLanguage) + if err != nil { + return nil, fmt.Errorf("load episode localizations: %w", err) + } + var fields []field + for _, episode := range episodes { + var locOverview, locSource string + if loc := locs[episode.ContentID]; loc != nil { + locOverview, locSource = loc.Overview, loc.OverviewSource + } + if translatableField(episode.Overview, locOverview, locSource, job.Force) { + fields = append(fields, field{kind: TargetEpisode, contentID: episode.ContentID, text: episode.Overview}) + } + } + return fields, nil +} + +func (s *Service) finishWithError(ctx context.Context, job *Job, err error) { + status := jobrunner.StatusFailed + msg := llm.Truncate(err.Error(), 500) + if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + status = jobrunner.StatusCancelled + msg = "cancelled" + } + if dbErr := s.repo.FailJob(context.WithoutCancel(ctx), job.ID, status, msg); dbErr != nil { + s.logger.Warn("failed to record metadata translation job failure", "job", job.ID, "error", dbErr) + } + if status == jobrunner.StatusFailed { + s.logger.Warn("metadata translation job failed", "job", job.ID, "content_id", job.ContentID, "error", err) + } +} diff --git a/internal/metadata/translation/service_test.go b/internal/metadata/translation/service_test.go new file mode 100644 index 00000000..7d34c74c --- /dev/null +++ b/internal/metadata/translation/service_test.go @@ -0,0 +1,544 @@ +package translation + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "sync" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" + "github.com/Silo-Server/silo-server/internal/models" +) + +// --- fakes --- + +type fakeRepo struct { + mu sync.Mutex + nextID int64 + jobs map[int64]*Job + completed chan int64 +} + +func newFakeRepo() *fakeRepo { + return &fakeRepo{jobs: map[int64]*Job{}, completed: make(chan int64, 16)} +} + +func (r *fakeRepo) InsertJob(_ context.Context, job *Job) error { + r.mu.Lock() + defer r.mu.Unlock() + r.nextID++ + job.ID = r.nextID + cp := *job + r.jobs[job.ID] = &cp + return nil +} + +func (r *fakeRepo) GetJob(_ context.Context, id int64) (*Job, error) { + r.mu.Lock() + defer r.mu.Unlock() + if j, ok := r.jobs[id]; ok { + cp := *j + return &cp, nil + } + return nil, nil +} + +func (r *fakeRepo) GetActiveJobByIdempotencyKey(_ context.Context, key string) (*Job, error) { + r.mu.Lock() + defer r.mu.Unlock() + for _, j := range r.jobs { + if j.IdempotencyKey == key && !j.Status.Terminal() { + cp := *j + return &cp, nil + } + } + return nil, nil +} + +func (r *fakeRepo) ListJobsByContent(_ context.Context, contentID string) ([]Job, error) { + r.mu.Lock() + defer r.mu.Unlock() + var out []Job + // Newest first, matching the SQL implementation's ORDER BY created_at DESC. + for id := r.nextID; id >= 1; id-- { + if j, ok := r.jobs[id]; ok && j.ContentID == contentID { + out = append(out, *j) + } + } + return out, nil +} + +func (r *fakeRepo) UpdateProgress(_ context.Context, id int64, status JobStatus, progress float64, message string, done, total int) error { + r.mu.Lock() + defer r.mu.Unlock() + if j, ok := r.jobs[id]; ok && !j.Status.Terminal() { + j.Status, j.Progress, j.ProgressMessage, j.FieldsDone, j.FieldsTotal = status, progress, message, done, total + } + return nil +} + +func (r *fakeRepo) CompleteJob(_ context.Context, id int64, message string, done, total int) error { + r.mu.Lock() + defer r.mu.Unlock() + if j, ok := r.jobs[id]; ok && !j.Status.Terminal() { + j.Status, j.Progress, j.ProgressMessage, j.FieldsDone, j.FieldsTotal = jobrunner.StatusCompleted, 1, message, done, total + } + select { + case r.completed <- id: + default: + } + return nil +} + +func (r *fakeRepo) FailJob(_ context.Context, id int64, status JobStatus, message string) error { + r.mu.Lock() + defer r.mu.Unlock() + if j, ok := r.jobs[id]; ok && !j.Status.Terminal() { + j.Status, j.ErrorMessage = status, message + } + select { + case r.completed <- id: + default: + } + return nil +} + +func (r *fakeRepo) Heartbeat(context.Context, int64) error { return nil } +func (r *fakeRepo) ResetStaleJobs(context.Context, time.Time, string) (int64, error) { + return 0, nil +} + +func (r *fakeRepo) job(id int64) Job { + r.mu.Lock() + defer r.mu.Unlock() + return *r.jobs[id] +} + +type fakeContent struct { + item *ItemText + seasons []ChildText + episodes []ChildText + missing int +} + +func (c *fakeContent) ItemText(context.Context, string) (*ItemText, error) { return c.item, nil } +func (c *fakeContent) SeasonTexts(context.Context, string) ([]ChildText, error) { + return c.seasons, nil +} +func (c *fakeContent) EpisodeTexts(context.Context, string) ([]ChildText, error) { + return c.episodes, nil +} +func (c *fakeContent) SeasonByID(context.Context, string) (*ChildText, string, error) { + if len(c.seasons) == 0 { + return nil, "", nil + } + return &c.seasons[0], c.item.ContentID, nil +} +func (c *fakeContent) EpisodeByID(context.Context, string) (*ChildText, string, error) { + if len(c.episodes) == 0 { + return nil, "", nil + } + return &c.episodes[0], c.item.ContentID, nil +} +func (c *fakeContent) CountMissingFields(context.Context, string, string) (int, error) { + return c.missing, nil +} + +type aiWrite struct { + kind TargetKind + contentID string + overview string + tagline string + force bool +} + +type fakeLocs struct { + mu sync.Mutex + itemLoc *models.MediaItemLocalization + seasonLocs map[string]*models.SeasonLocalization + episodeLocs map[string]*models.EpisodeLocalization + writes []aiWrite + failOnUpsert bool +} + +func (l *fakeLocs) ItemLocalization(context.Context, string, string) (*models.MediaItemLocalization, error) { + return l.itemLoc, nil +} + +func (l *fakeLocs) SeasonLocalizations(context.Context, []string, string) (map[string]*models.SeasonLocalization, error) { + if l.seasonLocs == nil { + return map[string]*models.SeasonLocalization{}, nil + } + return l.seasonLocs, nil +} + +func (l *fakeLocs) EpisodeLocalizations(context.Context, []string, string) (map[string]*models.EpisodeLocalization, error) { + if l.episodeLocs == nil { + return map[string]*models.EpisodeLocalization{}, nil + } + return l.episodeLocs, nil +} + +func (l *fakeLocs) UpsertItemAI(_ context.Context, contentID, _ string, overview, tagline *string, force bool) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.failOnUpsert { + return fmt.Errorf("boom") + } + w := aiWrite{kind: TargetItem, contentID: contentID, force: force} + if overview != nil { + w.overview = *overview + } + if tagline != nil { + w.tagline = *tagline + } + l.writes = append(l.writes, w) + return nil +} + +func (l *fakeLocs) UpsertSeasonAI(_ context.Context, contentID, _ string, overview string, force bool) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.failOnUpsert { + return fmt.Errorf("boom") + } + l.writes = append(l.writes, aiWrite{kind: TargetSeason, contentID: contentID, overview: overview, force: force}) + return nil +} + +func (l *fakeLocs) UpsertEpisodeAI(_ context.Context, contentID, _ string, overview string, force bool) error { + l.mu.Lock() + defer l.mu.Unlock() + if l.failOnUpsert { + return fmt.Errorf("boom") + } + l.writes = append(l.writes, aiWrite{kind: TargetEpisode, contentID: contentID, overview: overview, force: force}) + return nil +} + +func (l *fakeLocs) allWrites() []aiWrite { + l.mu.Lock() + defer l.mu.Unlock() + return append([]aiWrite(nil), l.writes...) +} + +// upperChat "translates" by upper-casing each indexed value; counts calls. +type upperChat struct { + mu sync.Mutex + calls int +} + +func (c *upperChat) fn(_ context.Context, _ string, user string) (string, error) { + c.mu.Lock() + c.calls++ + c.mu.Unlock() + start := strings.IndexByte(user, '{') + var m map[string]string + if err := json.Unmarshal([]byte(user[start:]), &m); err != nil { + return "", err + } + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = strings.ToUpper(v) + } + b, _ := json.Marshal(out) + return string(b), nil +} + +func (c *upperChat) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.calls +} + +// --- helpers --- + +func testService(t *testing.T, repo *fakeRepo, content *fakeContent, locs *fakeLocs, chat *upperChat) *Service { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + cfg := Config{Enabled: true, Configured: true, ChatModel: "test-model", OnView: "auto"} + return NewService(ctx, cfg, repo, content, locs, chat.fn, jobrunner.NewSemaphore(1), nil) +} + +func waitDone(t *testing.T, repo *fakeRepo) int64 { + t.Helper() + select { + case id := <-repo.completed: + return id + case <-time.After(5 * time.Second): + t.Fatal("job never reached a terminal state") + return 0 + } +} + +func seriesContent() *fakeContent { + return &fakeContent{ + item: &ItemText{ + ContentID: "series1", Type: "series", Title: "Test Show", Year: 2020, + Overview: "A show.", Tagline: "Watch it.", DefaultLanguage: "en", + }, + seasons: []ChildText{ + {ContentID: "sea1", SeasonNumber: 1, Overview: "Season one."}, + {ContentID: "sea2", SeasonNumber: 2, Overview: ""}, // no base text + }, + episodes: []ChildText{ + {ContentID: "ep1", SeasonNumber: 1, EpisodeNumber: 1, Overview: "Pilot episode."}, + {ContentID: "ep2", SeasonNumber: 1, EpisodeNumber: 2, Overview: "Second episode."}, + }, + } +} + +// --- tests --- + +func TestSeriesJobExpandsChildrenAndPersists(t *testing.T) { + repo, content, locs, chat := newFakeRepo(), seriesContent(), &fakeLocs{}, &upperChat{} + svc := testService(t, repo, content, locs, chat) + + job, err := svc.Enqueue(context.Background(), JobRequest{ + ContentID: "series1", TargetLanguage: "fr", IncludeChildren: true, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + waitDone(t, repo) + + final := repo.job(job.ID) + if final.Status != jobrunner.StatusCompleted { + t.Fatalf("status = %s (%s)", final.Status, final.ErrorMessage) + } + // item overview + tagline + 1 season with text + 2 episodes = 5 fields. + if final.FieldsTotal != 5 || final.FieldsDone != 5 { + t.Errorf("fields = %d/%d, want 5/5", final.FieldsDone, final.FieldsTotal) + } + writes := locs.allWrites() + if len(writes) != 5 { + t.Fatalf("writes = %d, want 5: %+v", len(writes), writes) + } + if writes[0].kind != TargetItem || writes[0].overview != "A SHOW." { + t.Errorf("first write = %+v", writes[0]) + } + var sawSeason, sawTagline bool + for _, w := range writes { + if w.kind == TargetSeason && w.contentID == "sea1" && w.overview == "SEASON ONE." { + sawSeason = true + } + if w.kind == TargetItem && w.tagline == "WATCH IT." { + sawTagline = true + } + if w.contentID == "sea2" { + t.Error("season without base text was translated") + } + } + if !sawSeason || !sawTagline { + t.Errorf("missing expected writes: %+v", writes) + } +} + +func TestSkipIfFilledShortCircuitsWithoutModelCalls(t *testing.T) { + repo, content, chat := newFakeRepo(), seriesContent(), &upperChat{} + locs := &fakeLocs{ + itemLoc: &models.MediaItemLocalization{ + Overview: "Déjà traduit.", OverviewSource: "provider", + Tagline: "Déjà.", TaglineSource: "ai", + }, + seasonLocs: map[string]*models.SeasonLocalization{ + "sea1": {Overview: "Saison.", OverviewSource: "ai"}, + }, + episodeLocs: map[string]*models.EpisodeLocalization{ + "ep1": {Overview: "Pilote.", OverviewSource: "provider"}, + "ep2": {Overview: "Deux.", OverviewSource: "manual"}, + }, + } + svc := testService(t, repo, content, locs, chat) + + job, err := svc.Enqueue(context.Background(), JobRequest{ + ContentID: "series1", TargetLanguage: "fr", IncludeChildren: true, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + waitDone(t, repo) + + final := repo.job(job.ID) + if final.Status != jobrunner.StatusCompleted || final.FieldsTotal != 0 { + t.Fatalf("job = %+v, want completed with 0 fields", final) + } + if chat.callCount() != 0 { + t.Errorf("model called %d times for a fully translated item", chat.callCount()) + } +} + +func TestForceRetranslatesProviderAndAIButNeverManual(t *testing.T) { + repo, content, chat := newFakeRepo(), seriesContent(), &upperChat{} + locs := &fakeLocs{ + itemLoc: &models.MediaItemLocalization{ + Overview: "Vieux.", OverviewSource: "provider", + Tagline: "Manuel.", TaglineSource: "manual", + }, + episodeLocs: map[string]*models.EpisodeLocalization{ + "ep1": {Overview: "IA.", OverviewSource: "ai"}, + "ep2": {Overview: "Manuel.", OverviewSource: "manual"}, + }, + } + svc := testService(t, repo, content, locs, chat) + + job, err := svc.Enqueue(context.Background(), JobRequest{ + ContentID: "series1", TargetLanguage: "fr", IncludeChildren: true, Force: true, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + waitDone(t, repo) + + final := repo.job(job.ID) + // item overview (provider, forced) + sea1 + ep1 (ai, forced) = 3; + // tagline and ep2 are manual and stay untouched even with force. + if final.FieldsTotal != 3 { + t.Fatalf("fields_total = %d, want 3", final.FieldsTotal) + } + for _, w := range locs.allWrites() { + if !w.force { + t.Errorf("write without force flag: %+v", w) + } + if w.tagline != "" || w.contentID == "ep2" { + t.Errorf("manual field was translated: %+v", w) + } + } +} + +func TestPersistFailureFailsJob(t *testing.T) { + repo, content, chat := newFakeRepo(), seriesContent(), &upperChat{} + locs := &fakeLocs{failOnUpsert: true} + svc := testService(t, repo, content, locs, chat) + + job, err := svc.Enqueue(context.Background(), JobRequest{ + ContentID: "series1", TargetLanguage: "fr", IncludeChildren: true, + }) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + waitDone(t, repo) + + final := repo.job(job.ID) + if final.Status != jobrunner.StatusFailed || !strings.Contains(final.ErrorMessage, "boom") { + t.Fatalf("job = %+v, want failed with persist error", final) + } +} + +func TestEnqueueDeduplicatesActiveJobs(t *testing.T) { + repo, content, locs, chat := newFakeRepo(), seriesContent(), &fakeLocs{}, &upperChat{} + // Block the runner so the first job stays active during the second enqueue. + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + sem := jobrunner.NewSemaphore(1) + sem <- struct{}{} // occupy the only slot + cfg := Config{Enabled: true, Configured: true, ChatModel: "test-model"} + svc := NewService(ctx, cfg, repo, content, locs, chat.fn, sem, nil) + + first, err := svc.Enqueue(context.Background(), JobRequest{ContentID: "series1", TargetLanguage: "fr"}) + if err != nil { + t.Fatalf("Enqueue: %v", err) + } + second, err := svc.Enqueue(context.Background(), JobRequest{ContentID: "series1", TargetLanguage: "fr"}) + if err != nil { + t.Fatalf("Enqueue dup: %v", err) + } + if second.ID != first.ID { + t.Errorf("duplicate enqueue created job %d, want %d", second.ID, first.ID) + } + <-sem // release so cleanup can proceed +} + +func TestAutoEnqueueSkipsWhenNothingMissing(t *testing.T) { + repo, content, locs, chat := newFakeRepo(), seriesContent(), &fakeLocs{}, &upperChat{} + content.missing = 0 + svc := testService(t, repo, content, locs, chat) + + svc.AutoEnqueue(context.Background(), "series1", "fr") + if len(repo.jobs) != 0 { + t.Errorf("AutoEnqueue created a job with nothing missing") + } + + content.missing = 3 + svc.AutoEnqueue(context.Background(), "series1", "fr") + if len(repo.jobs) != 1 { + t.Errorf("AutoEnqueue did not create a job with fields missing") + } + waitDone(t, repo) +} + +func TestEnqueueValidatesInput(t *testing.T) { + repo, content, locs, chat := newFakeRepo(), seriesContent(), &fakeLocs{}, &upperChat{} + svc := testService(t, repo, content, locs, chat) + + if _, err := svc.Enqueue(context.Background(), JobRequest{ContentID: "x", TargetLanguage: "definitely-not-a-language"}); err == nil { + t.Error("invalid language accepted") + } + if _, err := svc.Enqueue(context.Background(), JobRequest{TargetLanguage: "fr"}); err == nil { + t.Error("missing content id accepted") + } + if _, err := svc.Enqueue(context.Background(), JobRequest{ContentID: "x", TargetKind: "bogus", TargetLanguage: "fr"}); err == nil { + t.Error("bogus target kind accepted") + } + + disabled := NewService(context.Background(), Config{}, repo, content, locs, chat.fn, nil, nil) + if _, err := disabled.Enqueue(context.Background(), JobRequest{ContentID: "x", TargetLanguage: "fr"}); err != ErrNotConfigured { + t.Errorf("disabled service err = %v, want ErrNotConfigured", err) + } +} + +func TestRequestOnViewCooldownAndGating(t *testing.T) { + repo, content, locs, chat := newFakeRepo(), seriesContent(), &fakeLocs{}, &upperChat{} + svc := testService(t, repo, content, locs, chat) + + // A recently failed job for the same target+language suppresses re-enqueue. + failed := &Job{ + ContentID: "series1", TargetKind: TargetItem, TargetLanguage: "fr", + Status: jobrunner.StatusFailed, UpdatedAt: time.Now().Add(-time.Minute), + } + if err := repo.InsertJob(context.Background(), failed); err != nil { + t.Fatalf("seed failed job: %v", err) + } + repo.mu.Lock() + repo.jobs[failed.ID].Status = jobrunner.StatusFailed + repo.jobs[failed.ID].UpdatedAt = time.Now().Add(-time.Minute) + repo.mu.Unlock() + + job, err := svc.RequestOnView(context.Background(), "series1", "fr", nil) + if err != nil { + t.Fatalf("RequestOnView: %v", err) + } + if job.ID != failed.ID || job.Status != jobrunner.StatusFailed { + t.Fatalf("cooldown not applied: got job %d status %s", job.ID, job.Status) + } + + // Once the failure has aged past the cooldown, a new job is enqueued. + repo.mu.Lock() + repo.jobs[failed.ID].UpdatedAt = time.Now().Add(-2 * onViewFailureCooldown) + repo.mu.Unlock() + job, err = svc.RequestOnView(context.Background(), "series1", "fr", nil) + if err != nil { + t.Fatalf("RequestOnView after cooldown: %v", err) + } + if job.ID == failed.ID { + t.Fatal("stale failed job returned after cooldown expired") + } + waitDone(t, repo) + + // A different language is unaffected by the failure. + job2, err := svc.RequestOnView(context.Background(), "series1", "de", nil) + if err != nil || job2.ID == failed.ID { + t.Fatalf("other-language request blocked: job=%v err=%v", job2, err) + } + waitDone(t, repo) + + // OnView off (zero-value config) refuses viewer requests outright. + off := NewService(context.Background(), Config{Enabled: true, Configured: true, ChatModel: "m"}, repo, content, locs, chat.fn, nil, nil) + if _, err := off.RequestOnView(context.Background(), "series1", "fr", nil); err != ErrNotConfigured { + t.Errorf("on_view=off err = %v, want ErrNotConfigured", err) + } +} diff --git a/internal/metadata/translation/store.go b/internal/metadata/translation/store.go new file mode 100644 index 00000000..e72cf7cc --- /dev/null +++ b/internal/metadata/translation/store.go @@ -0,0 +1,51 @@ +package translation + +import ( + "context" + + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/models" +) + +// LocalizationStore reads existing localizations (for skip-if-filled) and +// writes AI translations with provenance enforced by the underlying repos. +type LocalizationStore interface { + ItemLocalization(ctx context.Context, contentID, language string) (*models.MediaItemLocalization, error) + SeasonLocalizations(ctx context.Context, ids []string, language string) (map[string]*models.SeasonLocalization, error) + EpisodeLocalizations(ctx context.Context, ids []string, language string) (map[string]*models.EpisodeLocalization, error) + UpsertItemAI(ctx context.Context, contentID, language string, overview, tagline *string, force bool) error + UpsertSeasonAI(ctx context.Context, contentID, language, overview string, force bool) error + UpsertEpisodeAI(ctx context.Context, contentID, language, overview string, force bool) error +} + +// CatalogLocalizationStore adapts the catalog localization repositories to +// LocalizationStore. +type CatalogLocalizationStore struct { + Items *catalog.MediaItemLocalizationRepository + Seasons *catalog.SeasonLocalizationRepository + Episodes *catalog.EpisodeLocalizationRepository +} + +func (s *CatalogLocalizationStore) ItemLocalization(ctx context.Context, contentID, language string) (*models.MediaItemLocalization, error) { + return s.Items.Get(ctx, contentID, language) +} + +func (s *CatalogLocalizationStore) SeasonLocalizations(ctx context.Context, ids []string, language string) (map[string]*models.SeasonLocalization, error) { + return s.Seasons.GetBySeasonIDs(ctx, ids, language) +} + +func (s *CatalogLocalizationStore) EpisodeLocalizations(ctx context.Context, ids []string, language string) (map[string]*models.EpisodeLocalization, error) { + return s.Episodes.GetByEpisodeIDs(ctx, ids, language) +} + +func (s *CatalogLocalizationStore) UpsertItemAI(ctx context.Context, contentID, language string, overview, tagline *string, force bool) error { + return s.Items.UpsertAITranslation(ctx, contentID, language, overview, tagline, force) +} + +func (s *CatalogLocalizationStore) UpsertSeasonAI(ctx context.Context, contentID, language, overview string, force bool) error { + return s.Seasons.UpsertAIOverview(ctx, contentID, language, overview, force) +} + +func (s *CatalogLocalizationStore) UpsertEpisodeAI(ctx context.Context, contentID, language, overview string, force bool) error { + return s.Episodes.UpsertAIOverview(ctx, contentID, language, overview, force) +} diff --git a/internal/models/media.go b/internal/models/media.go index 8320f856..e1712446 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -13,6 +13,7 @@ type MediaFolder struct { Name string Enabled bool MetadataLanguage string // ISO 639-1 code (e.g. "en", "ja") + AutoTranslateMetadata bool // AI-translate descriptions when providers lack this language ChapterThumbnailsEnabled bool IntroDetectionEnabled bool PosterPath string // S3 key for library poster image @@ -421,6 +422,16 @@ type EpisodeLibrary struct { FirstSeenAt time.Time } +// Localization field provenance values. Precedence when writing: +// manual beats provider beats ai — a provider refresh may overwrite an AI +// translation but never a manual edit, and AI never overwrites either +// (except provider, when the admin explicitly forces a re-translation). +const ( + LocalizationSourceProvider = "provider" + LocalizationSourceAI = "ai" + LocalizationSourceManual = "manual" +) + type MediaItemLocalization struct { ContentID string Language string @@ -433,6 +444,8 @@ type MediaItemLocalization struct { BackdropPath string BackdropThumbhash string LogoPath string + OverviewSource string // provider | ai | manual + TaglineSource string // provider | ai | manual CreatedAt time.Time UpdatedAt time.Time } @@ -444,6 +457,7 @@ type SeasonLocalization struct { Overview string PosterPath string PosterThumbhash string + OverviewSource string // provider | ai | manual CreatedAt time.Time UpdatedAt time.Time } @@ -453,6 +467,7 @@ type EpisodeLocalization struct { Language string Title string Overview string + OverviewSource string // provider | ai | manual CreatedAt time.Time UpdatedAt time.Time } diff --git a/internal/playback/audio_extract.go b/internal/playback/audio_extract.go new file mode 100644 index 00000000..b157f067 --- /dev/null +++ b/internal/playback/audio_extract.go @@ -0,0 +1,180 @@ +package playback + +import ( + "bytes" + "context" + "encoding/csv" + "encoding/json" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// AudioChunk is one extracted piece of an audio track. Start is the chunk's +// exact start within the extracted audio (seconds) as reported by ffmpeg's +// segment muxer — the muxer cuts at packet boundaries near, not exactly at, +// the requested length, so assuming index*chunkSeconds would accumulate +// subtitle timing error. +type AudioChunk struct { + Path string + Start float64 +} + +// ExtractAudioChunks extracts one audio track from a media file into +// fixed-length 16 kHz mono WAV chunks under dir — the input format Whisper +// endpoints want, sized to stay under typical upload limits (a 10-minute +// chunk is ~19 MB). One ffmpeg pass segments the whole track; the returned +// chunks are in chronological order with exact start offsets. The caller owns +// dir and its cleanup. +func ExtractAudioChunks(ctx context.Context, filePath string, audioTrackIndex int, dir, ffmpegPath string, chunkSeconds int) ([]AudioChunk, error) { + if chunkSeconds <= 0 { + chunkSeconds = 600 + } + if audioTrackIndex < 0 { + audioTrackIndex = 0 + } + + listPath := filepath.Join(dir, "segments.csv") + args := []string{ + "-i", filePath, + "-vn", "-sn", "-dn", + "-map", fmt.Sprintf("0:a:%d", audioTrackIndex), + "-ac", "1", + "-ar", "16000", + "-c:a", "pcm_s16le", + "-f", "segment", + "-segment_time", strconv.Itoa(chunkSeconds), + "-segment_list", listPath, + "-segment_list_type", "csv", + "-y", filepath.Join(dir, "chunk%05d.wav"), + } + + ffmpeg := "ffmpeg" + if ffmpegPath != "" { + ffmpeg = ffmpegPath + } + cmd := exec.CommandContext(ctx, ffmpeg, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("ffmpeg audio extraction failed: %w (stderr: %s)", + err, truncateStderr(stderr.String())) + } + + starts := parseSegmentList(listPath) + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read audio chunk dir: %w", err) + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasPrefix(e.Name(), "chunk") && strings.HasSuffix(e.Name(), ".wav") { + names = append(names, e.Name()) + } + } + sort.Strings(names) + if len(names) == 0 { + return nil, fmt.Errorf("ffmpeg produced no audio chunks for track %d", audioTrackIndex) + } + + chunks := make([]AudioChunk, len(names)) + for i, name := range names { + start, ok := starts[name] + if !ok { + // Segment list missing/unparsable: fall back to the nominal grid. + start = float64(i * chunkSeconds) + } + chunks[i] = AudioChunk{Path: filepath.Join(dir, name), Start: start} + } + return chunks, nil +} + +// parseSegmentList reads ffmpeg's CSV segment list (filename,start,end per +// line) into a filename → start map. Best effort: a missing or malformed list +// yields an empty map and callers fall back to nominal chunk starts. +func parseSegmentList(listPath string) map[string]float64 { + starts := map[string]float64{} + f, err := os.Open(listPath) + if err != nil { + return starts + } + defer f.Close() + rows, err := csv.NewReader(f).ReadAll() + if err != nil { + return starts + } + for _, row := range rows { + if len(row) < 2 { + continue + } + start, err := strconv.ParseFloat(strings.TrimSpace(row[1]), 64) + if err != nil || start < 0 { + continue + } + starts[filepath.Base(strings.TrimSpace(row[0]))] = start + } + return starts +} + +// maxPlausibleAudioDelay caps the probed audio/container start delta; beyond +// this it is far more likely a probing artifact than a real stream delay. +const maxPlausibleAudioDelay = 30.0 + +// ProbeAudioStartOffset returns the audio stream's start time relative to the +// container start (seconds). Whisper timestamps are relative to the first +// audio sample, while the playback timeline starts at the container start — +// in containers with delayed audio (TS remuxes especially) the difference is +// a constant subtitle sync error unless corrected. Best effort: any probe +// failure returns 0. +func ProbeAudioStartOffset(ctx context.Context, filePath string, audioTrackIndex int, ffmpegPath string) float64 { + if audioTrackIndex < 0 { + audioTrackIndex = 0 + } + ffprobe := "ffprobe" + if ffmpegPath != "" { + // ffprobe ships next to ffmpeg in every distribution Silo supports. + ffprobe = filepath.Join(filepath.Dir(ffmpegPath), "ffprobe") + } + + cmd := exec.CommandContext(ctx, ffprobe, + "-v", "error", + "-print_format", "json", + "-show_entries", "format=start_time", + "-select_streams", fmt.Sprintf("a:%d", audioTrackIndex), + "-show_entries", "stream=start_time", + filePath, + ) + out, err := cmd.Output() + if err != nil { + return 0 + } + + var probe struct { + Format struct { + StartTime string `json:"start_time"` + } `json:"format"` + Streams []struct { + StartTime string `json:"start_time"` + } `json:"streams"` + } + if err := json.Unmarshal(out, &probe); err != nil || len(probe.Streams) == 0 { + return 0 + } + formatStart, err1 := strconv.ParseFloat(probe.Format.StartTime, 64) + streamStart, err2 := strconv.ParseFloat(probe.Streams[0].StartTime, 64) + if err1 != nil || err2 != nil { + return 0 + } + delta := streamStart - formatStart + if math.IsNaN(delta) || math.Abs(delta) > maxPlausibleAudioDelay { + return 0 + } + return delta +} diff --git a/internal/subtitles/ai/client.go b/internal/subtitles/ai/client.go deleted file mode 100644 index 5fa2bd24..00000000 --- a/internal/subtitles/ai/client.go +++ /dev/null @@ -1,190 +0,0 @@ -package ai - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log/slog" - "net/http" - "strconv" - "strings" - "time" -) - -// Client is a minimal OpenAI-compatible chat-completions client. It follows the -// same retry/backoff conventions as the recommendations embedding client so the -// two behave consistently against OpenAI, Groq, Ollama, llama.cpp servers, etc. -type Client struct { - cfg Config - httpClient *http.Client -} - -// NewClient builds a client from the engine config. -func NewClient(cfg Config) *Client { - return &Client{ - cfg: cfg, - httpClient: &http.Client{Timeout: 10 * time.Minute}, - } -} - -type chatMessage struct { - Role string `json:"role"` - Content string `json:"content"` -} - -type chatResponseFormat struct { - Type string `json:"type"` -} - -type chatCompletionRequest struct { - Model string `json:"model"` - Messages []chatMessage `json:"messages"` - Temperature float32 `json:"temperature"` - ResponseFormat *chatResponseFormat `json:"response_format,omitempty"` -} - -type chatCompletionResponse struct { - Choices []struct { - Message chatMessage `json:"message"` - } `json:"choices"` - // Some OpenAI-compatible gateways (e.g. OpenRouter) return a 200 with an - // error object instead of an HTTP error status when an upstream provider - // fails. We surface and retry on it. - Error *struct { - Message string `json:"message"` - } `json:"error"` -} - -// chat performs one chat completion and returns the first choice's content. -// When jsonObject is true it requests response_format=json_object; providers -// that ignore the field still work because the prompt itself demands JSON. -func (c *Client) chat(ctx context.Context, messages []chatMessage, jsonObject bool) (string, error) { - reqBody := chatCompletionRequest{ - Model: c.cfg.ChatModel, - Messages: messages, - Temperature: 0.2, - } - if jsonObject { - reqBody.ResponseFormat = &chatResponseFormat{Type: "json_object"} - } - - body, err := json.Marshal(reqBody) - if err != nil { - return "", fmt.Errorf("marshal chat request: %w", err) - } - - url := strings.TrimRight(c.cfg.BaseURL, "/") + "/v1/chat/completions" - - const maxAttempts = 6 - var lastErr error - for attempt := 0; attempt < maxAttempts; attempt++ { - httpReq, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) - if reqErr != nil { - return "", fmt.Errorf("create request: %w", reqErr) - } - httpReq.Header.Set("Content-Type", "application/json") - if c.cfg.APIKey != "" { - httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey) - } - - resp, doErr := c.httpClient.Do(httpReq) - if doErr != nil { - lastErr = fmt.Errorf("chat request failed: %w", doErr) - if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { - return "", waitErr - } - continue - } - - respBody, readErr := io.ReadAll(resp.Body) - resp.Body.Close() - if readErr != nil { - // A truncated/failed read could otherwise be misparsed as a valid - // (empty) response; treat it as a retryable transport error. - lastErr = fmt.Errorf("read chat response: %w", readErr) - if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { - return "", waitErr - } - continue - } - - switch { - case resp.StatusCode == http.StatusTooManyRequests: - wait := rateLimitBackoff(resp, attempt) - slog.Warn("rate limited by subtitle AI chat API, waiting", "attempt", attempt+1, "wait", wait) - lastErr = fmt.Errorf("chat API returned 429: %s", truncate(string(respBody), 300)) - if waitErr := sleepCtx(ctx, wait); waitErr != nil { - return "", waitErr - } - continue - case resp.StatusCode >= 500: - lastErr = fmt.Errorf("chat API returned %d: %s", resp.StatusCode, truncate(string(respBody), 300)) - if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { - return "", waitErr - } - continue - case resp.StatusCode != http.StatusOK: - // 4xx other than 429: not retryable. - return "", fmt.Errorf("chat API returned %d: %s", resp.StatusCode, truncate(string(respBody), 300)) - } - - var parsed chatCompletionResponse - if err := json.Unmarshal(respBody, &parsed); err != nil { - lastErr = fmt.Errorf("decode chat response: %w", err) - } else if parsed.Error != nil && parsed.Error.Message != "" { - // 200 with an upstream error object — transient on gateways. - lastErr = fmt.Errorf("chat API error: %s", parsed.Error.Message) - } else if len(parsed.Choices) == 0 || parsed.Choices[0].Message.Content == "" { - lastErr = fmt.Errorf("chat API returned no choices") - slog.Warn("subtitle AI chat returned no choices, retrying", - "attempt", attempt+1, "model", c.cfg.ChatModel, "response_bytes", len(respBody)) - } else { - return parsed.Choices[0].Message.Content, nil - } - - // Empty-choices / 200-error / decode failure: retry with backoff. - if waitErr := sleepCtx(ctx, time.Duration(attempt+1)*time.Second); waitErr != nil { - return "", waitErr - } - } - - if lastErr == nil { - lastErr = fmt.Errorf("chat API: retries exhausted") - } - return "", lastErr -} - -// truncate caps a string for inclusion in an error or log line. -func truncate(s string, maxLen int) string { - if len(s) > maxLen { - return s[:maxLen] + "..." - } - return s -} - -// sleepCtx waits for d or until ctx is cancelled. -func sleepCtx(ctx context.Context, d time.Duration) error { - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(d): - return nil - } -} - -// rateLimitBackoff returns how long to wait after a 429, honoring Retry-After -// and otherwise backing off exponentially (capped at 60s). -func rateLimitBackoff(resp *http.Response, attempt int) time.Duration { - if ra := resp.Header.Get("Retry-After"); ra != "" { - if secs, err := strconv.Atoi(ra); err == nil && secs > 0 { - return time.Duration(secs) * time.Second - } - } - wait := 10 * time.Second * (1 << attempt) - if wait > 60*time.Second { - wait = 60 * time.Second - } - return wait -} diff --git a/internal/subtitles/ai/config.go b/internal/subtitles/ai/config.go index dafc15b8..fec748f8 100644 --- a/internal/subtitles/ai/config.go +++ b/internal/subtitles/ai/config.go @@ -1,20 +1,30 @@ package ai -// Config holds the runtime configuration for the AI subtitle engine. Values are -// loaded from server_settings (see internal/config) and mirror the existing -// recommendations embedding client: one OpenAI-compatible endpoint that the -// operator can point at OpenAI, Groq, a local Ollama/llama.cpp server, etc. +// Config holds the runtime configuration for the subtitle AI features. The +// shared endpoint connection (base URL, keys) lives in the llm.Client this +// package is wired with; this struct carries the feature toggles plus the +// model names used for job provenance and idempotency. type Config struct { - Enabled bool - BaseURL string // e.g. "https://api.openai.com" (no trailing /v1) - APIKey string // empty for keyless local servers + // Configured reports that the shared AI endpoint has a base URL; without + // one, no feature is available regardless of toggles. + Configured bool + // TranslateEnabled gates on-demand subtitle translation. + TranslateEnabled bool + // TranscribeEnabled gates Whisper ASR generation (transcribe / + // transcribe_translate jobs). + TranscribeEnabled bool ChatModel string // chat-completions model used for translation - MaxConcurrentJobs int // semaphore bound so jobs never starve transcodes + ASRModel string // audio-transcription model used for ASR BatchSize int // cues per translation request ContextNeighbors int // preceding source cues sent as untranslated context } -// Ready reports whether the engine is enabled and minimally configured. -func (c Config) Ready() bool { - return c.Enabled && c.BaseURL != "" && c.ChatModel != "" +// TranslateReady reports whether subtitle translation can currently run. +func (c Config) TranslateReady() bool { + return c.Configured && c.TranslateEnabled && c.ChatModel != "" +} + +// TranscribeReady reports whether ASR subtitle generation can currently run. +func (c Config) TranscribeReady() bool { + return c.Configured && c.TranscribeEnabled && c.ASRModel != "" } diff --git a/internal/subtitles/ai/job.go b/internal/subtitles/ai/job.go index 62a6ec2b..4aa1eb1b 100644 --- a/internal/subtitles/ai/job.go +++ b/internal/subtitles/ai/job.go @@ -5,38 +5,43 @@ import ( "encoding/hex" "fmt" "time" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" ) -// JobKind identifies what an AI subtitle job does. Only translation ships in -// the first iteration; transcription kinds are reserved for the Whisper ASR -// follow-up so the schema and API do not change again. +// JobKind identifies what an AI subtitle job does. type JobKind string const ( + // JobKindTranslate translates an existing text subtitle track. SourceIndex + // is the combined player subtitle index of the source track. JobKindTranslate JobKind = "translate" + // JobKindTranscribe generates a subtitle track from an audio track via + // Whisper ASR. SourceIndex is the 0-based audio track index (-1 = default + // track); TargetLanguage is optional and acts as a language hint. + JobKindTranscribe JobKind = "transcribe" + // JobKindTranscribeTranslate chains transcription with LLM translation to + // TargetLanguage, storing both the transcript and the translated track. + JobKindTranscribeTranslate JobKind = "transcribe_translate" ) -// JobStatus is the lifecycle state of a job. -type JobStatus string +// IsTranscribe reports whether the kind starts from audio. +func (k JobKind) IsTranscribe() bool { + return k == JobKindTranscribe || k == JobKindTranscribeTranslate +} + +// JobStatus is the lifecycle state of a job, shared with the other AI job +// services via jobrunner. +type JobStatus = jobrunner.Status const ( - JobStatusPending JobStatus = "pending" - JobStatusRunning JobStatus = "running" - JobStatusCompleted JobStatus = "completed" - JobStatusFailed JobStatus = "failed" - JobStatusCancelled JobStatus = "cancelled" + JobStatusPending = jobrunner.StatusPending + JobStatusRunning = jobrunner.StatusRunning + JobStatusCompleted = jobrunner.StatusCompleted + JobStatusFailed = jobrunner.StatusFailed + JobStatusCancelled = jobrunner.StatusCancelled ) -// Terminal reports whether a status is final. -func (s JobStatus) Terminal() bool { - switch s { - case JobStatusCompleted, JobStatusFailed, JobStatusCancelled: - return true - default: - return false - } -} - // Job is a persisted AI subtitle job. It is serialized to the API as-is. type Job struct { ID int64 `json:"id"` diff --git a/internal/subtitles/ai/service.go b/internal/subtitles/ai/service.go index ffbacf49..d3b0e55e 100644 --- a/internal/subtitles/ai/service.go +++ b/internal/subtitles/ai/service.go @@ -8,25 +8,19 @@ import ( "os" "sort" "strings" - "sync" "time" + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" + "github.com/Silo-Server/silo-server/internal/ai/llm" + aitranslate "github.com/Silo-Server/silo-server/internal/ai/translate" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/subtitles" ) -const providerTranslated = "translated" - const ( - // A running job refreshes its heartbeat every heartbeatInterval; one whose - // heartbeat has not advanced for staleJobThreshold is treated as orphaned by - // a crashed worker and reaped. The margin over heartbeatInterval avoids - // reaping a job that is merely mid–LLM-call. - heartbeatInterval = 30 * time.Second - staleJobThreshold = 2 * time.Minute - // How often the background reaper scans for orphaned jobs. - reaperInterval = time.Minute + providerTranslated = "translated" + providerTranscribed = "transcribed" ) var ( @@ -61,129 +55,116 @@ type SubtitleLister interface { ListDownloadedSubtitles(ctx context.Context, mediaFileID int) ([]subtitles.DownloadedSubtitle, error) } -// Service owns the AI subtitle job lifecycle: enqueue, bounded concurrent -// execution, progress/heartbeat, cancellation, and restart recovery. +// Service owns the AI subtitle job semantics: enqueue validation, source +// resolution, translation, and result storage. Lifecycle mechanics (bounded +// dispatch, heartbeat, stale-job reaping, cancellation) are delegated to the +// shared jobrunner so they stay identical across Silo's AI job services. type Service struct { - // baseCtx is the application context; dispatched jobs and the reaper derive - // from it so they stop when the server shuts down. - baseCtx context.Context - cfg Config - repo JobRepository - translator Translator - store SubtitleStore - lister SubtitleLister - files MediaFileResolver - notifier Notifier // optional - ffmpegPath string - logger *slog.Logger - - sem chan struct{} - mu sync.Mutex - cancels map[int64]context.CancelFunc - wg sync.WaitGroup + cfg Config + repo JobRepository + translator Translator + transcriber Transcriber // optional; nil disables ASR kinds + store SubtitleStore + lister SubtitleLister + files MediaFileResolver + notifier Notifier // optional + ffmpegPath string + logger *slog.Logger + runner *jobrunner.Runner } // NewService wires a translation service. notifier may be nil. appCtx is the // application lifecycle context; jobs and the reaper derive from it so they stop -// on shutdown. A nil appCtx falls back to context.Background(). +// on shutdown. A nil appCtx falls back to context.Background(). sem is the +// dispatch semaphore, normally shared with the other AI job services so the +// configured endpoint sees one global concurrency bound; nil gets a private +// default-size semaphore. func NewService( appCtx context.Context, cfg Config, repo JobRepository, translator Translator, + transcriber Transcriber, store SubtitleStore, lister SubtitleLister, files MediaFileResolver, notifier Notifier, ffmpegPath string, logger *slog.Logger, + sem chan struct{}, ) *Service { - maxConcurrent := cfg.MaxConcurrentJobs - if maxConcurrent <= 0 { - maxConcurrent = 2 - } if logger == nil { logger = slog.Default() } - if appCtx == nil { - appCtx = context.Background() - } return &Service{ - baseCtx: appCtx, - cfg: cfg, - repo: repo, - translator: translator, - store: store, - lister: lister, - files: files, - notifier: notifier, - ffmpegPath: ffmpegPath, - logger: logger, - sem: make(chan struct{}, maxConcurrent), - cancels: make(map[int64]context.CancelFunc), + cfg: cfg, + repo: repo, + translator: translator, + transcriber: transcriber, + store: store, + lister: lister, + files: files, + notifier: notifier, + ffmpegPath: ffmpegPath, + logger: logger, + runner: jobrunner.New(appCtx, sem, repo, "subtitle ai", logger), } } // Enabled reports whether translation can currently run. -func (s *Service) Enabled() bool { return s.cfg.Ready() } +func (s *Service) Enabled() bool { return s.cfg.TranslateReady() } + +// TranscribeEnabled reports whether ASR subtitle generation can currently run. +func (s *Service) TranscribeEnabled() bool { + return s.cfg.TranscribeReady() && s.transcriber != nil +} // Recover clears jobs orphaned by a crashed worker and starts a background -// reaper that keeps doing so. Reaping is heartbeat-based (not "every active -// job"), so it is safe when multiple instances share one database: a job still -// being heartbeat-updated by a live worker is never reset. Call once at startup; -// jobs and the reaper derive from the application context passed to NewService, -// so they stop on shutdown. +// reaper that keeps doing so. Call once at startup. func (s *Service) Recover() { - s.reapStaleJobs() - go s.reaperLoop() -} - -// reaperLoop periodically reaps orphaned jobs until the application context is -// cancelled (server shutdown). -func (s *Service) reaperLoop() { - ticker := time.NewTicker(reaperInterval) - defer ticker.Stop() - for { - select { - case <-s.baseCtx.Done(): - return - case <-ticker.C: - s.reapStaleJobs() - } - } -} - -// reapStaleJobs fails any pending/running job whose heartbeat has not advanced -// within staleJobThreshold. -func (s *Service) reapStaleJobs() { - before := time.Now().Add(-staleJobThreshold) - n, err := s.repo.ResetStaleJobs(context.WithoutCancel(s.baseCtx), before, "interrupted by server restart") - if err != nil { - s.logger.Warn("failed to reset stale subtitle ai jobs", "error", err) - return - } - if n > 0 { - s.logger.Info("reset stale subtitle ai jobs", "count", n) - } + s.runner.Recover() } // Enqueue validates and queues a job, returning immediately. If an identical // job is already pending or running, that job is returned instead of a new one. func (s *Service) Enqueue(ctx context.Context, req JobRequest) (*Job, error) { - if !s.cfg.Ready() { - return nil, ErrEngineNotConfigured - } if req.Kind == "" { req.Kind = JobKindTranslate } - - target, err := subtitles.NormalizeLanguageCode(req.TargetLanguage) - if err != nil { - return nil, fmt.Errorf("%w: invalid target language %q", ErrInvalidRequest, req.TargetLanguage) + jobModel := s.cfg.ChatModel + switch req.Kind { + case JobKindTranslate: + if !s.cfg.TranslateReady() { + return nil, ErrEngineNotConfigured + } + case JobKindTranscribe: + if !s.TranscribeEnabled() { + return nil, ErrEngineNotConfigured + } + jobModel = s.cfg.ASRModel + case JobKindTranscribeTranslate: + if !s.TranscribeEnabled() || !s.cfg.TranslateReady() { + return nil, ErrEngineNotConfigured + } + jobModel = s.cfg.ASRModel + "+" + s.cfg.ChatModel + default: + return nil, fmt.Errorf("%w: unsupported job kind %q", ErrInvalidRequest, req.Kind) } - req.TargetLanguage = target - key := idempotencyKey(req.MediaFileID, req.Kind, req.SourceIndex, req.TargetLanguage, s.cfg.ChatModel) + // A plain transcribe has no target language (the track comes out in the + // spoken language); when provided it acts as a language hint. Every other + // kind requires a valid target. + if req.Kind == JobKindTranscribe && strings.TrimSpace(req.TargetLanguage) == "" { + req.TargetLanguage = "" + } else { + target, err := subtitles.NormalizeLanguageCode(req.TargetLanguage) + if err != nil { + return nil, fmt.Errorf("%w: invalid target language %q", ErrInvalidRequest, req.TargetLanguage) + } + req.TargetLanguage = target + } + + key := idempotencyKey(req.MediaFileID, req.Kind, req.SourceIndex, req.TargetLanguage, jobModel) if existing, err := s.repo.GetActiveJobByIdempotencyKey(ctx, key); err != nil { return nil, err } else if existing != nil { @@ -197,7 +178,7 @@ func (s *Service) Enqueue(ctx context.Context, req JobRequest) (*Job, error) { SourceLanguage: req.SourceLanguage, TargetLanguage: req.TargetLanguage, Engine: "openai", - Model: s.cfg.ChatModel, + Model: jobModel, Status: JobStatusPending, ProgressMessage: "Queued", IdempotencyKey: key, @@ -244,12 +225,7 @@ func (s *Service) Cancel(ctx context.Context, id int64) error { return ErrJobNotFound } - s.mu.Lock() - cancel := s.cancels[id] - s.mu.Unlock() - - if cancel != nil { - cancel() + if s.runner.Cancel(id) { return nil } // No in-flight goroutine (e.g. another node, or never started): best-effort @@ -262,63 +238,18 @@ func (s *Service) Cancel(ctx context.Context, id int64) error { // dispatch launches a bounded background goroutine to run the job. func (s *Service) dispatch(job Job) { - // Derive from the application context so a server shutdown cancels in-flight - // translations (the per-job cancel still allows user-initiated cancellation). - runCtx, cancel := context.WithCancel(s.baseCtx) - s.mu.Lock() - s.cancels[job.ID] = cancel - s.mu.Unlock() - - s.wg.Add(1) - go func() { - defer s.wg.Done() - defer func() { - s.mu.Lock() - delete(s.cancels, job.ID) - s.mu.Unlock() - cancel() - }() - - // Heartbeat for the whole lifetime — crucially including while queued - // behind the semaphore — so the stale-job reaper never reaps a job that is - // alive but merely waiting for a slot (which would otherwise mark it failed - // and let it resurrect itself on acquire, or admit a duplicate). - stopHeartbeat := make(chan struct{}) - defer close(stopHeartbeat) - go s.heartbeatLoop(runCtx, job.ID, stopHeartbeat) - - // Bound concurrency so translation never starves transcodes. - select { - case s.sem <- struct{}{}: - case <-runCtx.Done(): - _ = s.repo.FailJob(context.Background(), job.ID, JobStatusCancelled, "cancelled before start") - return - } - defer func() { <-s.sem }() - - s.run(runCtx, &job) - }() -} - -// heartbeatLoop keeps a job's heartbeat_at fresh until the job ends or the -// context is cancelled, so the stale-job reaper only ever reaps jobs orphaned by -// a crashed worker. -func (s *Service) heartbeatLoop(ctx context.Context, jobID int64, stop <-chan struct{}) { - ticker := time.NewTicker(heartbeatInterval) - defer ticker.Stop() - for { - select { - case <-stop: - return - case <-ctx.Done(): - return - case <-ticker.C: - _ = s.repo.Heartbeat(context.WithoutCancel(ctx), jobID) - } - } + s.runner.Dispatch(job.ID, func(ctx context.Context) { + s.run(ctx, &job) + }, func(ctx context.Context) { + _ = s.repo.FailJob(ctx, job.ID, JobStatusCancelled, "cancelled before start") + }) } func (s *Service) run(ctx context.Context, job *Job) { + if job.Kind.IsTranscribe() { + s.runTranscribe(ctx, job) + return + } if err := s.repo.UpdateProgress(ctx, job.ID, JobStatusRunning, 0, "Loading subtitle"); err != nil { s.logger.Warn("failed to mark subtitle ai job running", "job", job.ID, "error", err) } @@ -397,9 +328,199 @@ func (s *Service) run(ctx context.Context, job *Job) { } } +// runTranscribe executes transcribe / transcribe_translate jobs: extract the +// audio track, transcribe it chunk by chunk (playhead-first), store the +// transcript as an ordinary downloaded subtitle, and for the chained kind run +// the regular translator over the transcript and store the translated track +// too. The transcript doubles as a cache: a later translation to another +// language can use it as a plain text source without re-running ASR. +func (s *Service) runTranscribe(ctx context.Context, job *Job) { + if err := s.repo.UpdateProgress(ctx, job.ID, JobStatusRunning, 0, "Preparing audio"); err != nil { + s.logger.Warn("failed to mark subtitle ai job running", "job", job.ID, "error", err) + } + + file, err := s.files.GetByID(ctx, job.MediaFileID) + if err != nil { + s.finishWithError(ctx, job, fmt.Errorf("load media file: %w", err)) + return + } + if file == nil { + s.finishWithError(ctx, job, fmt.Errorf("media file not found")) + return + } + if len(file.AudioTracks) == 0 { + s.finishWithError(ctx, job, fmt.Errorf("%w: file has no audio tracks", ErrSourceUnsupported)) + return + } + + audioIdx := job.SourceIndex + if audioIdx < 0 { + audioIdx = defaultAudioTrackIndex(file.AudioTracks) + } + if audioIdx >= len(file.AudioTracks) { + s.finishWithError(ctx, job, fmt.Errorf("%w: audio track index out of range", ErrInvalidRequest)) + return + } + + // Language hint: an explicit source language wins; otherwise the track's + // tagged language, when it normalizes to an ISO code. + hint := job.SourceLanguage + if hint == "" { + if code, err := subtitles.NormalizeLanguageCode(file.AudioTracks[audioIdx].Language); err == nil { + hint = code + } + } + if job.Kind == JobKindTranscribe && hint == "" && job.TargetLanguage != "" { + hint = job.TargetLanguage + } + + streaming := job.SessionID != "" && s.notifier != nil + trackKey := liveTrackKey(job.ID) + if streaming { + liveLang, liveLabel := job.TargetLanguage, transcribedReleaseName(hint) + if job.Kind == JobKindTranscribeTranslate { + liveLabel = translatedReleaseName(hint, job.TargetLanguage) + } else { + liveLang = hint + } + // Cue total is unknown before transcription; 0 means indeterminate. + s.notifier.TranslationStarted(ctx, job.SessionID, job.MediaFileID, job.ID, trackKey, liveLang, liveLabel, 0) + } + + // Only a plain transcribe streams transcript cues — the chained kind + // streams the translated cues from its translation stage instead. + streamTranscript := streaming && job.Kind == JobKindTranscribe + cues, detected, err := s.transcriber.Transcribe(ctx, TranscribeJobRequest{ + FilePath: file.FilePath, + AudioTrackIndex: audioIdx, + LanguageHint: hint, + StartPosition: job.StartPosition, + }, func(chunk []SubtitleCue, done, total int) { + // Transcription occupies the 5%..70% progress band (chunk granularity). + _ = s.repo.UpdateProgress(ctx, job.ID, JobStatusRunning, 0.05+0.65*float64(done)/float64(total), "Transcribing") + if streamTranscript { + s.notifier.TranslationCues(ctx, job.SessionID, job.MediaFileID, job.ID, trackKey, + toStreamCues(chunk), done, total) + } + }) + if err != nil { + s.finishWithError(ctx, job, err) + return + } + + language := hint + if language == "" { + language = detected + } + if job.SourceLanguage == "" { + job.SourceLanguage = language + } + + // A finished transcription should not be thrown away by a last-moment cancel. + storeCtx := context.WithoutCancel(ctx) + + transcriptCues := make([]SubtitleCue, len(cues)) + copy(transcriptCues, cues) + sortCuesByStart(transcriptCues) + transcriptLabel := transcribedReleaseName(language) + transcript, err := s.store.StoreSubtitle(storeCtx, subtitles.StoreSubtitleRequest{ + MediaFileID: job.MediaFileID, + UserID: job.RequestedBy, + Provider: providerTranscribed, + Language: language, + Format: subtitles.FormatSRT, + ReleaseName: transcriptLabel, + Data: SerializeSRT(transcriptCues), + }) + if err != nil { + s.finishWithError(ctx, job, fmt.Errorf("store transcribed subtitle: %w", err)) + return + } + if s.notifier != nil { + s.notifier.SubtitleReady(storeCtx, job.MediaFileID, transcript.ID, language, transcriptLabel) + } + + if job.Kind == JobKindTranscribe { + if err := s.repo.CompleteJob(storeCtx, job.ID, transcript.ID); err != nil { + s.logger.Warn("failed to mark subtitle ai job complete", "job", job.ID, "error", err) + } + if streaming { + s.notifier.TranslationCompleted(storeCtx, job.SessionID, job.MediaFileID, job.ID, trackKey, + transcript.ID, language, transcriptLabel) + } + return + } + + // transcribe_translate: run the transcript through the regular translator. + // Cues already arrive playhead-first from chunk ordering, so the viewer's + // region translates (and streams) first here too. + translated, err := s.translator.Translate(ctx, TranslateRequest{ + Cues: cues, + SourceLanguage: language, + TargetLanguage: job.TargetLanguage, + }, func(batch []SubtitleCue, done, total int) { + // Translation occupies the 70%..95% band. + _ = s.repo.UpdateProgress(ctx, job.ID, JobStatusRunning, 0.7+0.25*float64(done)/float64(total), "Translating") + if streaming { + s.notifier.TranslationCues(ctx, job.SessionID, job.MediaFileID, job.ID, trackKey, + toStreamCues(batch), done, total) + } + }) + if err != nil { + s.finishWithError(ctx, job, err) + return + } + + storeCtx = context.WithoutCancel(ctx) + sortCuesByStart(translated) + translatedLabel := translatedReleaseName(language, job.TargetLanguage) + sub, err := s.store.StoreSubtitle(storeCtx, subtitles.StoreSubtitleRequest{ + MediaFileID: job.MediaFileID, + UserID: job.RequestedBy, + Provider: providerTranslated, + Language: job.TargetLanguage, + Format: subtitles.FormatSRT, + ReleaseName: translatedLabel, + Data: SerializeSRT(translated), + }) + if err != nil { + s.finishWithError(ctx, job, fmt.Errorf("store translated subtitle: %w", err)) + return + } + if err := s.repo.CompleteJob(storeCtx, job.ID, sub.ID); err != nil { + s.logger.Warn("failed to mark subtitle ai job complete", "job", job.ID, "error", err) + } + if s.notifier != nil { + if streaming { + s.notifier.TranslationCompleted(storeCtx, job.SessionID, job.MediaFileID, job.ID, trackKey, + sub.ID, job.TargetLanguage, translatedLabel) + } + s.notifier.SubtitleReady(storeCtx, job.MediaFileID, sub.ID, job.TargetLanguage, translatedLabel) + } +} + +// defaultAudioTrackIndex returns the index of the default-flagged audio +// track, falling back to the first track. +func defaultAudioTrackIndex(tracks []models.AudioTrack) int { + for i, t := range tracks { + if t.Default { + return i + } + } + return 0 +} + +func transcribedReleaseName(lang string) string { + name := aitranslate.LanguageDisplayName(lang) + if name == "" { + name = "Audio" + } + return fmt.Sprintf("%s (AI transcribed)", name) +} + func (s *Service) finishWithError(ctx context.Context, job *Job, err error) { status := JobStatusFailed - msg := truncate(err.Error(), 500) + msg := llm.Truncate(err.Error(), 500) // Only a genuine cancellation (user cancel, or shutdown via cancel) becomes // "cancelled". A deadline/timeout (context.DeadlineExceeded) stays "failed". if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { @@ -505,11 +626,11 @@ func isParsableTextFormat(format string) bool { } func translatedReleaseName(sourceLang, targetLang string) string { - src := languageDisplayName(sourceLang) + src := aitranslate.LanguageDisplayName(sourceLang) if src == "" { src = "Original" } - tgt := languageDisplayName(targetLang) + tgt := aitranslate.LanguageDisplayName(targetLang) if tgt == "" { tgt = targetLang } diff --git a/internal/subtitles/ai/service_test.go b/internal/subtitles/ai/service_test.go index 2fceefb0..127ef9e1 100644 --- a/internal/subtitles/ai/service_test.go +++ b/internal/subtitles/ai/service_test.go @@ -5,6 +5,8 @@ import ( "sync" "testing" "time" + + "github.com/Silo-Server/silo-server/internal/ai/jobrunner" ) // recordingRepo is a JobRepository that records ResetStaleJobs calls and no-ops @@ -49,7 +51,7 @@ func TestRecoverReapsStaleJobsImmediately(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // stops the reaper goroutine started by Recover - svc := NewService(ctx, Config{}, repo, nil, nil, nil, nil, nil, "", nil) + svc := NewService(ctx, Config{}, repo, nil, nil, nil, nil, nil, nil, "", nil, nil) approxNow := time.Now() svc.Recover() @@ -58,7 +60,7 @@ func TestRecoverReapsStaleJobsImmediately(t *testing.T) { if resets < 1 { t.Fatalf("Recover did not reap immediately: resets=%d", resets) } - want := approxNow.Add(-staleJobThreshold) + want := approxNow.Add(-jobrunner.StaleJobThreshold) if diff := before.Sub(want); diff > 2*time.Second || diff < -2*time.Second { t.Errorf("stale cutoff = %v, want ~%v (now-staleJobThreshold)", before, want) } diff --git a/internal/subtitles/ai/transcriber.go b/internal/subtitles/ai/transcriber.go new file mode 100644 index 00000000..07762779 --- /dev/null +++ b/internal/subtitles/ai/transcriber.go @@ -0,0 +1,327 @@ +package ai + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + "unicode/utf8" + + "github.com/Silo-Server/silo-server/internal/ai/llm" + aitranslate "github.com/Silo-Server/silo-server/internal/ai/translate" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/subtitles" +) + +const ( + // Default/maximum audio chunk length sent per transcription request. + // 10 minutes of 16 kHz mono WAV is ~19 MB — under typical 25 MB API + // limits. Shorter chunks bound Whisper's within-chunk timestamp drift at + // the cost of more requests and more boundary word-clips; the operator + // tunes this via subtitle_ai.asr_chunk_seconds. + defaultASRChunkSeconds = 600 + minASRChunkSeconds = 60 + // Per-chunk request timeout: 3× realtime accommodates local Whisper + // servers on modest hardware. + asrChunkTimeoutFactor = 3 + // Cue text wrapping: standard subtitle conventions. + cueMaxLineLength = 42 + cueMaxLines = 2 + // Cue timing conventions. When word timings are available a cue closes + // at a speech pause, at text capacity, at the maximum duration, or after + // a sentence ends; without them a segment becomes one duration-capped + // cue. Sub-second cues are stretched to the minimum so short + // interjections ("What?") stay readable. + maxCueSeconds = 7.0 + minCueSeconds = 1.0 + cueSplitPauseSeconds = 1.0 +) + +// TranscribeJobRequest is the input to a Transcriber. +type TranscribeJobRequest struct { + FilePath string + AudioTrackIndex int // resolved 0-based audio stream index + LanguageHint string // ISO 639-1; "" lets the model detect + StartPosition float64 // seconds; chunks are processed playhead-first +} + +// Transcriber converts an audio track into subtitle cues. The built-in +// implementation is WhisperTranscriber; the interface is the seam for tests +// and future engines. onChunk, when non-nil, receives each chunk's cues as +// they land (chronological within a chunk, playhead-first across chunks) so +// callers can report progress and stream cues live. +type Transcriber interface { + Transcribe(ctx context.Context, req TranscribeJobRequest, + onChunk func(cues []SubtitleCue, done, total int)) ([]SubtitleCue, string, error) +} + +// audioTranscriber is the slice of llm.Client the transcriber needs. +type audioTranscriber interface { + Transcribe(ctx context.Context, req llm.TranscribeRequest) (*llm.Transcription, error) +} + +// WhisperTranscriber generates subtitles from audio via an OpenAI-compatible +// transcription endpoint: extract the track to 16 kHz mono WAV chunks, send +// each chunk for verbose_json transcription, offset the segment timestamps by +// the chunk start, and build wrapped cues. Chunk boundaries are fixed-length; +// a word straddling a boundary can be clipped — accepted v1 limitation, noted +// for a silence-aligned follow-up. +type WhisperTranscriber struct { + client audioTranscriber + ffmpegPath string + chunkSeconds int + // extract and probeOffset are playback helpers, injectable for tests. + extract func(ctx context.Context, filePath string, audioTrackIndex int, dir, ffmpegPath string, chunkSeconds int) ([]playback.AudioChunk, error) + probeOffset func(ctx context.Context, filePath string, audioTrackIndex int, ffmpegPath string) float64 +} + +// NewWhisperTranscriber builds a transcriber backed by the shared AI client. +// chunkSeconds outside [minASRChunkSeconds, defaultASRChunkSeconds] falls +// back to the default (longer chunks would exceed upload limits). +func NewWhisperTranscriber(client *llm.Client, ffmpegPath string, chunkSeconds int) *WhisperTranscriber { + if chunkSeconds < minASRChunkSeconds || chunkSeconds > defaultASRChunkSeconds { + chunkSeconds = defaultASRChunkSeconds + } + return &WhisperTranscriber{ + client: client, + ffmpegPath: ffmpegPath, + chunkSeconds: chunkSeconds, + extract: playback.ExtractAudioChunks, + probeOffset: playback.ProbeAudioStartOffset, + } +} + +// Transcribe implements Transcriber. The returned cues are NOT sorted (they +// arrive playhead-first across chunks); the detected language is the +// endpoint's report for the first processed chunk, normalized to an ISO code +// where possible. +func (t *WhisperTranscriber) Transcribe(ctx context.Context, req TranscribeJobRequest, + onChunk func(cues []SubtitleCue, done, total int)) ([]SubtitleCue, string, error) { + dir, err := os.MkdirTemp("", "silo-asr-*") + if err != nil { + return nil, "", fmt.Errorf("create ASR temp dir: %w", err) + } + defer os.RemoveAll(dir) + + chunks, err := t.extract(ctx, req.FilePath, req.AudioTrackIndex, dir, t.ffmpegPath, t.chunkSeconds) + if err != nil { + return nil, "", err + } + + // Audio streams can start after the container's timeline origin (TS + // remuxes especially); Whisper times are relative to the first audio + // sample, so the delta is a constant sync error unless added back. + startOffset := t.probeOffset(ctx, req.FilePath, req.AudioTrackIndex, t.ffmpegPath) + + order := chunkOrderForPosition(chunks, req.StartPosition) + timeout := time.Duration(t.chunkSeconds*asrChunkTimeoutFactor) * time.Second + + var all []SubtitleCue + detected := "" + for done, idx := range order { + if err := ctx.Err(); err != nil { + return nil, "", err + } + data, err := os.ReadFile(chunks[idx].Path) + if err != nil { + return nil, "", fmt.Errorf("read audio chunk: %w", err) + } + tr, err := t.client.Transcribe(ctx, llm.TranscribeRequest{ + Filename: filepath.Base(chunks[idx].Path), + Audio: data, + Language: req.LanguageHint, + Timeout: timeout, + }) + if err != nil { + return nil, "", fmt.Errorf("transcribe chunk %d/%d: %w", idx+1, len(chunks), err) + } + // Each chunk is read exactly once; deleting it as we go caps disk usage + // at one extraction rather than extraction + retranscription leftovers. + _ = os.Remove(chunks[idx].Path) + + if detected == "" { + detected = normalizeDetectedLanguage(tr.Language) + } + cues := cuesFromSegments(tr.Segments, chunks[idx].Start+startOffset) + all = append(all, cues...) + if onChunk != nil { + onChunk(cues, done+1, len(order)) + } + } + + if len(all) == 0 { + return nil, detected, fmt.Errorf("no speech recognized in the audio track") + } + return all, detected, nil +} + +// chunkOrderForPosition orders chunk indexes so the chunk containing +// startSeconds is processed first, then forward, then wrapping to the start — +// the viewer's current region fills first, mirroring translation's +// playhead-first cue order. +func chunkOrderForPosition(chunks []playback.AudioChunk, startSeconds float64) []int { + n := len(chunks) + pivot := 0 + if startSeconds > 0 { + for i := n - 1; i >= 0; i-- { + if chunks[i].Start <= startSeconds { + pivot = i + break + } + } + } + order := make([]int, 0, n) + for i := pivot; i < n; i++ { + order = append(order, i) + } + for i := 0; i < pivot; i++ { + order = append(order, i) + } + return order +} + +// cuesFromSegments converts transcription segments (timestamps relative to +// their chunk) to absolute-time cues, dropping speech-free segments and +// wrapping text to subtitle conventions. Segments with word timings are +// re-split into readable cues that end when speech stops; segments without +// them become single cues capped at the maximum duration, because Whisper +// segment end times otherwise stretch across silence to the next segment. +func cuesFromSegments(segments []llm.TranscriptionSegment, offsetSeconds float64) []SubtitleCue { + var out []SubtitleCue + for _, seg := range segments { + text := strings.TrimSpace(seg.Text) + if text == "" { + continue + } + if cues := cuesFromWords(seg.Words, offsetSeconds); len(cues) > 0 { + out = append(out, cues...) + continue + } + start := offsetSeconds + seg.Start + end := offsetSeconds + seg.End + if maxEnd := start + maxCueSeconds; end > maxEnd { + end = maxEnd + } + out = append(out, newCue(start, end, text)) + } + enforceMinCueDurations(out) + return out +} + +// cuesFromWords groups word timings into cues: a cue closes at a speech +// pause, when the text would exceed the wrap capacity, when it would exceed +// the maximum duration, or after a sentence ends once a full line is +// accumulated. Cue end times come from the last word, so cues disappear when +// speech stops instead of lingering through silence. +func cuesFromWords(words []llm.TranscriptionWord, offsetSeconds float64) []SubtitleCue { + maxRunes := cueMaxLineLength * cueMaxLines + var out []SubtitleCue + var texts []string + var start, end float64 + runes := 0 + flush := func() { + if len(texts) > 0 { + out = append(out, newCue(offsetSeconds+start, offsetSeconds+end, strings.Join(texts, " "))) + } + texts, runes = nil, 0 + } + for _, w := range words { + text := strings.TrimSpace(w.Text) + if text == "" { + continue + } + n := utf8.RuneCountInString(text) + if len(texts) > 0 && + (w.Start-end >= cueSplitPauseSeconds || runes+1+n > maxRunes || w.End-start > maxCueSeconds) { + flush() + } + if len(texts) == 0 { + start = w.Start + } + texts = append(texts, text) + runes += n + 1 + end = w.End + if runes >= cueMaxLineLength && endsSentence(text) { + flush() + } + } + flush() + return out +} + +// endsSentence reports whether a word closes a sentence, tolerating a +// trailing quote or bracket after the punctuation. +func endsSentence(word string) bool { + word = strings.TrimRight(word, `"')]”’`) + return strings.HasSuffix(word, ".") || strings.HasSuffix(word, "?") || + strings.HasSuffix(word, "!") || strings.HasSuffix(word, "…") +} + +// newCue builds a wrapped cue, guarding degenerate timestamps with a minimal +// visible duration. +func newCue(startSec, endSec float64, text string) SubtitleCue { + if endSec <= startSec { + endSec = startSec + 0.5 + } + return SubtitleCue{ + Start: time.Duration(startSec * float64(time.Second)), + End: time.Duration(endSec * float64(time.Second)), + Lines: wrapCueText(text, cueMaxLineLength, cueMaxLines), + } +} + +// enforceMinCueDurations stretches sub-minimum cues (word-accurate timing can +// produce a 0.2s "What?") up to the readable minimum, without overlapping the +// next cue. Cues are chronological within a chunk. +func enforceMinCueDurations(cues []SubtitleCue) { + minDur := time.Duration(minCueSeconds * float64(time.Second)) + for i := range cues { + minEnd := cues[i].Start + minDur + if i+1 < len(cues) && minEnd > cues[i+1].Start { + minEnd = cues[i+1].Start + } + if cues[i].End < minEnd { + cues[i].End = minEnd + } + } +} + +// wrapCueText greedily wraps text into at most maxLines lines of roughly +// maxLen characters (counted in runes, so multi-byte scripts like Arabic or +// Cyrillic wrap at the same visual width as Latin). The last line absorbs any +// overflow — text is never dropped. +func wrapCueText(text string, maxLen, maxLines int) []string { + words := strings.Fields(text) + if len(words) == 0 { + return nil + } + lines := []string{words[0]} + for _, w := range words[1:] { + last := len(lines) - 1 + switch { + case utf8.RuneCountInString(lines[last])+1+utf8.RuneCountInString(w) <= maxLen: + lines[last] += " " + w + case len(lines) < maxLines: + lines = append(lines, w) + default: + lines[last] += " " + w + } + } + return lines +} + +// normalizeDetectedLanguage maps a Whisper-reported language to an ISO 639-1 +// code: endpoints variously report codes ("en") or English names ("english"). +// Returns "" when it cannot be normalized. +func normalizeDetectedLanguage(reported string) string { + reported = strings.TrimSpace(reported) + if reported == "" { + return "" + } + if code, err := subtitles.NormalizeLanguageCode(reported); err == nil { + return code + } + return aitranslate.LanguageCodeFromName(reported) +} diff --git a/internal/subtitles/ai/transcriber_test.go b/internal/subtitles/ai/transcriber_test.go new file mode 100644 index 00000000..a8f2947d --- /dev/null +++ b/internal/subtitles/ai/transcriber_test.go @@ -0,0 +1,374 @@ +package ai + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/ai/llm" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// fakeASRClient transcribes by returning canned segments per chunk filename. +type fakeASRClient struct { + perChunk map[string][]llm.TranscriptionSegment + language string + requests []llm.TranscribeRequest + err error +} + +func (f *fakeASRClient) Transcribe(_ context.Context, req llm.TranscribeRequest) (*llm.Transcription, error) { + f.requests = append(f.requests, llm.TranscribeRequest{ + Filename: req.Filename, Language: req.Language, Timeout: req.Timeout, + }) + if f.err != nil { + return nil, f.err + } + return &llm.Transcription{Language: f.language, Segments: f.perChunk[req.Filename]}, nil +} + +// stubExtract writes fake chunk files into dir (one per start offset) and +// records dir for cleanup assertions. +func stubExtract(starts []float64, recordDir *string) func(context.Context, string, int, string, string, int) ([]playback.AudioChunk, error) { + return func(_ context.Context, _ string, _ int, dir, _ string, _ int) ([]playback.AudioChunk, error) { + *recordDir = dir + var chunks []playback.AudioChunk + for i, start := range starts { + p := filepath.Join(dir, fmt.Sprintf("chunk%05d.wav", i)) + if err := os.WriteFile(p, []byte("RIFF"), 0o644); err != nil { + return nil, err + } + chunks = append(chunks, playback.AudioChunk{Path: p, Start: start}) + } + return chunks, nil + } +} + +func evenStarts(n int) []float64 { + starts := make([]float64, n) + for i := range starts { + starts[i] = float64(i * 600) + } + return starts +} + +func newTestTranscriber(client *fakeASRClient, chunks int, recordDir *string) *WhisperTranscriber { + return &WhisperTranscriber{ + client: client, + chunkSeconds: 600, + extract: stubExtract(evenStarts(chunks), recordDir), + probeOffset: func(context.Context, string, int, string) float64 { return 0 }, + } +} + +func TestTranscribeOffsetsTimestampsByChunkStart(t *testing.T) { + var dir string + client := &fakeASRClient{ + language: "english", + perChunk: map[string][]llm.TranscriptionSegment{ + "chunk00000.wav": {{Start: 1, End: 3, Text: " hello"}}, + "chunk00001.wav": {{Start: 2, End: 4, Text: " world"}}, + }, + } + tr := newTestTranscriber(client, 2, &dir) + + cues, lang, err := tr.Transcribe(context.Background(), TranscribeJobRequest{FilePath: "/x.mkv"}, nil) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if lang != "en" { + t.Errorf("detected language = %q, want en (normalized from %q)", lang, client.language) + } + if len(cues) != 2 { + t.Fatalf("cues = %d, want 2", len(cues)) + } + // Chunk 1 segment offset by 600s. + if cues[1].Start != 602*time.Second || cues[1].End != 604*time.Second { + t.Errorf("offset cue = %v–%v, want 602s–604s", cues[1].Start, cues[1].End) + } + if strings.Join(cues[1].Lines, " ") != "world" { + t.Errorf("cue text = %q", cues[1].Lines) + } + if dir == "" { + t.Fatal("extract dir not recorded") + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Errorf("temp dir %s not cleaned up", dir) + } +} + +func TestTranscribeProcessesChunksPlayheadFirst(t *testing.T) { + var dir string + client := &fakeASRClient{ + language: "ja", + perChunk: map[string][]llm.TranscriptionSegment{ + "chunk00000.wav": {{Start: 0, End: 1, Text: "a"}}, + "chunk00001.wav": {{Start: 0, End: 1, Text: "b"}}, + "chunk00002.wav": {{Start: 0, End: 1, Text: "c"}}, + }, + } + tr := newTestTranscriber(client, 3, &dir) + + var chunkOrder []string + cues, _, err := tr.Transcribe(context.Background(), TranscribeJobRequest{ + FilePath: "/x.mkv", StartPosition: 1300, // inside chunk 2 + }, func(chunk []SubtitleCue, done, total int) { + if total != 3 { + t.Errorf("total = %d, want 3", total) + } + chunkOrder = append(chunkOrder, strings.Join(chunk[0].Lines, "")) + }) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if got := strings.Join(chunkOrder, ""); got != "cab" { + t.Errorf("chunk processing order = %q, want cab (playhead-first, wrapping)", got) + } + if len(cues) != 3 { + t.Errorf("cues = %d, want 3", len(cues)) + } +} + +func TestTranscribePassesHintAndChunkSizedTimeout(t *testing.T) { + var dir string + client := &fakeASRClient{ + language: "fr", + perChunk: map[string][]llm.TranscriptionSegment{ + "chunk00000.wav": {{Start: 0, End: 1, Text: "bonjour"}}, + }, + } + tr := newTestTranscriber(client, 1, &dir) + + _, _, err := tr.Transcribe(context.Background(), TranscribeJobRequest{FilePath: "/x.mkv", LanguageHint: "fr"}, nil) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + req := client.requests[0] + if req.Language != "fr" { + t.Errorf("hint = %q, want fr", req.Language) + } + if want := 1800 * time.Second; req.Timeout != want { + t.Errorf("timeout = %v, want %v (3x chunk duration)", req.Timeout, want) + } +} + +func TestTranscribeAllSilentChunksFailsClearly(t *testing.T) { + var dir string + client := &fakeASRClient{language: "en", perChunk: map[string][]llm.TranscriptionSegment{}} + tr := newTestTranscriber(client, 2, &dir) + + _, _, err := tr.Transcribe(context.Background(), TranscribeJobRequest{FilePath: "/x.mkv"}, nil) + if err == nil || !strings.Contains(err.Error(), "no speech") { + t.Fatalf("err = %v, want no-speech failure", err) + } + if _, statErr := os.Stat(dir); !os.IsNotExist(statErr) { + t.Errorf("temp dir %s not cleaned up on failure", dir) + } +} + +func TestWrapCueText(t *testing.T) { + cases := []struct { + text string + want []string + }{ + {"short line", []string{"short line"}}, + {"", nil}, + { + "this sentence is long enough that it needs to wrap onto a second line", + []string{"this sentence is long enough that it", "needs to wrap onto a second line"}, + }, + } + for _, c := range cases { + got := wrapCueText(c.text, 42, 2) + if fmt.Sprint(got) != fmt.Sprint(c.want) { + t.Errorf("wrapCueText(%q) = %q, want %q", c.text, got, c.want) + } + } + + // Overflow beyond two lines is absorbed, never dropped. + long := strings.Repeat("word ", 40) + got := wrapCueText(strings.TrimSpace(long), 42, 2) + if len(got) != 2 { + t.Fatalf("lines = %d, want 2", len(got)) + } + if joined := strings.Join(got, " "); strings.Count(joined, "word") != 40 { + t.Errorf("overflow dropped words: %d/40", strings.Count(joined, "word")) + } +} + +func TestCuesFromSegmentsCapsWordlessSegmentDuration(t *testing.T) { + // Without word timings, Whisper segment ends stretch wall-to-wall across + // silence (a 0.8s line reported as 30s); cap rather than linger. + cues := cuesFromSegments([]llm.TranscriptionSegment{ + {Start: 10, End: 40, Text: "Days like today."}, + }, 0) + if len(cues) != 1 { + t.Fatalf("cues = %d, want 1", len(cues)) + } + if want := time.Duration(maxCueSeconds * float64(time.Second)); cues[0].End-cues[0].Start != want { + t.Errorf("capped duration = %v, want %v", cues[0].End-cues[0].Start, want) + } +} + +func wordSeq(startAt, dur, gap float64, words ...string) []llm.TranscriptionWord { + out := make([]llm.TranscriptionWord, 0, len(words)) + at := startAt + for _, w := range words { + out = append(out, llm.TranscriptionWord{Start: at, End: at + dur, Text: " " + w}) + at += dur + gap + } + return out +} + +func TestCuesFromWordsEndsCueWhenSpeechStops(t *testing.T) { + // One segment whose reported end (60s) is far past the last word (12.4s): + // the cue must end at the words, not the segment. + words := wordSeq(10, 0.4, 0.1, "Are", "you", "okay?") + cues := cuesFromSegments([]llm.TranscriptionSegment{ + {Start: 10, End: 60, Text: " Are you okay?", Words: words}, + }, 0) + if len(cues) != 1 { + t.Fatalf("cues = %d, want 1", len(cues)) + } + if cues[0].Start != 10*time.Second { + t.Errorf("start = %v, want 10s", cues[0].Start) + } + // Last word ends at 11.3s; the minimum-duration stretch may pad slightly, + // but nothing close to the segment's reported 60s. + if cues[0].End > 12*time.Second { + t.Errorf("end = %v, want ~11.3s (last word), not segment end", cues[0].End) + } +} + +func TestCuesFromWordsSplitsAtPause(t *testing.T) { + words := append(wordSeq(0, 0.4, 0.1, "First", "thought"), + wordSeq(5, 0.4, 0.1, "second", "thought")...) + cues := cuesFromWords(words, 0) + if len(cues) != 2 { + t.Fatalf("cues = %d, want 2 (split at 4s pause)", len(cues)) + } + if cues[1].Start != 5*time.Second { + t.Errorf("second cue start = %v, want 5s", cues[1].Start) + } +} + +func TestCuesFromWordsSplitsParagraphAtSentencesAndCapacity(t *testing.T) { + // A paragraph-length segment (the "465-char single cue" failure) must + // split into readable cues. Sentence ends close a cue once a line's worth + // of text has accumulated; capacity closes one regardless. + var words []llm.TranscriptionWord + for i := 0; i < 6; i++ { + words = append(words, wordSeq(float64(i)*3, 0.3, 0.1, + "this", "sentence", "keeps", "going", "and", "going", "until", "it", "stops.")...) + } + cues := cuesFromWords(words, 0) + if len(cues) < 4 { + t.Fatalf("cues = %d, want the paragraph split into several", len(cues)) + } + maxRunes := cueMaxLineLength * cueMaxLines + for i, c := range cues { + text := strings.Join(c.Lines, " ") + if got := len([]rune(text)); got > maxRunes { + t.Errorf("cue %d has %d runes, over capacity %d: %q", i, got, maxRunes, text) + } + if dur := c.End - c.Start; dur > time.Duration(maxCueSeconds*float64(time.Second))+time.Second { + t.Errorf("cue %d duration %v exceeds max", i, dur) + } + } +} + +func TestEnforceMinCueDurationsClampsToNextCue(t *testing.T) { + cues := []SubtitleCue{ + {Start: 0, End: 200 * time.Millisecond, Lines: []string{"What?"}}, + {Start: 600 * time.Millisecond, End: 2 * time.Second, Lines: []string{"LSD."}}, + {Start: 3 * time.Second, End: 3200 * time.Millisecond, Lines: []string{"Oh."}}, + } + enforceMinCueDurations(cues) + if cues[0].End != 600*time.Millisecond { + t.Errorf("cue 0 end = %v, want clamped to next cue start (600ms)", cues[0].End) + } + if cues[1].End != 2*time.Second { + t.Errorf("cue 1 end = %v, want unchanged", cues[1].End) + } + if cues[2].End != 4*time.Second { + t.Errorf("cue 2 end = %v, want stretched to 1s minimum", cues[2].End) + } +} + +func TestCuesFromSegmentsGuardsDegenerateTimes(t *testing.T) { + cues := cuesFromSegments([]llm.TranscriptionSegment{ + {Start: 5, End: 5, Text: "zero duration"}, + {Start: 1, End: 2, Text: " "}, + }, 0) + if len(cues) != 1 { + t.Fatalf("cues = %d, want 1 (whitespace dropped)", len(cues)) + } + if cues[0].End <= cues[0].Start { + t.Errorf("degenerate cue not given a minimum duration: %v–%v", cues[0].Start, cues[0].End) + } +} + +func TestChunkOrderForPosition(t *testing.T) { + chunks := []playback.AudioChunk{{Start: 0}, {Start: 600.06}, {Start: 1200.13}, {Start: 1800.2}} + if got := fmt.Sprint(chunkOrderForPosition(chunks, 0)); got != "[0 1 2 3]" { + t.Errorf("no playhead: %s", got) + } + if got := fmt.Sprint(chunkOrderForPosition(chunks, 1900)); got != "[3 0 1 2]" { + t.Errorf("late playhead: %s", got) + } + if got := fmt.Sprint(chunkOrderForPosition(chunks, 99999)); got != "[3 0 1 2]" { + t.Errorf("beyond-last playhead starts at the final chunk: %s", got) + } +} + +// Cue timing must use the muxer-reported chunk start (not index*chunkSeconds) +// plus the probed audio start offset, or sync drifts on long files and +// delayed-audio containers. +func TestTranscribeUsesExactChunkStartsAndAudioOffset(t *testing.T) { + var dir string + client := &fakeASRClient{ + language: "en", + perChunk: map[string][]llm.TranscriptionSegment{ + "chunk00000.wav": {{Start: 1, End: 2, Text: "first"}}, + "chunk00001.wav": {{Start: 1, End: 2, Text: "second"}}, + }, + } + tr := &WhisperTranscriber{ + client: client, + chunkSeconds: 600, + // The second chunk really starts at 600.5s, not 600s. + extract: stubExtract([]float64{0, 600.5}, &dir), + probeOffset: func(context.Context, string, int, string) float64 { return 1.25 }, + } + + cues, _, err := tr.Transcribe(context.Background(), TranscribeJobRequest{FilePath: "/x.mkv"}, nil) + if err != nil { + t.Fatalf("Transcribe: %v", err) + } + if want := time.Duration(2.25 * float64(time.Second)); cues[0].Start != want { + t.Errorf("cue 0 start = %v, want %v (1s segment + 1.25s audio offset)", cues[0].Start, want) + } + if want := time.Duration(602.75 * float64(time.Second)); cues[1].Start != want { + t.Errorf("cue 1 start = %v, want %v (600.5s chunk + 1s segment + 1.25s offset)", cues[1].Start, want) + } +} + +func TestNormalizeDetectedLanguage(t *testing.T) { + cases := map[string]string{ + "english": "en", + "en": "en", + "eng": "en", + "JAPANESE": "ja", + "": "", + "klingon": "", + } + for in, want := range cases { + if got := normalizeDetectedLanguage(in); got != want { + t.Errorf("normalizeDetectedLanguage(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/subtitles/ai/translator.go b/internal/subtitles/ai/translator.go index b8b64265..fb065e13 100644 --- a/internal/subtitles/ai/translator.go +++ b/internal/subtitles/ai/translator.go @@ -2,30 +2,29 @@ package ai import ( "context" - "encoding/json" "fmt" "strconv" "strings" + + "github.com/Silo-Server/silo-server/internal/ai/llm" + aitranslate "github.com/Silo-Server/silo-server/internal/ai/translate" ) -// LLMTranslator translates subtitle cues with an OpenAI-compatible chat model. -// -// Cues are translated in batches. Each batch is sent as a JSON object keyed by -// cue number; the model must return the same keys with translated values. Only -// the text is sent to the model — timestamps never leave the server — so timing -// alignment is structurally guaranteed. A few preceding source cues are -// included as untranslated context so the model can keep scene continuity -// across batch boundaries. +// LLMTranslator translates subtitle cues with an OpenAI-compatible chat model +// via the shared aitranslate batch protocol. Only the text is sent to the +// model — timestamps never leave the server — so timing alignment is +// structurally guaranteed. A few preceding source cues are included as +// untranslated context so the model can keep scene continuity across batch +// boundaries. type LLMTranslator struct { - client *Client + client *llm.Client batchSize int contextNeighbors int - maxRetries int } -// NewLLMTranslator builds a translator. batchSize and contextNeighbors fall back -// to sane defaults when non-positive. -func NewLLMTranslator(client *Client, batchSize, contextNeighbors int) *LLMTranslator { +// NewLLMTranslator builds a translator. batchSize and contextNeighbors fall +// back to sane defaults when non-positive. +func NewLLMTranslator(client *llm.Client, batchSize, contextNeighbors int) *LLMTranslator { if batchSize <= 0 { batchSize = 40 } @@ -36,7 +35,6 @@ func NewLLMTranslator(client *Client, batchSize, contextNeighbors int) *LLMTrans client: client, batchSize: batchSize, contextNeighbors: contextNeighbors, - maxRetries: 2, } } @@ -45,9 +43,6 @@ func (t *LLMTranslator) Translate(ctx context.Context, req TranslateRequest, onB if t.client == nil { return nil, fmt.Errorf("translator client is nil") } - if t.batchSize <= 0 { - return nil, fmt.Errorf("invalid batch size: %d", t.batchSize) - } if strings.TrimSpace(req.TargetLanguage) == "" { return nil, fmt.Errorf("target language is required") } @@ -60,97 +55,48 @@ func (t *LLMTranslator) Translate(ctx context.Context, req TranslateRequest, onB out := make([]SubtitleCue, total) copy(out, req.Cues) - srcName := languageDisplayName(req.SourceLanguage) - tgtName := languageDisplayName(req.TargetLanguage) - system := translationSystemPrompt(srcName, tgtName) + segments := make([]aitranslate.Segment, total) + for i, c := range req.Cues { + segments[i] = aitranslate.Segment{ID: strconv.Itoa(i + 1), Text: strings.Join(c.Lines, "\n")} + } - for start := 0; start < total; start += t.batchSize { - if err := ctx.Err(); err != nil { - return nil, err - } - end := min(start+t.batchSize, total) + srcName := aitranslate.LanguageDisplayName(req.SourceLanguage) + tgtName := aitranslate.LanguageDisplayName(req.TargetLanguage) - contextStart := max(0, start-t.contextNeighbors) - translated, err := t.translateBatch(ctx, system, tgtName, req.Cues[contextStart:start], req.Cues[start:end]) - if err != nil { - return nil, fmt.Errorf("translate cues %d-%d: %w", start+1, end, err) + chat := func(ctx context.Context, system, user string) (string, error) { + messages := []llm.Message{ + {Role: "system", Content: system}, + {Role: "user", Content: user}, } - for i, lines := range translated { - out[start+i].Lines = lines + return t.client.Chat(ctx, messages, true) + } + + translated, err := aitranslate.Translate(ctx, chat, aitranslate.Request{ + Segments: segments, + SystemPrompt: translationSystemPrompt(srcName, tgtName), + TargetName: tgtName, + EntryNoun: "cues", + BatchSize: t.batchSize, + ContextNeighbors: t.contextNeighbors, + }, func(batch []aitranslate.Segment, done, totalSegs int) { + // Batches are sequential ranges, so this batch covers [done-len, done). + start := done - len(batch) + for i, seg := range batch { + out[start+i].Lines = splitCueLines(seg.Text) } if onBatch != nil { - onBatch(out[start:end], end, total) + onBatch(out[start:done], done, totalSegs) } - } - - return out, nil -} - -func (t *LLMTranslator) translateBatch(ctx context.Context, system, targetName string, contextCues, batch []SubtitleCue) ([][]string, error) { - texts := make([]string, len(batch)) - for i, c := range batch { - texts[i] = strings.Join(c.Lines, "\n") - } - payload, err := buildIndexedJSON(texts) + }) if err != nil { return nil, err } - var user strings.Builder - if len(contextCues) > 0 { - user.WriteString("Preceding lines for context only — do not translate or include them in your output:\n") - for _, c := range contextCues { - user.WriteString(strings.Join(c.Lines, " ")) - user.WriteByte('\n') - } - user.WriteByte('\n') + // Map the full result too: covers callers without an onBatch callback. + for i, seg := range translated { + out[i].Lines = splitCueLines(seg.Text) } - fmt.Fprintf(&user, "Translate these %d cues into %s. Respond with only the JSON object:\n%s", len(batch), targetName, payload) - - messages := []chatMessage{ - {Role: "system", Content: system}, - {Role: "user", Content: user.String()}, - } - - var lastErr error - for attempt := 0; attempt <= t.maxRetries; attempt++ { - if err := ctx.Err(); err != nil { - return nil, err - } - content, err := t.client.chat(ctx, messages, true) - if err != nil { - return nil, err // transport/API errors are already retried inside chat - } - - obj, err := extractJSONObject(content) - if err != nil { - lastErr = err - continue - } - var m map[string]string - if err := json.Unmarshal([]byte(obj), &m); err != nil { - lastErr = fmt.Errorf("decode translation JSON: %w", err) - continue - } - - out := make([][]string, len(batch)) - complete := true - for i := range batch { - v, ok := m[strconv.Itoa(i+1)] - if !ok { - complete = false - break - } - out[i] = splitCueLines(v) - } - if !complete { - lastErr = fmt.Errorf("model omitted one or more cues") - continue - } - return out, nil - } - - return nil, fmt.Errorf("invalid model response after %d attempts: %w", t.maxRetries+1, lastErr) + return out, nil } func translationSystemPrompt(srcName, tgtName string) string { @@ -169,57 +115,6 @@ func translationSystemPrompt(srcName, tgtName string) string { ) } -// buildIndexedJSON renders texts as a JSON object {"1":..., "2":...} keyed by -// 1-based cue number, escaping each value safely. It is built by hand rather -// than json.Marshal'ing a map so the keys stay in numeric order — that reads -// more naturally for the model than the lexicographic order Go emits for maps -// ("1","10","11",...,"2"). Correctness doesn't depend on order (results are -// mapped back by key), but ordered input gives the model better scene context. -func buildIndexedJSON(texts []string) (string, error) { - var b strings.Builder - b.WriteByte('{') - for i, text := range texts { - if i > 0 { - b.WriteByte(',') - } - key, err := json.Marshal(strconv.Itoa(i + 1)) - if err != nil { - return "", err - } - val, err := json.Marshal(text) - if err != nil { - return "", err - } - b.Write(key) - b.WriteByte(':') - b.Write(val) - } - b.WriteByte('}') - return b.String(), nil -} - -// extractJSONObject pulls the first balanced-looking JSON object out of a model -// reply, tolerating ``` code fences and surrounding prose. -func extractJSONObject(s string) (string, error) { - s = strings.TrimSpace(s) - if strings.HasPrefix(s, "```") { - s = strings.TrimPrefix(s, "```") - if nl := strings.IndexByte(s, '\n'); nl >= 0 { - s = s[nl+1:] - } - if idx := strings.LastIndex(s, "```"); idx >= 0 { - s = s[:idx] - } - s = strings.TrimSpace(s) - } - start := strings.IndexByte(s, '{') - end := strings.LastIndexByte(s, '}') - if start < 0 || end < 0 || end < start { - return "", fmt.Errorf("no JSON object found in model response") - } - return s[start : end+1], nil -} - func splitCueLines(v string) []string { lines := strings.Split(v, "\n") for len(lines) > 0 && strings.TrimSpace(lines[len(lines)-1]) == "" { diff --git a/internal/userstore/pgstore/profiles.go b/internal/userstore/pgstore/profiles.go index e2a8cc8f..4131a23a 100644 --- a/internal/userstore/pgstore/profiles.go +++ b/internal/userstore/pgstore/profiles.go @@ -20,7 +20,7 @@ func scanProfile(scanner interface { var createdAt, updatedAt time.Time err := scanner.Scan( &p.ID, &p.Name, &p.Avatar, &p.PINHash, &p.IsChild, &p.IsPrimary, &p.MaxContentRating, - &p.QualityPreference, &p.Language, &p.SubtitleLanguage, &p.SubtitleMode, + &p.QualityPreference, &p.Language, &p.PreferredMetadataLanguage, &p.SubtitleLanguage, &p.SubtitleMode, &p.AutoSkipIntro, &p.AutoSkipCredits, &p.AutoSkipRecap, &p.AutoPlayNextPreview, &p.LibraryRestrictionsEnabled, &p.ShowForcedSubtitles, &p.MaxPlaybackQuality, &createdAt, &updatedAt, @@ -66,13 +66,13 @@ func (s *PostgresUserStore) CreateProfile(ctx context.Context, p userstore.Profi _, err := s.pool.Exec(ctx, ` INSERT INTO user_profiles ( id, user_id, name, avatar, pin_hash, is_child, is_primary, max_content_rating, - quality_preference, language, subtitle_language, subtitle_mode, + quality_preference, language, preferred_metadata_language, subtitle_language, subtitle_mode, auto_skip_intro, auto_skip_credits, auto_skip_recap, auto_play_next_preview, library_restrictions_enabled, show_forced_subtitles, max_playback_quality, created_at, updated_at - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21)`, + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22)`, p.ID, s.userID, p.Name, p.Avatar, p.PINHash, p.IsChild, p.IsPrimary, p.MaxContentRating, - p.QualityPreference, p.Language, p.SubtitleLanguage, p.SubtitleMode, + p.QualityPreference, p.Language, p.PreferredMetadataLanguage, p.SubtitleLanguage, p.SubtitleMode, p.AutoSkipIntro, p.AutoSkipCredits, p.AutoSkipRecap, p.AutoPlayNextPreview, p.LibraryRestrictionsEnabled, p.ShowForcedSubtitles, p.MaxPlaybackQuality, p.CreatedAt, p.UpdatedAt, @@ -89,7 +89,7 @@ func (s *PostgresUserStore) CreateProfile(ctx context.Context, p userstore.Profi func (s *PostgresUserStore) GetProfile(ctx context.Context, id string) (*userstore.Profile, error) { row := s.pool.QueryRow(ctx, ` SELECT id, name, avatar, pin_hash, is_child, is_primary, max_content_rating, - quality_preference, language, subtitle_language, subtitle_mode, + quality_preference, language, preferred_metadata_language, subtitle_language, subtitle_mode, auto_skip_intro, auto_skip_credits, auto_skip_recap, auto_play_next_preview, library_restrictions_enabled, show_forced_subtitles, max_playback_quality, created_at, updated_at FROM user_profiles WHERE user_id = $1 AND id = $2`, s.userID, id) @@ -111,7 +111,7 @@ func (s *PostgresUserStore) GetProfile(ctx context.Context, id string) (*usersto func (s *PostgresUserStore) ListProfiles(ctx context.Context) ([]userstore.Profile, error) { rows, err := s.pool.Query(ctx, ` SELECT id, name, avatar, pin_hash, is_child, is_primary, max_content_rating, - quality_preference, language, subtitle_language, subtitle_mode, + quality_preference, language, preferred_metadata_language, subtitle_language, subtitle_mode, auto_skip_intro, auto_skip_credits, auto_skip_recap, auto_play_next_preview, library_restrictions_enabled, show_forced_subtitles, max_playback_quality, created_at, updated_at FROM user_profiles WHERE user_id = $1 ORDER BY created_at ASC`, s.userID) @@ -185,6 +185,9 @@ func (s *PostgresUserStore) UpdateProfile(ctx context.Context, id string, u user if u.Language != nil { addArg("language", *u.Language) } + if u.PreferredMetadataLanguage != nil { + addArg("preferred_metadata_language", *u.PreferredMetadataLanguage) + } if u.SubtitleLanguage != nil { addArg("subtitle_language", *u.SubtitleLanguage) } diff --git a/internal/userstore/types.go b/internal/userstore/types.go index bc06098e..e8d34ac1 100644 --- a/internal/userstore/types.go +++ b/internal/userstore/types.go @@ -13,6 +13,7 @@ type Profile struct { MaxContentRating string QualityPreference string Language string + PreferredMetadataLanguage string // ISO 639-1; "" = inherit library metadata language SubtitleLanguage string SubtitleMode string AutoSkipIntro bool @@ -58,6 +59,7 @@ type UpdateProfileInput struct { MaxContentRating *string QualityPreference *string Language *string + PreferredMetadataLanguage *string SubtitleLanguage *string SubtitleMode *string AutoSkipIntro *bool diff --git a/migrations/sql/20260610143224_ai_metadata_translation_and_asr.sql b/migrations/sql/20260610143224_ai_metadata_translation_and_asr.sql new file mode 100644 index 00000000..809133ba --- /dev/null +++ b/migrations/sql/20260610143224_ai_metadata_translation_and_asr.sql @@ -0,0 +1,73 @@ +-- +goose Up +-- +goose StatementBegin +-- AI metadata translation jobs: on-demand (and per-library auto-fallback) +-- translation of overviews/taglines into the localization tables. Each row is +-- one job covering a media item (optionally expanded to its seasons/episodes), +-- one season, or one episode. Results are ordinary localization rows, so they +-- reach every client through the existing localized detail responses. +CREATE TABLE public.metadata_translation_jobs ( + id bigserial PRIMARY KEY, + target_kind text NOT NULL, -- 'item' | 'season' | 'episode' + content_id text NOT NULL, + include_children boolean NOT NULL DEFAULT true, -- series item: include season/episode overviews + source_language text NOT NULL DEFAULT '', + target_language text NOT NULL, + engine text NOT NULL DEFAULT 'openai', + model text NOT NULL DEFAULT '', -- snapshot of the model used, for provenance + status text NOT NULL DEFAULT 'pending', -- pending|running|completed|failed|cancelled + progress double precision NOT NULL DEFAULT 0, -- 0..1 + progress_message text NOT NULL DEFAULT '', + fields_done integer NOT NULL DEFAULT 0, + fields_total integer NOT NULL DEFAULT 0, + force boolean NOT NULL DEFAULT false, -- overwrite provider/ai fields (never manual) + error_message text NOT NULL DEFAULT '', + idempotency_key text NOT NULL, + requested_by integer, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + heartbeat_at timestamptz NOT NULL DEFAULT now() +); + +-- Prevent duplicate in-flight jobs for the same work. Completed/failed/cancelled +-- rows do not block a re-run, so a user can retry after a failure. +CREATE UNIQUE INDEX metadata_translation_jobs_active_idempotency_idx + ON public.metadata_translation_jobs (idempotency_key) + WHERE status IN ('pending', 'running'); + +-- Listing recent jobs for an item (metadata editor progress UI, admin views). +CREATE INDEX metadata_translation_jobs_content_idx + ON public.metadata_translation_jobs (content_id, created_at DESC); + +-- Startup recovery scan for jobs left running by a crashed process. +CREATE INDEX metadata_translation_jobs_status_idx + ON public.metadata_translation_jobs (status) WHERE status IN ('pending', 'running'); + +-- Field provenance on the AI-writable localization fields, so refreshes never +-- regress quality: 'manual' beats 'provider' beats 'ai'. A provider refresh +-- overwrites an AI translation; the reverse never happens. +ALTER TABLE media_item_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')), + ADD COLUMN tagline_source text NOT NULL DEFAULT 'provider' + CHECK (tagline_source IN ('provider', 'ai', 'manual')); +ALTER TABLE season_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')); +ALTER TABLE episode_localizations + ADD COLUMN overview_source text NOT NULL DEFAULT 'provider' + CHECK (overview_source IN ('provider', 'ai', 'manual')); + +-- Per-library opt-in: when metadata providers have no localization for the +-- library's metadata language, fall back to AI translation on refresh. +ALTER TABLE media_folders + ADD COLUMN auto_translate_metadata boolean NOT NULL DEFAULT false; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE media_folders DROP COLUMN auto_translate_metadata; +ALTER TABLE episode_localizations DROP COLUMN overview_source; +ALTER TABLE season_localizations DROP COLUMN overview_source; +ALTER TABLE media_item_localizations DROP COLUMN tagline_source, DROP COLUMN overview_source; +DROP TABLE IF EXISTS public.metadata_translation_jobs; +-- +goose StatementEnd diff --git a/migrations/sql/20260610153638_profile_preferred_metadata_language.sql b/migrations/sql/20260610153638_profile_preferred_metadata_language.sql new file mode 100644 index 00000000..7acb28e0 --- /dev/null +++ b/migrations/sql/20260610153638_profile_preferred_metadata_language.sql @@ -0,0 +1,13 @@ +-- +goose Up +-- +goose StatementBegin +-- Per-profile metadata (presentation) language. Empty = inherit the library's +-- metadata_language. When set, catalog serving prefers this language's +-- localizations, and the on-view AI translation flow targets it. +ALTER TABLE user_profiles + ADD COLUMN preferred_metadata_language text NOT NULL DEFAULT ''; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE user_profiles DROP COLUMN preferred_metadata_language; +-- +goose StatementEnd diff --git a/web/src/api/types.ts b/web/src/api/types.ts index ae62dc73..beea5d27 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -125,6 +125,7 @@ export interface Profile { max_content_rating: string; quality_preference: string; language: string; + preferred_metadata_language?: string; subtitle_language: string; subtitle_mode: string; show_forced_subtitles?: boolean; @@ -152,6 +153,7 @@ export interface CreateProfileRequest { max_content_rating?: string; quality_preference?: string; language?: string; + preferred_metadata_language?: string; subtitle_language?: string; subtitle_mode?: string; show_forced_subtitles?: boolean; @@ -958,6 +960,12 @@ export interface ItemDetail { year: number; overview: string; tagline?: string; + /** + * When set, the viewer's presentation language is missing a localized + * description; on-view AI translation (auto or button, per server config) + * keys off this. + */ + pending_translation_language?: string; runtime: number; content_rating: string; genres: string[]; @@ -2364,6 +2372,7 @@ export interface Library { name: string; enabled: boolean; metadata_language: string; + auto_translate_metadata: boolean; chapter_thumbnails_enabled: boolean; chapter_thumbnails_supported: boolean; intro_detection_enabled: boolean; @@ -2534,6 +2543,7 @@ export interface CreateLibraryRequest { name: string; enabled?: boolean; metadata_language?: string; + auto_translate_metadata?: boolean; chapter_thumbnails_enabled?: boolean; intro_detection_enabled?: boolean; } diff --git a/web/src/components/EditMetadataDialog.tsx b/web/src/components/EditMetadataDialog.tsx index 053da6e0..3ebd0be4 100644 --- a/web/src/components/EditMetadataDialog.tsx +++ b/web/src/components/EditMetadataDialog.tsx @@ -5,6 +5,7 @@ import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; import { ConfirmDialog } from "@/components/ConfirmDialog"; +import { MetadataTranslatePanel } from "@/components/MetadataTranslatePanel"; import { TagInput } from "@/components/TagInput"; import ImageSelectorTab from "@/components/ImageSelectorTab"; import type { ItemDetail } from "@/api/types"; @@ -340,6 +341,10 @@ export default function EditMetadataDialog({ item, open, onOpenChange }: EditMet /> + {(item.type === "movie" || item.type === "series") && ( + + )} +
(jobsData?.jobs ?? []).find(isActive), [jobsData]); + const lastJob = jobsData?.jobs?.[0]; + + // When the job we were watching reaches a terminal state, surface the + // outcome and refresh the (now possibly re-localized) detail surfaces. + const sawActiveRef = useRef(false); + useEffect(() => { + if (activeJob) { + sawActiveRef.current = true; + return; + } + if (!sawActiveRef.current || !lastJob) return; + sawActiveRef.current = false; + setWatching(false); + if (lastJob.status === "completed") { + toast.success( + lastJob.fields_total === 0 + ? "Nothing to translate — all descriptions are already localized." + : `Translated ${lastJob.fields_done} description${lastJob.fields_done === 1 ? "" : "s"}.`, + ); + void invalidateMediaSurfaceQueries(queryClient, { itemId: item.content_id }); + } else if (lastJob.status === "failed") { + toast.error(lastJob.error_message || "Translation failed."); + } + }, [activeJob, lastJob, queryClient, item.content_id]); + + if (!enabled) return null; + + const busy = translateMutation.isPending || Boolean(activeJob); + + function start() { + if (!targetLang) { + toast.error("Pick a target language first."); + return; + } + translateMutation.mutate( + { target_language: targetLang, include_children: true, force }, + { onSuccess: () => setWatching(true) }, + ); + } + + return ( +
+
+ + Translate with AI +
+

+ Translates the overview and tagline + {item.type === "series" ? ", plus all season and episode overviews," : ""} into the chosen + language. Translations are served to libraries using that metadata language; provider data + replaces them when it becomes available. +

+
+
+ + +
+
+ + +
+ +
+ {activeJob && ( +

+ {activeJob.progress_message || "Working"}…{" "} + {activeJob.fields_total > 0 && + `${activeJob.fields_done}/${activeJob.fields_total} fields`} +

+ )} +
+ ); +} diff --git a/web/src/components/admin/libraries/LibraryFormSections.tsx b/web/src/components/admin/libraries/LibraryFormSections.tsx index b9c2d895..3bed5a67 100644 --- a/web/src/components/admin/libraries/LibraryFormSections.tsx +++ b/web/src/components/admin/libraries/LibraryFormSections.tsx @@ -286,6 +286,21 @@ export function MetadataFields({ form }: { form: LibraryFormController }) { Preferred language for titles, summaries, and artwork fetched from providers.

+
+
+ +

+ When providers have no translation for this library's language, translate + descriptions with AI after each refresh. Requires AI description translation in Admin + Settings → AI Services. +

+
+ +
{form.contentLevels.length > 0 && (
diff --git a/web/src/components/admin/libraries/useLibraryForm.ts b/web/src/components/admin/libraries/useLibraryForm.ts index 164b7b47..d454b4e1 100644 --- a/web/src/components/admin/libraries/useLibraryForm.ts +++ b/web/src/components/admin/libraries/useLibraryForm.ts @@ -147,6 +147,9 @@ export function useLibraryForm({ const [type, setType] = useState(library?.type ?? "movies"); const [enabled, setEnabled] = useState(library?.enabled ?? true); const [metadataLanguage, setMetadataLanguage] = useState(library?.metadata_language ?? "en"); + const [autoTranslateMetadata, setAutoTranslateMetadata] = useState( + library?.auto_translate_metadata ?? false, + ); const [chapterThumbnailsEnabled, setChapterThumbnailsEnabled] = useState( library?.chapter_thumbnails_enabled ?? false, ); @@ -284,6 +287,7 @@ export function useLibraryForm({ type, enabled, metadata_language: metadataLanguage, + auto_translate_metadata: autoTranslateMetadata, chapter_thumbnails_enabled: chapterThumbnailsEnabled, intro_detection_enabled: introDetectionEnabled, }; @@ -343,6 +347,8 @@ export function useLibraryForm({ setEnabled, metadataLanguage, setMetadataLanguage, + autoTranslateMetadata, + setAutoTranslateMetadata, chapterThumbnailsEnabled, setChapterThumbnailsEnabled, introDetectionEnabled, diff --git a/web/src/hooks/queries/items.ts b/web/src/hooks/queries/items.ts index af851781..3a9a95f0 100644 --- a/web/src/hooks/queries/items.ts +++ b/web/src/hooks/queries/items.ts @@ -431,3 +431,81 @@ export function useApplyItemImage() { }, }); } + +// --------------------------------------------------------------------------- +// Metadata AI translation (descriptions into the localization tables) +// --------------------------------------------------------------------------- + +export interface MetadataTranslationJob { + id: number; + target_kind: string; + content_id: string; + include_children: boolean; + source_language: string; + target_language: string; + status: "pending" | "running" | "completed" | "failed" | "cancelled"; + progress: number; + progress_message: string; + fields_done: number; + fields_total: number; + force: boolean; + error_message?: string; + created_at: string; + updated_at: string; +} + +/** Whether the server has metadata AI translation configured, and the + * viewer-facing on-view mode. */ +export function useMetadataAIStatus(enabled = true) { + return useQuery({ + queryKey: ["metadata-ai", "status"], + queryFn: () => + api<{ enabled: boolean; on_view?: "off" | "button" | "auto" }>("/metadata/ai/status"), + staleTime: 5 * 60 * 1000, + enabled, + }); +} + +export interface TranslateItemMetadataRequest { + target_language: string; + include_children?: boolean; + force?: boolean; +} + +export function useTranslateItemMetadata(contentId: string) { + const queryClient = useQueryClient(); + return useMutation({ + mutationFn: (body: TranslateItemMetadataRequest) => + api<{ job: MetadataTranslationJob }>( + `/admin/items/${itemPathID(contentId)}/metadata-translation`, + { method: "POST", body: JSON.stringify(body) }, + ), + onSuccess: () => { + void queryClient.invalidateQueries({ + queryKey: ["metadata-translation-jobs", contentId], + }); + }, + onError: (err) => { + toast.error(err instanceof Error ? err.message : "Failed to start translation"); + }, + }); +} + +/** + * Recent translation jobs for an item. Polls while a job is active so the + * metadata editor can show live progress without a websocket. + */ +export function useMetadataTranslationJobs(contentId: string, enabled: boolean) { + return useQuery({ + queryKey: ["metadata-translation-jobs", contentId], + queryFn: () => + api<{ jobs: MetadataTranslationJob[] }>( + `/admin/items/${itemPathID(contentId)}/metadata-translation/jobs`, + ), + enabled, + refetchInterval: (query) => { + const jobs = query.state.data?.jobs ?? []; + return jobs.some((j) => j.status === "pending" || j.status === "running") ? 1500 : false; + }, + }); +} diff --git a/web/src/hooks/useOnViewTranslation.ts b/web/src/hooks/useOnViewTranslation.ts new file mode 100644 index 00000000..829471fe --- /dev/null +++ b/web/src/hooks/useOnViewTranslation.ts @@ -0,0 +1,85 @@ +import { useCallback, useEffect, useRef, useState } from "react"; +import { useQueryClient } from "@tanstack/react-query"; + +import { api } from "@/api/client"; +import type { ItemDetail } from "@/api/types"; +import { useMetadataAIStatus } from "@/hooks/queries/items"; + +// Give up shimmering after this long; the original text stays and the +// server-side cooldown keeps a failing endpoint from being re-hit per view. +const TRANSLATE_TIMEOUT_MS = 45_000; +const POLL_INTERVAL_MS = 2_000; + +/** + * Viewer-facing on-demand description translation for the detail page. + * + * When the detail response carries `pending_translation_language` (the + * description is not in this profile's language and no localization exists), + * the server's `metadata_ai.on_view` mode decides the UX: + * - "auto": fire the translation on view (once per item+language) and pulse + * the description until the refetched detail comes back localized. + * - "button": expose a translate trigger for the chip under the overview. + * Completion is observed as the flag clearing on refetch — the job's first + * batch translates the item's own overview, so this lands in seconds. + */ +export function useOnViewTranslation(item: ItemDetail | undefined) { + const queryClient = useQueryClient(); + const { data: status } = useMetadataAIStatus(); + const mode = status?.on_view ?? "off"; + + const contentId = item?.content_id ?? ""; + const pendingLanguage = item?.pending_translation_language ?? ""; + + const [translating, setTranslating] = useState(false); + // Tracks which item+language we already fired for, so auto mode triggers + // once per view rather than re-firing on every refetch while polling. + const firedForRef = useRef(""); + + const trigger = useCallback(() => { + if (!contentId || !pendingLanguage) return; + const key = `${contentId}:${pendingLanguage}`; + if (firedForRef.current === key) return; + firedForRef.current = key; + setTranslating(true); + api(`/items/${encodeURIComponent(contentId)}/translate-description`, { + method: "POST", + body: JSON.stringify({ target_language: pendingLanguage }), + }).catch(() => { + setTranslating(false); + }); + }, [contentId, pendingLanguage]); + + // Auto mode: translate on view. + useEffect(() => { + if (mode === "auto" && pendingLanguage) trigger(); + }, [mode, pendingLanguage, trigger]); + + // While translating, poll the detail; the localized overview replaces the + // text and clears the pending flag, which ends the shimmer below. + useEffect(() => { + if (!translating || !contentId) return; + const startedAt = Date.now(); + const timer = setInterval(() => { + if (Date.now() - startedAt > TRANSLATE_TIMEOUT_MS) { + setTranslating(false); + return; + } + // Prefix invalidation covers the per-library detail key variants + // (["catalog", "items", id, "detail", ]). + void queryClient.invalidateQueries({ queryKey: ["catalog", "items", contentId] }); + }, POLL_INTERVAL_MS); + return () => clearInterval(timer); + }, [translating, contentId, queryClient]); + + // The refetched detail no longer reports a missing language: done. + useEffect(() => { + if (translating && !pendingLanguage) setTranslating(false); + }, [translating, pendingLanguage]); + + return { + /** Pulse the description text. */ + translating, + /** Render the explicit translate chip (button mode only). */ + onTranslate: mode === "button" && pendingLanguage && !translating ? trigger : undefined, + }; +} diff --git a/web/src/pages/ItemDetail/DetailHero.tsx b/web/src/pages/ItemDetail/DetailHero.tsx index db59dc37..0c113243 100644 --- a/web/src/pages/ItemDetail/DetailHero.tsx +++ b/web/src/pages/ItemDetail/DetailHero.tsx @@ -1,4 +1,5 @@ import { type ReactNode, useState } from "react"; +import { Languages } from "lucide-react"; import { decodeThumbhash } from "@/lib/thumbhash"; interface DetailHeroProps { @@ -21,6 +22,10 @@ interface DetailHeroProps { */ genreHref?: (genre: string) => string; overview?: string; + /** Pulses the overview while an on-view AI translation is in flight. */ + overviewTranslating?: boolean; + /** When set, renders a small "Translate" chip under the overview. */ + onTranslateOverview?: () => void; actions?: ReactNode; aside?: ReactNode; studioLabel?: string; @@ -46,6 +51,8 @@ export default function DetailHero({ genres, genreHref, overview, + overviewTranslating = false, + onTranslateOverview, actions, aside, studioLabel, @@ -227,13 +234,31 @@ export default function DetailHero({ {scoreRow &&
{scoreRow}
} {overview && ( -

- {overview} -

+
+

+ {overview} +

+ {overviewTranslating && ( + + + Translating… + + )} + {!overviewTranslating && onTranslateOverview && ( + + )} +
)} {/* Crew line replaces genres when provided */} diff --git a/web/src/pages/ItemDetail/MovieContent.test.tsx b/web/src/pages/ItemDetail/MovieContent.test.tsx index cbba5089..79cb9ad4 100644 --- a/web/src/pages/ItemDetail/MovieContent.test.tsx +++ b/web/src/pages/ItemDetail/MovieContent.test.tsx @@ -33,6 +33,10 @@ const mocks = vi.hoisted(() => { }; }); +vi.mock("@/hooks/useOnViewTranslation", () => ({ + useOnViewTranslation: () => ({ translating: false, onTranslate: undefined }), +})); + vi.mock("@/hooks/queries/favorites", () => ({ useIsFavorite: mocks.useIsFavorite, useToggleFavorite: mocks.useToggleFavorite, diff --git a/web/src/pages/ItemDetail/MovieContent.tsx b/web/src/pages/ItemDetail/MovieContent.tsx index 8bc1a6d1..6d39ae1d 100644 --- a/web/src/pages/ItemDetail/MovieContent.tsx +++ b/web/src/pages/ItemDetail/MovieContent.tsx @@ -19,6 +19,7 @@ import MatchItemDialog from "@/components/MatchItemDialog"; import PageBack from "@/components/PageBack"; import RecommendationGrid from "@/components/RecommendationGrid"; import DetailHero from "./DetailHero"; +import { useOnViewTranslation } from "@/hooks/useOnViewTranslation"; import MetadataBadges from "./components/MetadataBadges"; import QualityBadges from "./components/QualityBadges"; import ScoreRow from "./components/ScoreRow"; @@ -43,6 +44,8 @@ function formatDuration(minutes: number): string { } export default function MovieContent({ item }: { item: ItemDetail & { type: "movie" } }) { + const { translating: overviewTranslating, onTranslate: onTranslateOverview } = + useOnViewTranslation(item); const navigate = useNavigate(); useAmbientColor(item.backdrop_thumbhash); const { user } = useAuth(); @@ -213,6 +216,8 @@ export default function MovieContent({ item }: { item: ItemDetail & { type: "mov /> } overview={item.overview} + overviewTranslating={overviewTranslating} + onTranslateOverview={onTranslateOverview} crewLine={} actions={ { }; }); +vi.mock("@/hooks/useOnViewTranslation", () => ({ + useOnViewTranslation: () => ({ translating: false, onTranslate: undefined }), +})); + vi.mock("@/hooks/useAuth", () => ({ useAuth: mocks.useAuth, useOptionalAuth: mocks.useAuth, diff --git a/web/src/pages/ItemDetail/SeriesContent.tsx b/web/src/pages/ItemDetail/SeriesContent.tsx index bfd52db5..1e022ede 100644 --- a/web/src/pages/ItemDetail/SeriesContent.tsx +++ b/web/src/pages/ItemDetail/SeriesContent.tsx @@ -17,6 +17,7 @@ import MatchItemDialog from "@/components/MatchItemDialog"; import PageBack from "@/components/PageBack"; import RecommendationGrid from "@/components/RecommendationGrid"; import DetailHero from "./DetailHero"; +import { useOnViewTranslation } from "@/hooks/useOnViewTranslation"; import SeasonCarousel from "./SeasonCarousel"; import SeasonEpisodeGrid from "./components/SeasonEpisodeGrid"; import MetadataBadges from "./components/MetadataBadges"; @@ -29,6 +30,8 @@ import { getWatchedActionLabel } from "./watchedState"; import { canCurateMetadata as canCurateMetadataForUser } from "@/lib/permissions"; export default function SeriesContent({ item }: { item: ItemDetail & { type: "series" } }) { + const { translating: overviewTranslating, onTranslate: onTranslateOverview } = + useOnViewTranslation(item); const navigate = useNavigate(); useAmbientColor(item.backdrop_thumbhash); const { user } = useAuth(); @@ -144,6 +147,8 @@ export default function SeriesContent({ item }: { item: ItemDetail & { type: "se /> } overview={item.overview} + overviewTranslating={overviewTranslating} + onTranslateOverview={onTranslateOverview} crewLine={ } diff --git a/web/src/pages/admin-settings/AIServicesSettings.tsx b/web/src/pages/admin-settings/AIServicesSettings.tsx new file mode 100644 index 00000000..3966571b --- /dev/null +++ b/web/src/pages/admin-settings/AIServicesSettings.tsx @@ -0,0 +1,408 @@ +import { useState, useEffect } from "react"; +import { toast } from "sonner"; +import { + useAdminSensitiveStatus, + useAdminServerSettings, + useUpdateServerSetting, +} from "@/hooks/queries/admin/settings"; + +import { Button } from "@/components/ui/button"; +import { CredentialStatus } from "./CredentialStatus"; +import { SettingField } from "./SettingField"; + +// Connection settings live under the ai.* keys; reads fall back to the legacy +// subtitle_ai.* rows (mirroring the server's loader) so an existing setup +// shows its effective values, while saves always write the new keys. + +// Chat-only gateways have no timestamped transcription API; the server +// rejects them for the transcription URL — mirror that check for instant +// feedback. Keep in sync with llm.IsChatOnlyGateway. +const CHAT_ONLY_GATEWAY_HOSTS = ["openrouter.ai"]; + +function isChatOnlyGateway(rawUrl: string): boolean { + const trimmed = rawUrl.trim(); + if (!trimmed) return false; + try { + const host = new URL( + trimmed.includes("://") ? trimmed : `https://${trimmed}`, + ).hostname.toLowerCase(); + return CHAT_ONLY_GATEWAY_HOSTS.some((g) => host === g || host.endsWith(`.${g}`)); + } catch { + return false; + } +} + +// Recommended transcription endpoints, fastest path first. Clicking a preset +// fills the URL + model; the API key still comes from the operator. +const TRANSCRIPTION_PRESETS: { + id: string; + label: string; + description: string; + baseUrl: string; + model: string; +}[] = [ + { + id: "local", + label: "Self-hosted · recommended", + description: + "A speaches/faster-whisper server on your own hardware — private, free, and no rate limits. Adjust the URL to where it runs; no API key needed.", + baseUrl: "http://localhost:8000", + model: "deepdml/faster-whisper-large-v3-turbo-ct2", + }, + { + id: "groq-turbo", + label: "Groq · hosted fallback", + description: + "whisper-large-v3-turbo on Groq — fastest hosted option, very low cost (free tier covers ~2 audio-hours per hour). Needs a Groq API key in the transcription key field.", + baseUrl: "https://api.groq.com/openai", + model: "whisper-large-v3-turbo", + }, + { + id: "groq-accurate", + label: "Groq · most accurate", + description: + "whisper-large-v3 on Groq — best multilingual accuracy among hosted options, slightly slower and pricier than turbo.", + baseUrl: "https://api.groq.com/openai", + model: "whisper-large-v3", + }, + { + id: "openai", + label: "OpenAI", + description: + "whisper-1 on OpenAI — solid quality, higher cost than Groq. Uses the main API key if the transcription key is blank.", + baseUrl: "https://api.openai.com", + model: "whisper-1", + }, +]; + +function AIConnectionCard() { + const { data: settings } = useAdminServerSettings(); + const { data: sensitive } = useAdminSensitiveStatus(); + const updateSetting = useUpdateServerSetting(); + + const configuredKeys = new Set(sensitive?.configured ?? []); + const apiKeyConfigured = + configuredKeys.has("ai.api_key") || configuredKeys.has("subtitle_ai.api_key"); + const asrApiKeyConfigured = configuredKeys.has("ai.asr_api_key"); + + const [baseUrl, setBaseUrl] = useState(""); + const [chatModel, setChatModel] = useState(""); + const [apiKey, setApiKey] = useState(""); + const [asrModel, setAsrModel] = useState(""); + const [asrBaseUrl, setAsrBaseUrl] = useState(""); + const [asrApiKey, setAsrApiKey] = useState(""); + const [maxConcurrent, setMaxConcurrent] = useState("2"); + + useEffect(() => { + if (!settings) return; + setBaseUrl( + settings["ai.base_url"] ?? settings["subtitle_ai.base_url"] ?? "https://api.openai.com", + ); + setChatModel(settings["ai.chat_model"] ?? settings["subtitle_ai.chat_model"] ?? "gpt-4o-mini"); + setAsrModel(settings["ai.asr_model"] ?? "whisper-1"); + setAsrBaseUrl(settings["ai.asr_base_url"] ?? ""); + setMaxConcurrent( + settings["ai.max_concurrent_jobs"] ?? settings["subtitle_ai.max_concurrent_jobs"] ?? "2", + ); + }, [settings]); + + function save() { + const trimmedBaseUrl = baseUrl.trim(); + const trimmedChatModel = chatModel.trim(); + const parsedMaxConcurrent = Number.parseInt(maxConcurrent, 10); + + if (trimmedBaseUrl === "" || trimmedChatModel === "") { + toast.error("Base URL and chat model are required."); + return; + } + if (!Number.isInteger(parsedMaxConcurrent) || parsedMaxConcurrent < 1) { + toast.error("Max concurrent jobs must be a positive whole number."); + return; + } + if (isChatOnlyGateway(asrBaseUrl)) { + toast.error( + "That endpoint can't produce timestamped transcriptions (chat-only gateway). Pick a transcription preset below or use a Whisper-capable server.", + ); + return; + } + + const updates = [ + updateSetting.mutateAsync({ key: "ai.base_url", value: trimmedBaseUrl }), + updateSetting.mutateAsync({ key: "ai.chat_model", value: trimmedChatModel }), + updateSetting.mutateAsync({ key: "ai.asr_model", value: asrModel.trim() }), + updateSetting.mutateAsync({ key: "ai.asr_base_url", value: asrBaseUrl.trim() }), + updateSetting.mutateAsync({ + key: "ai.max_concurrent_jobs", + value: String(parsedMaxConcurrent), + }), + ]; + if (apiKey.trim() !== "") { + updates.push(updateSetting.mutateAsync({ key: "ai.api_key", value: apiKey })); + } + if (asrApiKey.trim() !== "") { + updates.push(updateSetting.mutateAsync({ key: "ai.asr_api_key", value: asrApiKey })); + } + void Promise.all(updates).then(() => { + setApiKey(""); + setAsrApiKey(""); + }); + } + + return ( +
+
+
+

Endpoint

+

+ One OpenAI-compatible endpoint shared by every AI feature (OpenAI, Groq, a local Ollama + server, …). Transcription can use a separate Whisper-compatible server. +

+
+ +
+ + + +
+

Transcription

+

+ Subtitle generation needs a Whisper endpoint that returns segment timestamps. Pick a + preset or configure your own: +

+
+ {TRANSCRIPTION_PRESETS.map((preset) => { + const active = asrBaseUrl.trim() === preset.baseUrl && asrModel.trim() === preset.model; + return ( + + ); + })} +
+
+ + + + +
+ +

+ Changes take effect after a server restart. +

+
+
+ ); +} + +function AIFeaturesCard() { + const { data: settings } = useAdminServerSettings(); + const updateSetting = useUpdateServerSetting(); + + const [subtitleTranslate, setSubtitleTranslate] = useState("false"); + const [transcribe, setTranscribe] = useState("false"); + const [metadataTranslate, setMetadataTranslate] = useState("false"); + const [onView, setOnView] = useState("off"); + const [batchSize, setBatchSize] = useState("40"); + const [contextNeighbors, setContextNeighbors] = useState("2"); + const [asrChunkSeconds, setAsrChunkSeconds] = useState("600"); + + useEffect(() => { + if (!settings) return; + setSubtitleTranslate(settings["subtitle_ai.enabled"] ?? "false"); + setTranscribe(settings["subtitle_ai.transcribe_enabled"] ?? "false"); + setMetadataTranslate(settings["metadata_ai.enabled"] ?? "false"); + setOnView(settings["metadata_ai.on_view"] ?? "off"); + setBatchSize(settings["subtitle_ai.batch_size"] ?? "40"); + setContextNeighbors(settings["subtitle_ai.context_neighbors"] ?? "2"); + setAsrChunkSeconds(settings["subtitle_ai.asr_chunk_seconds"] ?? "600"); + }, [settings]); + + function save() { + const parsedBatch = Number.parseInt(batchSize, 10); + const parsedNeighbors = Number.parseInt(contextNeighbors, 10); + if (!Number.isInteger(parsedBatch) || parsedBatch < 1) { + toast.error("Batch size must be a positive whole number."); + return; + } + if (!Number.isInteger(parsedNeighbors) || parsedNeighbors < 0) { + toast.error("Context lines must be zero or a positive whole number."); + return; + } + const parsedChunkSeconds = Number.parseInt(asrChunkSeconds, 10); + if ( + !Number.isInteger(parsedChunkSeconds) || + parsedChunkSeconds < 60 || + parsedChunkSeconds > 600 + ) { + toast.error("Transcription chunk length must be between 60 and 600 seconds."); + return; + } + void Promise.all([ + updateSetting.mutateAsync({ key: "subtitle_ai.enabled", value: subtitleTranslate }), + updateSetting.mutateAsync({ key: "subtitle_ai.transcribe_enabled", value: transcribe }), + updateSetting.mutateAsync({ key: "metadata_ai.enabled", value: metadataTranslate }), + updateSetting.mutateAsync({ key: "metadata_ai.on_view", value: onView }), + updateSetting.mutateAsync({ key: "subtitle_ai.batch_size", value: String(parsedBatch) }), + updateSetting.mutateAsync({ + key: "subtitle_ai.context_neighbors", + value: String(parsedNeighbors), + }), + updateSetting.mutateAsync({ + key: "subtitle_ai.asr_chunk_seconds", + value: String(parsedChunkSeconds), + }), + ]); + } + + return ( +
+
+

Features

+

+ Everything runs once on the server and is served to every client through the normal + subtitle and metadata pipelines. +

+
+ + + + + + + +
+ +

+ Changes take effect after a server restart. +

+
+
+ ); +} + +export default function AIServicesSettings() { + return ( +
+
+

AI Services

+

+ Shared AI endpoint and feature toggles for subtitle translation, subtitle generation from + audio, and description translation. +

+
+ +
+ + +
+
+ ); +} diff --git a/web/src/pages/admin-settings/AdminSettingsLayout.tsx b/web/src/pages/admin-settings/AdminSettingsLayout.tsx index d729312f..57ac5bb0 100644 --- a/web/src/pages/admin-settings/AdminSettingsLayout.tsx +++ b/web/src/pages/admin-settings/AdminSettingsLayout.tsx @@ -15,6 +15,7 @@ import { Paintbrush, Layers, Subtitles, + Sparkles, } from "lucide-react"; import type { LucideIcon } from "lucide-react"; @@ -26,6 +27,7 @@ import PlaybackSettings from "./PlaybackSettings"; import ScannerSettings from "./ScannerSettings"; import IntroSettings from "./IntroSettings"; import SubtitlesSettings from "./SubtitlesSettings"; +import AIServicesSettings from "./AIServicesSettings"; import RateLimitSettings from "./RateLimitSettings"; import WatchProvidersSettings from "./WatchProvidersSettings"; import IntegrationsSettings from "./IntegrationsSettings"; @@ -66,6 +68,7 @@ const SETTINGS_GROUPS: SettingsNavGroup[] = [ { id: "scanner", label: "Scanner & Matcher", icon: ScanSearch, component: ScannerSettings }, { id: "intro", label: "Intro Markers", icon: Captions, component: IntroSettings }, { id: "subtitles", label: "Subtitles", icon: Subtitles, component: SubtitlesSettings }, + { id: "ai", label: "AI Services", icon: Sparkles, component: AIServicesSettings }, { id: "playback", label: "Playback", icon: PlayCircle, component: PlaybackSettings }, { id: "downloads", label: "Downloads", icon: Download, component: DownloadSettings }, ], diff --git a/web/src/pages/admin-settings/SubtitlesSettings.tsx b/web/src/pages/admin-settings/SubtitlesSettings.tsx index bfd7b805..6452765c 100644 --- a/web/src/pages/admin-settings/SubtitlesSettings.tsx +++ b/web/src/pages/admin-settings/SubtitlesSettings.tsx @@ -1,15 +1,9 @@ import { useState, useEffect } from "react"; -import { toast } from "sonner"; import { useSubtitleProviders, useUpdateSubtitleProvider, useTestSubtitleProvider, } from "@/hooks/queries/admin/subtitles"; -import { - useAdminSensitiveStatus, - useAdminServerSettings, - useUpdateServerSetting, -} from "@/hooks/queries/admin/settings"; import type { SubtitleProviderConfig } from "@/api/types"; import { Button } from "@/components/ui/button"; @@ -19,7 +13,6 @@ import { Input } from "@/components/ui/input"; import { Skeleton } from "@/components/ui/skeleton"; import { Eye, EyeOff } from "lucide-react"; import { CredentialStatus } from "./CredentialStatus"; -import { SettingField } from "./SettingField"; // ============================================================================ // Search providers @@ -252,138 +245,19 @@ function SearchProvidersContent() { ); } -// ============================================================================ -// AI translation -// ============================================================================ - -function AISubtitleTranslationCard() { - const { data: settings } = useAdminServerSettings(); - const { data: sensitive } = useAdminSensitiveStatus(); - const updateSetting = useUpdateServerSetting(); - - const apiKeyConfigured = new Set(sensitive?.configured ?? []).has("subtitle_ai.api_key"); - - const [enabled, setEnabled] = useState("false"); - const [baseUrl, setBaseUrl] = useState(""); - const [chatModel, setChatModel] = useState(""); - const [maxConcurrent, setMaxConcurrent] = useState("2"); - const [apiKey, setApiKey] = useState(""); - - // Hydrate the form from current server settings once loaded. - useEffect(() => { - if (!settings) return; - setEnabled(settings["subtitle_ai.enabled"] ?? "false"); - setBaseUrl(settings["subtitle_ai.base_url"] ?? "https://api.openai.com"); - setChatModel(settings["subtitle_ai.chat_model"] ?? "gpt-4o-mini"); - setMaxConcurrent(settings["subtitle_ai.max_concurrent_jobs"] ?? "2"); - }, [settings]); - - function save() { - const trimmedBaseUrl = baseUrl.trim(); - const trimmedChatModel = chatModel.trim(); - const parsedMaxConcurrent = Number.parseInt(maxConcurrent, 10); - - // Don't let an admin persist a config that would break translation for - // everyone (a blank endpoint/model when enabled, or a bad concurrency value). - if (enabled === "true" && (trimmedBaseUrl === "" || trimmedChatModel === "")) { - toast.error("Base URL and chat model are required to enable AI translation."); - return; - } - if (!Number.isInteger(parsedMaxConcurrent) || parsedMaxConcurrent < 1) { - toast.error("Max concurrent jobs must be a positive whole number."); - return; - } - - const updates = [ - updateSetting.mutateAsync({ key: "subtitle_ai.enabled", value: enabled }), - updateSetting.mutateAsync({ key: "subtitle_ai.base_url", value: trimmedBaseUrl }), - updateSetting.mutateAsync({ key: "subtitle_ai.chat_model", value: trimmedChatModel }), - updateSetting.mutateAsync({ - key: "subtitle_ai.max_concurrent_jobs", - value: String(parsedMaxConcurrent), - }), - ]; - if (apiKey.trim() !== "") { - updates.push(updateSetting.mutateAsync({ key: "subtitle_ai.api_key", value: apiKey })); - } - void Promise.all(updates).then(() => setApiKey("")); - } - - return ( -
-
-
-

AI Subtitle Translation

-

- On-demand subtitle translation via any OpenAI-compatible chat API (OpenAI, Groq, a local - Ollama server, …). Translated tracks are generated once on the server and served to - every client. -

-
- -
- - - - - -
- -

- Changes take effect after a server restart. -

-
-
- ); -} - export default function SubtitlesSettings() { return (

Subtitles

- Search providers for downloading subtitles and AI translation for generating new language - tracks. + Search providers for downloading subtitles. AI translation and transcription live under AI + Services.

-
); diff --git a/web/src/pages/settings/PlaybackSettings.tsx b/web/src/pages/settings/PlaybackSettings.tsx index 84adc48a..c4d2603c 100644 --- a/web/src/pages/settings/PlaybackSettings.tsx +++ b/web/src/pages/settings/PlaybackSettings.tsx @@ -114,6 +114,7 @@ export default function PlaybackSettings() { const saveProfileField = (body: { quality_preference?: string; language?: string; + preferred_metadata_language?: string; auto_skip_intro?: boolean; auto_skip_credits?: boolean; auto_skip_recap?: boolean; @@ -203,6 +204,39 @@ export default function PlaybackSettings() { )} /> + ( +
+ +
+ )} + /> + void; sessionId?: string; getSubtitleStartPosition?: () => number; + audioTracks?: PlayerAudioTrack[]; } const DELAY_STEP_MS = 100; @@ -50,11 +51,13 @@ export function SubtitleMenu({ onRefreshSubtitles, sessionId, getSubtitleStartPosition, + audioTracks, }: SubtitleMenuProps) { const [open, setOpen] = useState(false); const [searchOpen, setSearchOpen] = useState(false); const [translateOpen, setTranslateOpen] = useState(false); const [aiEnabled, setAiEnabled] = useState(false); + const [aiTranscribeEnabled, setAiTranscribeEnabled] = useState(false); const [appearanceOpen, setAppearanceOpen] = useState(false); const menuRef = useRef(null); @@ -67,12 +70,19 @@ export function SubtitleMenu({ useEffect(() => { if (!playerConfig) return; let cancelled = false; - playerFetch<{ enabled: boolean }>(playerConfig, "/subtitles/ai/status") + playerFetch<{ enabled: boolean; transcribe_enabled?: boolean }>( + playerConfig, + "/subtitles/ai/status", + ) .then((res) => { - if (!cancelled) setAiEnabled(Boolean(res?.enabled)); + if (cancelled) return; + setAiEnabled(Boolean(res?.enabled)); + setAiTranscribeEnabled(Boolean(res?.transcribe_enabled)); }) .catch(() => { - if (!cancelled) setAiEnabled(false); + if (cancelled) return; + setAiEnabled(false); + setAiTranscribeEnabled(false); }); return () => { cancelled = true; @@ -301,23 +311,26 @@ export function SubtitleMenu({ Search Online… )} - {aiEnabled && mediaFileId && playerConfig && tracks.some(isTranslatableSource) && ( - - )} + {mediaFileId && + playerConfig && + ((aiEnabled && tracks.some(isTranslatableSource)) || + (aiTranscribeEnabled && (audioTracks?.length ?? 0) > 0)) && ( + + )} + ))} +
+ )} + + {!canTranslate && !canTranscribe ? (

No text subtitle track is available to translate. Add or download one first.

) : ( <> - + {mode === "subtitles" ? ( + + ) : ( + + )}