feat(subtitles): on-demand AI subtitle translation with live streaming

Add server-side AI subtitle translation backed by any OpenAI-compatible
chat endpoint (OpenAI, Groq, a local Ollama/llama.cpp server). A viewer
picks a source track and target language in the player; the server runs a
bounded, resumable job pipeline that translates SRT/VTT cues in batches and
streams them back over the realtime websocket so playback pauses, fills in
cues near the playhead, and resumes. The finished track is persisted as an
ordinary downloaded subtitle, so it reaches every client through the
existing subtitle pipeline with no client changes.

- Job lifecycle persisted in subtitle_ai_jobs (migration 168): enqueue with
  idempotency, bounded concurrency, progress/heartbeat, cancellation, and
  crash recovery.
- New realtime events (subtitle_ready + subtitle_translation_*) with a
  per-session notifier; the player renders a synthetic "live" track fed by
  websocket cues. Timestamps never leave the server, so timing can't drift.
- Admin settings card for endpoint / model / concurrency.

Player + lifecycle hardening (from the code review of this feature):
- Hand off from the live track to the persisted track on completion
  (selected by downloaded-subtitle id) and on the subtitle_ready broadcast,
  so the saved track survives a reload and a mid-stream socket drop.
- Never persist the synthetic live-track sentinel index as a subtitle
  preference; restore the prior selection on failure; only auto-resume
  playback if the viewer was actually playing.
- Resume promptly when the playhead is past the last cue; rebuild the live
  track on a new job; O(batch) live-cue ingestion instead of O(n^2).

Reliability:
- Root translation jobs in the application context so shutdown cancels them.
- Heartbeat-based stale-job reaper (safe across multiple instances) replaces
  the table-wide startup reset.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Quick
2026-05-29 23:59:12 -04:00
co-authored by Claude Opus 4.8
parent 9123cfb1f4
commit e441d2d6e9
34 changed files with 3173 additions and 59 deletions
@@ -4,7 +4,11 @@ import {
useUpdateSubtitleProvider,
useTestSubtitleProvider,
} from "@/hooks/queries/admin/subtitles";
import { useAdminSensitiveStatus, useUpdateServerSetting } from "@/hooks/queries/admin/settings";
import {
useAdminSensitiveStatus,
useAdminServerSettings,
useUpdateServerSetting,
} from "@/hooks/queries/admin/settings";
import type { SubtitleProviderConfig } from "@/api/types";
import { Button } from "@/components/ui/button";
@@ -444,6 +448,99 @@ function IntroDBCredentialCard() {
);
}
function AISubtitleTranslationCard() {
const { data: settings } = useAdminServerSettings();
const { data: sensitive } = useAdminSensitiveStatus();
const updateSetting = useUpdateServerSetting();
const apiKeyConfigured = new Set(sensitive?.configured ?? []).has("subtitle_ai.api_key");
const [enabled, setEnabled] = useState("false");
const [baseUrl, setBaseUrl] = useState("");
const [chatModel, setChatModel] = useState("");
const [maxConcurrent, setMaxConcurrent] = useState("2");
const [apiKey, setApiKey] = useState("");
// Hydrate the form from current server settings once loaded.
useEffect(() => {
if (!settings) return;
setEnabled(settings["subtitle_ai.enabled"] ?? "false");
setBaseUrl(settings["subtitle_ai.base_url"] ?? "https://api.openai.com");
setChatModel(settings["subtitle_ai.chat_model"] ?? "gpt-4o-mini");
setMaxConcurrent(settings["subtitle_ai.max_concurrent_jobs"] ?? "2");
}, [settings]);
function save() {
const updates = [
updateSetting.mutateAsync({ key: "subtitle_ai.enabled", value: enabled }),
updateSetting.mutateAsync({ key: "subtitle_ai.base_url", value: baseUrl }),
updateSetting.mutateAsync({ key: "subtitle_ai.chat_model", value: chatModel }),
updateSetting.mutateAsync({ key: "subtitle_ai.max_concurrent_jobs", value: maxConcurrent }),
];
if (apiKey.trim() !== "") {
updates.push(updateSetting.mutateAsync({ key: "subtitle_ai.api_key", value: apiKey }));
}
void Promise.all(updates).then(() => setApiKey(""));
}
return (
<div className="border-border bg-surface max-w-2xl rounded-lg border px-5 py-4">
<div className="mb-2 flex items-center justify-between gap-3">
<div>
<h3 className="text-sm font-semibold">AI Subtitle Translation</h3>
<p className="text-muted-foreground text-xs">
On-demand subtitle translation via any OpenAI-compatible chat API (OpenAI, Groq, a local
Ollama server, …). Translated tracks are generated once on the server and served to
every client.
</p>
</div>
<SubtitleCredentialStatus configured={apiKeyConfigured} />
</div>
<SettingField
label="Enabled"
type="toggle"
value={enabled}
onChange={setEnabled}
hint="Show the “Translate with AI” action in the player."
/>
<SettingField
label="Base URL"
type="text"
value={baseUrl}
onChange={setBaseUrl}
hint="https://api.openai.com"
/>
<SettingField
label="Chat model"
type="text"
value={chatModel}
onChange={setChatModel}
hint="e.g. gpt-4o-mini, llama3.1"
/>
<SettingField
label="API Key"
type="password"
value={apiKey}
onChange={setApiKey}
sensitiveConfigured={apiKeyConfigured}
hint="Leave blank to keep current. Empty is fine for keyless local servers."
/>
<SettingField
label="Max concurrent jobs"
type="number"
value={maxConcurrent}
onChange={setMaxConcurrent}
hint="Caps simultaneous translations so they don't starve transcodes."
/>
<div className="pt-2">
<Button type="button" onClick={save} disabled={updateSetting.isPending}>
{updateSetting.isPending ? "Saving..." : "Save AI Translation Settings"}
</Button>
</div>
</div>
);
}
export default function IntegrationsSettings() {
return (
<div className="flex h-full flex-col">
@@ -461,6 +558,9 @@ export default function IntegrationsSettings() {
<div className="mb-8">
<IntroDBCredentialCard />
</div>
<div className="mb-8">
<AISubtitleTranslationCard />
</div>
<SubtitlesContent />
</div>
);
@@ -44,6 +44,8 @@ interface PlayerControlsProps {
mediaFileId?: number;
playerConfig?: PlayerConfig;
onRefreshSubtitles?: () => void;
sessionId?: string;
getSubtitleStartPosition?: () => number;
// Audio
audioTracks: PlayerAudioTrack[];
activeAudioIndex: number;
@@ -102,6 +104,8 @@ export function PlayerControls({
mediaFileId,
playerConfig,
onRefreshSubtitles,
sessionId,
getSubtitleStartPosition,
audioTracks,
activeAudioIndex,
onAudioSelect,
@@ -304,6 +308,8 @@ export function PlayerControls({
mediaFileId={mediaFileId}
playerConfig={playerConfig}
onRefreshSubtitles={onRefreshSubtitles}
sessionId={sessionId}
getSubtitleStartPosition={getSubtitleStartPosition}
/>
<QualityMenu
+58 -2
View File
@@ -1,10 +1,12 @@
import { useState, useCallback, useEffect, useRef, useMemo } from "react";
import { createPortal } from "react-dom";
import { Captions, CaptionsOff, Minus, Plus, SlidersHorizontal } from "lucide-react";
import { Captions, CaptionsOff, Languages, Minus, Plus, SlidersHorizontal } from "lucide-react";
import type { PlayerSubtitleInfo } from "../types";
import type { PlayerConfig } from "../context/PlayerConfigContext";
import { SubtitleSearchModal } from "./SubtitleSearchModal";
import { SubtitleTranslateModal } from "./SubtitleTranslateModal";
import { SubtitleAppearancePanel } from "./SubtitleAppearancePanel";
import { playerFetch } from "../player-fetch";
import { getLanguageName } from "../utils/languageNames";
import { sortSubtitlesBySource } from "../utils/subtitleSort";
import { getSubtitleFormatLabel } from "../utils/assSubtitles";
@@ -18,6 +20,8 @@ interface SubtitleMenuProps {
mediaFileId?: number;
playerConfig?: PlayerConfig;
onRefreshSubtitles?: () => void;
sessionId?: string;
getSubtitleStartPosition?: () => number;
}
const DELAY_STEP_MS = 100;
@@ -44,14 +48,37 @@ export function SubtitleMenu({
mediaFileId,
playerConfig,
onRefreshSubtitles,
sessionId,
getSubtitleStartPosition,
}: SubtitleMenuProps) {
const [open, setOpen] = useState(false);
const [searchOpen, setSearchOpen] = useState(false);
const [translateOpen, setTranslateOpen] = useState(false);
const [aiEnabled, setAiEnabled] = useState(false);
const [appearanceOpen, setAppearanceOpen] = useState(false);
const menuRef = useRef<HTMLDivElement>(null);
const sortedTracks = useMemo(() => sortSubtitlesBySource(tracks), [tracks]);
// Discover whether the server has AI subtitle translation configured, so we
// only surface the entry point when it can actually do something. This is a
// server-wide capability, so we fetch it once per session (keyed on the stable
// playerConfig) rather than re-checking on every file change.
useEffect(() => {
if (!playerConfig) return;
let cancelled = false;
playerFetch<{ enabled: boolean }>(playerConfig, "/subtitles/ai/status")
.then((res) => {
if (!cancelled) setAiEnabled(Boolean(res?.enabled));
})
.catch(() => {
if (!cancelled) setAiEnabled(false);
});
return () => {
cancelled = true;
};
}, [playerConfig]);
const clampedDelay = useCallback(
(ms: number) => Math.max(-DELAY_MAX_MS, Math.min(DELAY_MAX_MS, ms)),
[],
@@ -274,9 +301,26 @@ export function SubtitleMenu({
Search Online…
</button>
)}
{aiEnabled && mediaFileId && playerConfig && tracks.length > 0 && (
<button
ref={(el) => {
menuItemsRef.current[menuItemIndex + 2] = el;
}}
role="menuitem"
type="button"
className="flex w-full items-center gap-2 px-3 py-2 text-left text-sm text-white/70 hover:bg-white/10 focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none"
onClick={() => {
setTranslateOpen(true);
setOpen(false);
}}
>
<Languages className="h-3.5 w-3.5 text-white/50" />
Translate with AI…
</button>
)}
<button
ref={(el) => {
menuItemsRef.current[menuItemIndex + 2] = el;
menuItemsRef.current[menuItemIndex + 3] = el;
}}
role="menuitem"
type="button"
@@ -311,6 +355,18 @@ export function SubtitleMenu({
/>,
document.body,
)}
{translateOpen && mediaFileId && playerConfig && (
<SubtitleTranslateModal
mediaFileId={mediaFileId}
playerConfig={playerConfig}
tracks={tracks}
isOpen={translateOpen}
sessionId={sessionId}
getStartPosition={getSubtitleStartPosition}
onClose={() => setTranslateOpen(false)}
/>
)}
</div>
);
}
@@ -0,0 +1,188 @@
import { useState, useEffect, useCallback, useMemo } from "react";
import { createPortal } from "react-dom";
import type { PlayerConfig } from "../context/PlayerConfigContext";
import type { PlayerSubtitleInfo } from "../types";
import { playerFetch } from "../player-fetch";
import { LANGUAGES, getLanguageName } from "../utils/languageNames";
interface SubtitleTranslateModalProps {
mediaFileId: number;
playerConfig: PlayerConfig;
tracks: PlayerSubtitleInfo[];
isOpen: boolean;
sessionId?: string;
getStartPosition?: () => number;
onClose: () => void;
}
function sourceLabel(track: PlayerSubtitleInfo): string {
const lang = getLanguageName(track.language) || track.language || "Unknown";
const origin = track.source ? ` · ${track.source}` : "";
return `${lang}${origin}`;
}
export function SubtitleTranslateModal({
mediaFileId,
playerConfig,
tracks,
isOpen,
sessionId,
getStartPosition,
onClose,
}: SubtitleTranslateModalProps) {
// Live (in-progress) tracks can't be a translation source.
const sourceTracks = useMemo(() => tracks.filter((t) => !t.live), [tracks]);
const [sourceIndex, setSourceIndex] = useState<number | null>(null);
const [targetLang, setTargetLang] = useState("en");
const [submitting, setSubmitting] = useState(false);
const [error, setError] = useState<string | null>(null);
const effectiveSourceIndex = sourceIndex ?? sourceTracks[0]?.index ?? null;
useEffect(() => {
if (!isOpen) return;
const onKey = (e: KeyboardEvent) => {
if (e.key === "Escape") onClose();
};
document.addEventListener("keydown", onKey);
return () => document.removeEventListener("keydown", onKey);
}, [isOpen, onClose]);
const handleTranslate = useCallback(async () => {
if (effectiveSourceIndex === null) return;
const source = sourceTracks.find((t) => t.index === effectiveSourceIndex);
setSubmitting(true);
setError(null);
try {
await playerFetch(playerConfig, "/subtitles/ai/translate", {
method: "POST",
body: JSON.stringify({
media_file_id: mediaFileId,
source_index: effectiveSourceIndex,
source_language: source?.language ?? "",
target_language: targetLang,
session_id: sessionId ?? "",
start_position: getStartPosition?.() ?? 0,
}),
});
// The player takes over from here: it pauses, streams cues in as they're
// translated, then resumes once your position is covered.
onClose();
} catch (err) {
setError(err instanceof Error ? err.message : "Couldn't start translation.");
} finally {
setSubmitting(false);
}
}, [
effectiveSourceIndex,
sourceTracks,
mediaFileId,
targetLang,
sessionId,
getStartPosition,
playerConfig,
onClose,
]);
if (!isOpen) return null;
const modal = (
<div
className="fixed inset-0 z-50 flex items-center justify-center bg-black/80"
onClick={onClose}
role="dialog"
aria-modal="true"
aria-label="Translate subtitles with AI"
>
<div
className="w-full max-w-[440px] rounded-lg bg-neutral-900 text-white shadow-xl"
onClick={(e) => e.stopPropagation()}
>
<div className="flex items-center justify-between border-b border-white/10 px-4 py-3">
<h2 className="text-sm font-semibold">Translate subtitles with AI</h2>
<button
type="button"
className="rounded text-white/60 hover:text-white focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none"
onClick={onClose}
aria-label="Close"
>
✕
</button>
</div>
<div className="space-y-3 px-4 py-4">
{sourceTracks.length === 0 ? (
<p className="py-4 text-center text-xs text-white/50">
No text subtitle track is available to translate. Add or download one first.
</p>
) : (
<>
<label className="block">
<span className="mb-1 block text-xs font-medium text-white/60">Translate from</span>
<select
className="w-full rounded bg-neutral-800 px-2 py-1.5 text-sm text-white focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none disabled:opacity-50"
value={effectiveSourceIndex ?? ""}
onChange={(e) => setSourceIndex(Number(e.target.value))}
disabled={submitting}
>
{sourceTracks.map((track) => (
<option key={track.index} value={track.index}>
{sourceLabel(track)}
</option>
))}
</select>
</label>
<label className="block">
<span className="mb-1 block text-xs font-medium text-white/60">Translate to</span>
<select
className="w-full rounded bg-neutral-800 px-2 py-1.5 text-sm text-white focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none disabled:opacity-50"
value={targetLang}
onChange={(e) => setTargetLang(e.target.value)}
disabled={submitting}
>
{LANGUAGES.map((lang) => (
<option key={lang.code} value={lang.code}>
{lang.label}
</option>
))}
</select>
</label>
{error && (
<div role="alert" className="rounded bg-red-900/40 px-3 py-2 text-xs text-red-300">
{error}
</div>
)}
<div className="flex justify-end gap-2 pt-1">
<button
type="button"
className="rounded px-3 py-1.5 text-sm text-white/60 hover:bg-white/10 focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none"
onClick={onClose}
>
Cancel
</button>
<button
type="button"
className="rounded bg-white/10 px-3 py-1.5 text-sm font-medium hover:bg-white/20 focus-visible:ring-2 focus-visible:ring-white/70 focus-visible:outline-none disabled:opacity-50"
onClick={handleTranslate}
disabled={submitting || effectiveSourceIndex === null}
>
{submitting ? "Starting…" : "Translate"}
</button>
</div>
<p className="text-[11px] leading-relaxed text-white/35">
Playback pauses while the first lines are translated, then resumes with subtitles
streaming in. The finished track is saved for everyone.
</p>
</>
)}
</div>
</div>
</div>
);
return createPortal(modal, document.body);
}
+231 -8
View File
@@ -1,4 +1,5 @@
import { useCallback, useEffect, useRef, useState } from "react";
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import type { ParsedCue } from "../utils/parseVTT";
import { resolveSubtitleAutoSelect } from "../utils/subtitleSort";
import type HlsType from "hls.js";
import { PlayerControls } from "./PlayerControls";
@@ -50,6 +51,13 @@ import { toMediaTime, toPlayerTime } from "../utils/mediaTimeline";
import { buildWatchTogetherInviteUrl } from "@/lib/watchTogether";
import { toast } from "sonner";
// Reserved index for the in-progress live AI translation track. Sits well above
// any real subtitle index so it never collides.
const LIVE_SUBTITLE_INDEX = 1_000_000;
// Resume playback once translated cues cover at least this far ahead of the
// playhead; a hard cap also resumes so we never wait forever.
const TRANSLATION_RESUME_TIMEOUT_MS = 30_000;
interface VideoPlayerProps {
title: string;
year?: number;
@@ -244,6 +252,57 @@ export function VideoPlayer({
setSubtitleDelayMs(0);
}, [activeFileId]);
// -- Live AI subtitle translation (streamed over the realtime websocket) --
// While a translation runs, a synthetic "live" track is added to the list and
// selected; cues arrive over the websocket and the player pauses until the
// region near the playhead is covered, then resumes.
const [liveTranslation, setLiveTranslation] = useState<{
trackKey: string;
language: string;
label: string;
} | null>(null);
const [liveCues, setLiveCues] = useState<ParsedCue[]>([]);
const [translationBuffering, setTranslationBuffering] = useState(false);
const translationPauseRef = useRef(false);
const translationResumeTimerRef = useRef<number | null>(null);
// Whether playback should auto-resume once buffering ends. Captured at
// translation start: if the viewer had deliberately paused, we don't yank
// them back into playback.
const translationResumeOnFinishRef = useRef(false);
// The subtitle selection active before a translation hijacked it, so a failed
// translation can restore it instead of leaving subtitles off.
const preTranslationSubtitleIndexRef = useRef<number | null>(null);
// The persisted downloaded-subtitle id to switch to once a completed
// translation's track lands in the refreshed list.
const pendingTranslatedSubtitleIdRef = useRef<number | null>(null);
// Drop any live translation when the media changes so a stale track from the
// previous file never lingers.
useEffect(() => {
setLiveTranslation(null);
setLiveCues([]);
setTranslationBuffering(false);
translationPauseRef.current = false;
pendingTranslatedSubtitleIdRef.current = null;
}, [activeFileId]);
// Merge the live track into the track list the player + menu see.
const effectiveSubtitleTracks = useMemo(() => {
if (!liveTranslation) return subtitleUrls;
return [
...subtitleUrls,
{
index: LIVE_SUBTITLE_INDEX,
language: liveTranslation.language,
label: liveTranslation.label || "AI translation",
source: "downloaded" as const,
codec: "srt",
url: "",
live: true,
},
];
}, [subtitleUrls, liveTranslation]);
// -- Transcode quality switching --
// Remux also uses HLS (codec copy) via the transcode pipeline.
const transcodeQuality = useTranscodeQuality({
@@ -739,11 +798,161 @@ export function VideoPlayer({
(index: number | null) => {
subtitleSelectionWasManualRef.current = true;
setActiveSubtitleIndex(index);
// The in-progress live translation track is synthetic (a sentinel index
// that exists only in memory); never persist it as the saved preference or
// we'd store a nonexistent track and lose the real selection.
if (index === LIVE_SUBTITLE_INDEX) return;
onSubtitleChanged?.(index);
},
[onSubtitleChanged],
);
// The media-time playhead, sent with a translate request so the server starts
// where the viewer is watching.
const getSubtitleStartPosition = useCallback(
() => toMediaTime(videoRef.current?.currentTime ?? 0, streamOriginRef.current ?? 0),
[],
);
const resumeFromTranslationPause = useCallback(() => {
if (translationResumeTimerRef.current !== null) {
window.clearTimeout(translationResumeTimerRef.current);
translationResumeTimerRef.current = null;
}
if (translationPauseRef.current) {
translationPauseRef.current = false;
// Only resume if the viewer was playing when the translation began; if
// they had paused on purpose, leave them paused.
if (translationResumeOnFinishRef.current) {
void videoRef.current?.play().catch(() => {});
}
}
setTranslationBuffering(false);
}, []);
// Intercept live-translation events; forward everything else to the parent.
const handleRealtimeEvent = useCallback(
(event: PlaybackRealtimeEventEnvelope) => {
switch (event.name) {
case "subtitle_ready": {
// Broadcast to every viewer of the file when a generated track is
// persisted. Refresh the list so it appears (the requesting session
// also auto-selects it via the completed handler below).
if (event.payload.file_id === activeFileId) {
onRefreshSubtitles?.();
}
break;
}
case "subtitle_translation_started": {
// Remember the real selection we're displacing and whether we were
// playing, so completion/failure can restore the right state.
const wasPlaying = !(videoRef.current?.paused ?? true);
translationResumeOnFinishRef.current = wasPlaying;
setActiveSubtitleIndex((idx) => {
if (idx !== LIVE_SUBTITLE_INDEX) {
preTranslationSubtitleIndexRef.current = idx;
}
return LIVE_SUBTITLE_INDEX;
});
pendingTranslatedSubtitleIdRef.current = null;
setLiveCues([]);
setLiveTranslation({
trackKey: event.payload.track_key,
language: event.payload.language,
label: event.payload.label ?? "",
});
subtitleSelectionWasManualRef.current = true;
translationPauseRef.current = true;
setTranslationBuffering(true);
// Only pause if the viewer was playing; don't disturb a deliberate pause.
if (wasPlaying) videoRef.current?.pause();
if (translationResumeTimerRef.current !== null) {
window.clearTimeout(translationResumeTimerRef.current);
}
translationResumeTimerRef.current = window.setTimeout(
resumeFromTranslationPause,
TRANSLATION_RESUME_TIMEOUT_MS,
);
break;
}
case "subtitle_translation_cues": {
const cues = event.payload.cues.map((c) => ({
start: c.start,
end: c.end,
text: c.text,
}));
setLiveCues((prev) => [...prev, ...cues]);
break;
}
case "subtitle_translation_completed": {
resumeFromTranslationPause();
// Hand off from the ephemeral live track to the persisted downloaded
// track: refresh the list and let the effect below select it by id
// once it lands. Without a refresh callback we keep the live track
// (which already holds the full cue set) as a best-effort fallback.
if (onRefreshSubtitles) {
pendingTranslatedSubtitleIdRef.current = event.payload.subtitle_id;
onRefreshSubtitles();
}
break;
}
case "subtitle_translation_failed": {
resumeFromTranslationPause();
setLiveTranslation(null);
setLiveCues([]);
pendingTranslatedSubtitleIdRef.current = null;
// Restore the selection the translation displaced rather than leaving
// subtitles off.
const restore = preTranslationSubtitleIndexRef.current;
setActiveSubtitleIndex((idx) => (idx === LIVE_SUBTITLE_INDEX ? restore : idx));
toast.error(
event.payload.message
? `Translation failed: ${event.payload.message}`
: "Subtitle translation failed",
);
break;
}
default:
onRealtimeEvent?.(event);
}
},
[onRealtimeEvent, onRefreshSubtitles, activeFileId, resumeFromTranslationPause],
);
// Once a completed translation's persisted track lands in the refreshed list,
// switch to it (selecting by downloaded-subtitle id) and drop the live track,
// so the viewer ends up on the real saved subtitle rather than the synthetic
// one that would vanish on reload.
useEffect(() => {
const pendingId = pendingTranslatedSubtitleIdRef.current;
if (pendingId == null) return;
const match = subtitleUrls.find((t) => t.id === pendingId);
if (!match) return;
pendingTranslatedSubtitleIdRef.current = null;
setLiveTranslation(null);
setLiveCues([]);
handleSubtitleSelect(match.index);
}, [subtitleUrls, handleSubtitleSelect]);
// Resume as soon as the first translated cues arrive. Playhead-first
// translation means the cues covering the current position are delivered
// first, so the first batch is enough; and when the playhead is past the last
// cue (e.g. end credits) there is nothing at the playhead to wait for, so we
// still resume here rather than stalling until the 30s timeout.
useEffect(() => {
if (!translationPauseRef.current || liveCues.length === 0) return;
resumeFromTranslationPause();
}, [liveCues, resumeFromTranslationPause]);
useEffect(
() => () => {
if (translationResumeTimerRef.current !== null) {
window.clearTimeout(translationResumeTimerRef.current);
}
},
[],
);
// -- PiP toggle --
const handleTogglePiP = useCallback(async () => {
const video = videoRef.current;
@@ -1335,10 +1544,12 @@ export function VideoPlayer({
// (which has browser bugs with stale cues persisting after seek).
const activeCueTexts = useSubtitleTracks(
videoRef,
subtitleUrls,
effectiveSubtitleTracks,
activeSubtitleIndex,
streamOriginRef,
subtitleDelayMs,
liveCues,
liveTranslation?.trackKey ?? null,
);
// -- ASS/SSA subtitle rendering via JASSUB (client-side libass) --
@@ -1356,14 +1567,14 @@ export function VideoPlayer({
if (subtitleSelectionWasManualRef.current) {
const selectionStillExists =
activeSubtitleIndex === null ||
subtitleUrls.some((track) => track.index === activeSubtitleIndex);
effectiveSubtitleTracks.some((track) => track.index === activeSubtitleIndex);
if (selectionStillExists) {
return;
}
subtitleSelectionWasManualRef.current = false;
}
if (subtitleUrls.length === 0) {
if (effectiveSubtitleTracks.length === 0) {
setActiveSubtitleIndex(null);
lastSubtitleIndexRef.current = null;
return;
@@ -1376,7 +1587,7 @@ export function VideoPlayer({
const match = resolveSubtitleAutoSelect({
mode: effectiveMode,
tracks: subtitleUrls,
tracks: effectiveSubtitleTracks,
preferredLanguage: preferredSubtitleLanguage ?? null,
preferredTrackSignature: preferredSubtitleTrackSignature ?? null,
audioLanguage: audioLang,
@@ -1395,7 +1606,7 @@ export function VideoPlayer({
activeSubtitleIndex,
preferredSubtitleLanguage,
preferredSubtitleTrackSignature,
subtitleUrls,
effectiveSubtitleTracks,
subtitleMode,
showForcedSubtitles,
profileLanguage,
@@ -1719,7 +1930,7 @@ export function VideoPlayer({
const realtime = usePlaybackRealtime({
sessionId,
onCommand: executeRealtimeCommand,
onEvent: onRealtimeEvent,
onEvent: handleRealtimeEvent,
});
useEffect(() => {
@@ -2044,6 +2255,16 @@ export function VideoPlayer({
/>
)}
{/* Live translation buffering indicator */}
{translationBuffering && (
<div className="pointer-events-none absolute inset-0 z-30 flex items-center justify-center">
<div className="flex items-center gap-3 rounded-lg bg-black/80 px-4 py-3 text-sm text-white shadow-lg">
<span className="h-4 w-4 animate-spin rounded-full border-2 border-white/30 border-t-white" />
Preparing {liveTranslation?.label || "translated"} subtitles…
</div>
</div>
)}
{/* Controls */}
{!isDetached && isPlayerReady && (
<PlayerControls
@@ -2058,7 +2279,7 @@ export function VideoPlayer({
volume={volume}
muted={muted}
isFullscreen={isFullscreen}
subtitleTracks={subtitleUrls}
subtitleTracks={effectiveSubtitleTracks}
activeSubtitleIndex={activeSubtitleIndex}
onSubtitleSelect={handleSubtitleSelect}
subtitleDelayMs={subtitleDelayMs}
@@ -2066,6 +2287,8 @@ export function VideoPlayer({
mediaFileId={activeFileId ?? undefined}
playerConfig={playerConfig}
onRefreshSubtitles={onRefreshSubtitles}
sessionId={sessionId}
getSubtitleStartPosition={getSubtitleStartPosition}
audioTracks={audioTracks}
activeAudioIndex={activeAudioIndex}
onAudioSelect={onAudioSelect}
+4
View File
@@ -172,6 +172,10 @@ export function WatchPage({
if (!seriesId) return;
const track = index !== null ? playableSubtitles.find((s) => s.index === index) : null;
// Never persist an index we can't resolve to a real track (e.g. the
// in-progress AI live track's sentinel index): it would store a
// nonexistent track with empty language and clobber the saved preference.
if (index !== null && !track) return;
const trackSignature: PlayerSubtitleTrackSignature | null = track
? {
source: track.source,
@@ -513,6 +513,7 @@ export function usePlaybackSession(
const token = config.getAccessToken();
const newTracks: PlayerSubtitleInfo[] = downloaded.map((dl, i) => ({
index: baseIndex + i,
id: dl.id,
language: dl.language,
codec: dl.format,
label: `${dl.release_name} (${dl.provider})`,
+91 -26
View File
@@ -23,6 +23,29 @@ function stripVTTTags(text: string): string {
return text.replace(/<[^>]+>/g, "");
}
/**
* Add parsed cues to a TextTrack, applying the stream origin and user delay and
* deduping against `seen`. Shared by the URL fetcher and the live-cue path.
*/
function addCuesToTrack(
track: TextTrack,
cues: ParsedCue[],
origin: number,
delaySec: number,
seen: Set<string>,
): void {
for (const parsed of cues) {
if (parsed.end <= parsed.start) continue;
const startTime = Math.max(0, parsed.start - origin + delaySec);
const endTime = parsed.end - origin + delaySec;
if (endTime <= 0) continue;
const key = `${startTime}|${endTime}|${parsed.text}`;
if (seen.has(key)) continue;
seen.add(key);
track.addCue(new VTTCue(startTime, endTime, parsed.text));
}
}
/** Append or replace the `position` query param on a subtitle URL. */
function appendPosition(url: string, position: number): string {
const sep = url.includes("?") ? "&" : "?";
@@ -55,6 +78,10 @@ export function useSubtitleTracks(
activeSubtitleIndex: number | null,
streamOriginRef: React.RefObject<number>,
subtitleDelayMs: number,
liveCues?: ParsedCue[] | null,
// Identifies the current live translation job. Changing it (a new job) rebuilds
// the track and resets the dedup set so cues from a prior run never linger.
liveTrackKey?: string | null,
): string[] {
const [activeCueTexts, setActiveCueTexts] = useState<string[]>([]);
@@ -65,12 +92,22 @@ export function useSubtitleTracks(
const activeUrl = activeSub?.url ?? null;
const activeCodec = activeSub?.codec;
const activeLang = activeSub?.language ?? "";
// A live track's cues arrive over the websocket (liveCues) instead of from a
// URL; the main effect builds the track but skips the sliding-window fetcher.
const activeIsLive = activeSub?.live === true;
// Track which delay is currently baked into the VTTCues on the active track,
// so the delay-update effect below can compute the exact shift to apply.
// Cue-add paths also read this to keep new cues aligned with existing ones.
const appliedDelayMsRef = useRef(0);
const trackRef = useRef<TextTrack | null>(null);
// Cue dedup set, held in a ref so the live-cue effect and the URL fetcher
// share it. Reset whenever a fresh track is built.
const seenCueKeysRef = useRef<Set<string>>(new Set());
// How many of `liveCues` have already been pushed onto the live track. Lets the
// live-cue effect add only the new tail each batch instead of rescanning the
// whole (growing) array. Reset on every track rebuild.
const processedLiveCuesRef = useRef(0);
useEffect(() => {
const video = videoRef.current;
@@ -80,7 +117,11 @@ export function useSubtitleTracks(
setActiveCueTexts([]);
// Skip entirely for ASS/SSA: JASSUB handles those via useASSSubtitles.
if (!activeUrl || isASSCodec(activeCodec)) {
if (isASSCodec(activeCodec)) {
return;
}
// Need either a URL to stream from or a live cue source.
if (!activeUrl && !activeIsLive) {
return;
}
@@ -91,6 +132,8 @@ export function useSubtitleTracks(
const track = videoEl.addTextTrack("subtitles", "Silo", activeLang || undefined);
track.mode = "hidden";
trackRef.current = track;
seenCueKeysRef.current = new Set();
processedLiveCuesRef.current = 0;
let cancelled = false;
let hasFetched = false;
@@ -101,10 +144,6 @@ export function useSubtitleTracks(
let atEOF = false;
let inflight: AbortController | null = null;
// Persistent dedup: cues from overlapping windows key to the same string.
// Cleared alongside track cues on backward-seek resets.
const seenCueKeys = new Set<string>();
function handleCueChange() {
const active = track.activeCues;
if (!active || active.length === 0) {
@@ -124,7 +163,7 @@ export function useSubtitleTracks(
for (const cue of Array.from(cues)) {
track.removeCue(cue);
}
seenCueKeys.clear();
seenCueKeysRef.current.clear();
}
function addParsedCues(newCues: ParsedCue[]) {
@@ -136,17 +175,7 @@ export function useSubtitleTracks(
// baked in here so new cues line up with existing ones.
const origin = streamOriginRef.current ?? 0;
const delaySec = appliedDelayMsRef.current / 1000;
for (const parsed of newCues) {
if (parsed.end <= parsed.start) continue;
const startTime = Math.max(0, parsed.start - origin + delaySec);
const endTime = parsed.end - origin + delaySec;
if (endTime <= 0) continue;
const key = `${startTime}|${endTime}|${parsed.text}`;
if (seenCueKeys.has(key)) continue;
seenCueKeys.add(key);
track.addCue(new VTTCue(startTime, endTime, parsed.text));
}
addCuesToTrack(track, newCues, origin, delaySec, seenCueKeysRef.current);
}
async function fetchWindow(seekStart: number, resetExisting: boolean) {
@@ -251,15 +280,19 @@ export function useSubtitleTracks(
}
}
// Kick off the first window before any player event fires so cues
// are already in flight for the current position.
maybeFetch();
// URL-backed tracks run the sliding-window fetcher; live tracks receive
// their cues from the liveCues effect below instead.
if (!activeIsLive) {
// Kick off the first window before any player event fires so cues
// are already in flight for the current position.
maybeFetch();
// Cue activation is driven by the browser via `cuechange`; these
// listeners exist only to keep the sliding-window fetcher scheduled.
videoEl.addEventListener("timeupdate", maybeFetch);
videoEl.addEventListener("seeking", maybeFetch);
videoEl.addEventListener("seeked", maybeFetch);
// Cue activation is driven by the browser via `cuechange`; these
// listeners exist only to keep the sliding-window fetcher scheduled.
videoEl.addEventListener("timeupdate", maybeFetch);
videoEl.addEventListener("seeking", maybeFetch);
videoEl.addEventListener("seeked", maybeFetch);
}
return () => {
cancelled = true;
@@ -281,7 +314,7 @@ export function useSubtitleTracks(
// `subtitleDelayMs` is intentionally excluded — nudging delay must not
// tear down and refetch the track. The delay-update effect below shifts
// existing cues in place instead.
}, [activeUrl, activeCodec, activeLang, streamOriginRef, videoRef]);
}, [activeUrl, activeCodec, activeLang, activeIsLive, liveTrackKey, streamOriginRef, videoRef]);
// Apply delay changes to already-loaded cues without rebuilding the track.
// Runs after the main effect, so trackRef is current.
@@ -303,5 +336,37 @@ export function useSubtitleTracks(
}
}, [subtitleDelayMs]);
// Feed websocket-pushed cues into the active live track as they arrive. Only
// the new tail since the last push is added (liveCues is append-only within a
// job), so ingestion stays O(batch) rather than O(total) per push. When the
// job restarts liveCues is replaced with a shorter array, which the length
// check below detects to start over (the track itself is rebuilt via
// liveTrackKey, so a fresh seen-set and pointer are already in place).
useEffect(() => {
if (!activeIsLive) return;
const track = trackRef.current;
if (!track || !liveCues) return;
if (liveCues.length < processedLiveCuesRef.current) {
processedLiveCuesRef.current = 0;
}
const fresh = liveCues.slice(processedLiveCuesRef.current);
if (fresh.length === 0) return;
processedLiveCuesRef.current = liveCues.length;
const origin = streamOriginRef.current ?? 0;
const delaySec = appliedDelayMsRef.current / 1000;
addCuesToTrack(track, fresh, origin, delaySec, seenCueKeysRef.current);
// While paused, adding a cue over the playhead doesn't reliably fire
// `cuechange`, so refresh the on-screen text by hand. While playing the
// browser drives `cuechange`, so skip the redundant state update.
if (videoRef.current?.paused) {
const active = track.activeCues;
setActiveCueTexts(
active && active.length > 0
? Array.from(active).map((c) => stripVTTTags((c as VTTCue).text))
: [],
);
}
}, [liveCues, activeIsLive, activeSubtitleIndex, liveTrackKey, streamOriginRef, videoRef]);
return activeCueTexts;
}
+43
View File
@@ -104,6 +104,49 @@ describe("realtime protocol", () => {
});
});
it("parses subtitle ready events", () => {
const event = parsePlaybackRealtimeMessage(
JSON.stringify({
type: "event",
session_id: "session-1",
name: "subtitle_ready",
payload: {
session_id: "session-1",
file_id: 42,
subtitle_id: 7,
language: "es",
label: "English → Spanish (AI)",
},
}),
);
expect(event).toEqual({
type: "event",
session_id: "session-1",
name: "subtitle_ready",
payload: {
session_id: "session-1",
file_id: 42,
subtitle_id: 7,
language: "es",
label: "English → Spanish (AI)",
},
});
});
it("rejects subtitle ready events missing the subtitle id", () => {
const event = parsePlaybackRealtimeMessage(
JSON.stringify({
type: "event",
session_id: "session-1",
name: "subtitle_ready",
payload: { session_id: "session-1", file_id: 42, language: "es" },
}),
);
expect(event).toBeNull();
});
it("builds hello, ack, and result envelopes", () => {
expect(buildPlaybackRealtimeHello("session-1")).toEqual({
type: "hello",
+208 -1
View File
@@ -17,7 +17,14 @@ export type PlaybackCommandName =
export type PlaybackRealtimeAckStatus = "accepted";
export type PlaybackRealtimeResultStatus = "completed" | "rejected";
export type PlaybackRealtimeEventName = "chapter_thumbnail_ready" | "markers_updated";
export type PlaybackRealtimeEventName =
| "chapter_thumbnail_ready"
| "markers_updated"
| "subtitle_ready"
| "subtitle_translation_started"
| "subtitle_translation_cues"
| "subtitle_translation_completed"
| "subtitle_translation_failed";
export interface PlaybackRealtimeCommandEnvelope {
type: "command";
@@ -66,6 +73,64 @@ export interface PlaybackMarkersUpdatedPayload {
preview?: PlaybackTimeRangePayload | null;
}
/**
* Broadcast to every session watching a file when a newly generated subtitle
* track (AI translation, later ASR) has been persisted, so players can refresh
* their track list and pick it up without a manual reload.
*/
export interface PlaybackSubtitleReadyPayload {
session_id: string;
file_id: number;
subtitle_id: number;
language: string;
label?: string;
}
/** One translated subtitle cue pushed during a live translation (media seconds). */
export interface PlaybackStreamCue {
start: number;
end: number;
text: string;
}
export interface PlaybackSubtitleTranslationStartedPayload {
session_id: string;
file_id: number;
job_id: number;
track_key: string;
language: string;
label?: string;
total_cues: number;
}
export interface PlaybackSubtitleTranslationCuesPayload {
session_id: string;
file_id: number;
job_id: number;
track_key: string;
cues: PlaybackStreamCue[];
done: number;
total: number;
}
export interface PlaybackSubtitleTranslationCompletedPayload {
session_id: string;
file_id: number;
job_id: number;
track_key: string;
subtitle_id: number;
language: string;
label?: string;
}
export interface PlaybackSubtitleTranslationFailedPayload {
session_id: string;
file_id: number;
job_id: number;
track_key: string;
message?: string;
}
export interface PlaybackRealtimeEventEnvelopeBase {
type: "event";
session_id: string;
@@ -79,6 +144,26 @@ export type PlaybackRealtimeEventEnvelope =
| (PlaybackRealtimeEventEnvelopeBase & {
name: "markers_updated";
payload: PlaybackMarkersUpdatedPayload;
})
| (PlaybackRealtimeEventEnvelopeBase & {
name: "subtitle_ready";
payload: PlaybackSubtitleReadyPayload;
})
| (PlaybackRealtimeEventEnvelopeBase & {
name: "subtitle_translation_started";
payload: PlaybackSubtitleTranslationStartedPayload;
})
| (PlaybackRealtimeEventEnvelopeBase & {
name: "subtitle_translation_cues";
payload: PlaybackSubtitleTranslationCuesPayload;
})
| (PlaybackRealtimeEventEnvelopeBase & {
name: "subtitle_translation_completed";
payload: PlaybackSubtitleTranslationCompletedPayload;
})
| (PlaybackRealtimeEventEnvelopeBase & {
name: "subtitle_translation_failed";
payload: PlaybackSubtitleTranslationFailedPayload;
});
export interface PlaybackRealtimeAckEnvelope {
@@ -164,6 +249,79 @@ function isMarkersUpdatedPayload(value: unknown): value is PlaybackMarkersUpdate
);
}
function isSubtitleReadyPayload(value: unknown): value is PlaybackSubtitleReadyPayload {
return (
isRecord(value) &&
typeof value.session_id === "string" &&
typeof value.file_id === "number" &&
typeof value.subtitle_id === "number" &&
typeof value.language === "string"
);
}
function isStreamCue(value: unknown): value is PlaybackStreamCue {
return (
isRecord(value) &&
typeof value.start === "number" &&
typeof value.end === "number" &&
typeof value.text === "string"
);
}
function isTranslationStartedPayload(
value: unknown,
): value is PlaybackSubtitleTranslationStartedPayload {
return (
isRecord(value) &&
typeof value.session_id === "string" &&
typeof value.file_id === "number" &&
typeof value.job_id === "number" &&
typeof value.track_key === "string" &&
typeof value.language === "string" &&
typeof value.total_cues === "number"
);
}
function isTranslationCuesPayload(value: unknown): value is PlaybackSubtitleTranslationCuesPayload {
return (
isRecord(value) &&
typeof value.session_id === "string" &&
typeof value.file_id === "number" &&
typeof value.job_id === "number" &&
typeof value.track_key === "string" &&
Array.isArray(value.cues) &&
value.cues.every(isStreamCue) &&
typeof value.done === "number" &&
typeof value.total === "number"
);
}
function isTranslationCompletedPayload(
value: unknown,
): value is PlaybackSubtitleTranslationCompletedPayload {
return (
isRecord(value) &&
typeof value.session_id === "string" &&
typeof value.file_id === "number" &&
typeof value.job_id === "number" &&
typeof value.track_key === "string" &&
typeof value.subtitle_id === "number" &&
typeof value.language === "string"
);
}
function isTranslationFailedPayload(
value: unknown,
): value is PlaybackSubtitleTranslationFailedPayload {
return (
isRecord(value) &&
typeof value.session_id === "string" &&
typeof value.file_id === "number" &&
typeof value.job_id === "number" &&
typeof value.track_key === "string"
);
}
export function parsePlaybackRealtimeMessage(
data: string,
): PlaybackRealtimeCommandEnvelope | PlaybackRealtimeEventEnvelope | null {
@@ -206,6 +364,14 @@ export function parsePlaybackRealtimeMessage(
payload: value.payload,
};
}
if (value.name === "subtitle_ready" && isSubtitleReadyPayload(value.payload)) {
return {
type: "event",
session_id: value.session_id,
name: value.name,
payload: value.payload,
};
}
if (value.name === "markers_updated" && isMarkersUpdatedPayload(value.payload)) {
return {
type: "event",
@@ -214,6 +380,47 @@ export function parsePlaybackRealtimeMessage(
payload: value.payload,
};
}
if (
value.name === "subtitle_translation_started" &&
isTranslationStartedPayload(value.payload)
) {
return {
type: "event",
session_id: value.session_id,
name: value.name,
payload: value.payload,
};
}
if (value.name === "subtitle_translation_cues" && isTranslationCuesPayload(value.payload)) {
return {
type: "event",
session_id: value.session_id,
name: value.name,
payload: value.payload,
};
}
if (
value.name === "subtitle_translation_completed" &&
isTranslationCompletedPayload(value.payload)
) {
return {
type: "event",
session_id: value.session_id,
name: value.name,
payload: value.payload,
};
}
if (
value.name === "subtitle_translation_failed" &&
isTranslationFailedPayload(value.payload)
) {
return {
type: "event",
session_id: value.session_id,
name: value.name,
payload: value.payload,
};
}
}
return null;
} catch {
+12
View File
@@ -110,6 +110,12 @@ export interface PlaybackSessionPlaybackInfo {
/** Subtitle track information. */
export interface PlayerSubtitleInfo {
index: number;
/**
* Downloaded-subtitle row id, when this track is a stored downloaded subtitle.
* Lets the player match a translation-completed / `subtitle_ready` event
* (which carries the DB id) to a track after a list refresh.
*/
id?: number;
language: string;
codec?: string;
label: string;
@@ -117,6 +123,12 @@ export interface PlayerSubtitleInfo {
forced?: boolean;
hearing_impaired?: boolean;
url: string;
/**
* When true, this is an in-progress AI translation whose cues arrive over the
* realtime websocket rather than from `url`. `useSubtitleTracks` feeds it from
* the `liveCues` source instead of fetching.
*/
live?: boolean;
}
export interface PlayerSubtitleTrackSignature {