fix(playback): adopt renamed FormatFFmpegProbeFailure after rebase

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

internal/playback/gpudetect.go

@@ -13,6 +13,8 @@ import (
 	"time"
 )
 
+const darwinGOOS = "darwin"
+
 var (
 	defaultDRIDir              = "/dev/dri"
 	defaultNVIDIAControlDevice = "/dev/nvidiactl"
@@ -34,6 +36,13 @@ var nvencProbeCache = struct {
 	byPath: make(map[string]nvencProbeResult),
 }
 
+var videoToolboxProbeCache = struct {
+	sync.Mutex
+	byPath map[string]nvencProbeResult
+}{
+	byPath: make(map[string]nvencProbeResult),
+}
+
 // HWAccelInfo describes the detected hardware acceleration capability.
 type HWAccelInfo struct {
 	Resolved            string             `json:"resolved"`
@@ -99,6 +108,16 @@ func ResolveHWAccelWithFFmpeg(hwAccel string, ffmpegPath string) string {
 	if hwAccel != "auto" {
 		return hwAccel
 	}
+	if currentGOOS == darwinGOOS {
+		if ok, reason := ffmpegSupportsVideoToolbox(ffmpegPath); ok {
+			slog.Info("hw_accel=auto: macOS detected, using VideoToolbox")
+			return transcodeHWVideoToolbox
+		} else {
+			slog.Warn("hw_accel=auto: macOS detected but FFmpeg VideoToolbox probe failed",
+				"ffmpeg", normalizeFFmpegPath(ffmpegPath), "reason", reason)
+		}
+		return transcodeHWNone
+	}
 	if currentGOOS != "linux" {
 		return "none"
 	}
@@ -168,6 +187,70 @@ func ffmpegSupportsNVENC(ffmpegPath string) (bool, string) {
 	return result.available, result.reason
 }
 
+func ffmpegSupportsVideoToolbox(ffmpegPath string) (bool, string) {
+	ffmpegPath = normalizeFFmpegPath(ffmpegPath)
+	videoToolboxProbeCache.Lock()
+	if result, ok := videoToolboxProbeCache.byPath[ffmpegPath]; ok {
+		videoToolboxProbeCache.Unlock()
+		return result.available, result.reason
+	}
+	videoToolboxProbeCache.Unlock()
+
+	result := probeFFmpegVideoToolbox(ffmpegPath)
+	videoToolboxProbeCache.Lock()
+	videoToolboxProbeCache.byPath[ffmpegPath] = result
+	videoToolboxProbeCache.Unlock()
+	return result.available, result.reason
+}
+
+// probeFFmpegVideoToolbox verifies the configured FFmpeg exposes VideoToolbox
+// decode plus both encoders, then smoke-encodes one frame per encoder in the
+// same constant-quality mode the uncapped transcode path emits (-q:v needs an
+// Apple Silicon compression session; Intel Macs must fail here so auto
+// resolves to software instead of approving commands that cannot run). No
+// filter probes: the videotoolbox pipeline keeps decoded frames in system
+// memory, so the regular software filter graph applies (see appendHWAccelArgs).
+func probeFFmpegVideoToolbox(ffmpegPath string) nvencProbeResult {
+	if output, err := runFFmpegProbe(ffmpegPath, "-hide_banner", "-hwaccels"); err != nil {
+		return nvencProbeResult{reason: "hwaccels probe failed: " + FormatFFmpegProbeFailure(err, output)}
+	} else if !ffmpegOutputHasToken(output, "videotoolbox") {
+		return nvencProbeResult{reason: "videotoolbox hwaccel unavailable"}
+	}
+
+	if output, err := runFFmpegProbe(ffmpegPath, "-hide_banner", "-encoders"); err != nil {
+		return nvencProbeResult{reason: "encoders probe failed: " + FormatFFmpegProbeFailure(err, output)}
+	} else if !ffmpegOutputHasToken(output, "h264_videotoolbox") {
+		return nvencProbeResult{reason: "h264_videotoolbox encoder unavailable"}
+	} else if !ffmpegOutputHasToken(output, "hevc_videotoolbox") {
+		return nvencProbeResult{reason: "hevc_videotoolbox encoder unavailable"}
+	}
+
+	for _, smoke := range []struct {
+		encoder string
+		quality string
+	}{
+		{"h264_videotoolbox", "65"},
+		{"hevc_videotoolbox", "60"},
+	} {
+		if output, err := runFFmpegProbe(ffmpegPath,
+			"-hide_banner",
+			"-loglevel", "error",
+			"-f", "lavfi",
+			"-i", "testsrc2=size=640x360:rate=1",
+			"-frames:v", "1",
+			"-an",
+			"-c:v", smoke.encoder,
+			"-q:v", smoke.quality,
+			"-f", "null",
+			"-",
+		); err != nil {
+			return nvencProbeResult{reason: smoke.encoder + " smoke encode failed: " + FormatFFmpegProbeFailure(err, output)}
+		}
+	}
+
+	return nvencProbeResult{available: true}
+}
+
 func normalizeFFmpegPath(ffmpegPath string) string {
 	ffmpegPath = strings.TrimSpace(ffmpegPath)
 	if ffmpegPath == "" {

internal/playback/gpudetect_test.go

@@ -16,13 +16,16 @@ type hwAccelTestEnv struct {
 }
 
 type fakeFFmpegProbe struct {
-	cuda       bool
-	h264NVENC  bool
-	hevcNVENC  bool
-	scaleCUDA  bool
-	uploadCUDA bool
-	smokeOK    bool
-	hang       bool
+	cuda         bool
+	h264NVENC    bool
+	hevcNVENC    bool
+	scaleCUDA    bool
+	uploadCUDA   bool
+	videotoolbox bool
+	h264VT       bool
+	hevcVT       bool
+	smokeOK      bool
+	hang         bool
 }
 
 type fakeFFmpegBinary struct {
@@ -289,6 +292,9 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary {
 	if probe.cuda {
 		script += "    echo 'cuda'\n"
 	}
+	if probe.videotoolbox {
+		script += "    echo 'videotoolbox'\n"
+	}
 	script += "    exit 0 ;;\n"
 	script += "  *-encoders*)\n"
 	if probe.h264NVENC {
@@ -297,6 +303,12 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary {
 	if probe.hevcNVENC {
 		script += "    echo ' V..... hevc_nvenc NVIDIA NVENC hevc encoder'\n"
 	}
+	if probe.h264VT {
+		script += "    echo ' V..... h264_videotoolbox VideoToolbox H.264 encoder'\n"
+	}
+	if probe.hevcVT {
+		script += "    echo ' V..... hevc_videotoolbox VideoToolbox hevc encoder'\n"
+	}
 	script += "    exit 0 ;;\n"
 	script += "  *-filters*)\n"
 	if probe.scaleCUDA {
@@ -323,6 +335,83 @@ func writeFakeFFmpeg(t *testing.T, probe fakeFFmpegProbe) fakeFFmpegBinary {
 
 func resetNVENCProbeCacheForTest() {
 	nvencProbeCache.Lock()
-	defer nvencProbeCache.Unlock()
 	nvencProbeCache.byPath = make(map[string]nvencProbeResult)
+	nvencProbeCache.Unlock()
+	videoToolboxProbeCache.Lock()
+	videoToolboxProbeCache.byPath = make(map[string]nvencProbeResult)
+	videoToolboxProbeCache.Unlock()
+}
+
+func successfulVideoToolboxProbe() fakeFFmpegProbe {
+	return fakeFFmpegProbe{videotoolbox: true, h264VT: true, hevcVT: true, smokeOK: true}
+}
+
+func TestResolveHWAccelWithFFmpegDarwinUsesVideoToolbox(t *testing.T) {
+	setupHWAccelTest(t)
+	currentGOOS = "darwin"
+	ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe())
+
+	if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "videotoolbox" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got)
+	}
+}
+
+func TestResolveHWAccelWithFFmpegDarwinFallsBackToNoneWhenProbeFails(t *testing.T) {
+	setupHWAccelTest(t)
+	currentGOOS = "darwin"
+	ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{})
+
+	if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none", got)
+	}
+}
+
+func TestResolveHWAccelWithFFmpegDarwinRequiresBothVTEncoders(t *testing.T) {
+	setupHWAccelTest(t)
+	currentGOOS = "darwin"
+	ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{videotoolbox: true, h264VT: true, smokeOK: true})
+
+	if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none when hevc_videotoolbox is missing", got)
+	}
+}
+
+func TestResolveHWAccelWithFFmpegDarwinFallsBackToNoneWhenSmokeEncodeFails(t *testing.T) {
+	setupHWAccelTest(t)
+	currentGOOS = "darwin"
+	ffmpeg := writeFakeFFmpeg(t, fakeFFmpegProbe{videotoolbox: true, h264VT: true, hevcVT: true})
+
+	if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "none" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want none when smoke encode fails", got)
+	}
+}
+
+func TestVideoToolboxProbeSmokesBothEncodersInConstantQualityMode(t *testing.T) {
+	setupHWAccelTest(t)
+	currentGOOS = "darwin"
+	ffmpeg := writeFakeFFmpeg(t, successfulVideoToolboxProbe())
+
+	if got := ResolveHWAccelWithFFmpeg("auto", ffmpeg.path); got != "videotoolbox" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got)
+	}
+	log, err := os.ReadFile(ffmpeg.logPath)
+	if err != nil {
+		t.Fatalf("read probe log: %v", err)
+	}
+	// The smoke encodes must exercise the exact uncapped (-q:v) mode the
+	// transcode arg builder emits — Intel Macs fail that mode at session
+	// creation, so probing without it would approve unrunnable commands.
+	for _, want := range []string{"-c:v h264_videotoolbox -q:v 65", "-c:v hevc_videotoolbox -q:v 60"} {
+		if !strings.Contains(string(log), want) {
+			t.Fatalf("probe log missing %q:\n%s", want, log)
+		}
+	}
+}
+
+func TestExplicitVideoToolboxBypassesFFmpegProbe(t *testing.T) {
+	setupHWAccelTest(t)
+
+	if got := ResolveHWAccelWithFFmpeg("videotoolbox", "/does/not/exist/ffmpeg"); got != "videotoolbox" {
+		t.Fatalf("ResolveHWAccelWithFFmpeg() = %q, want videotoolbox", got)
+	}
 }

internal/playback/transcode.go

@@ -59,7 +59,7 @@ type TranscodeOpts struct {
 	SegmentDuration        int    // seconds, default 6
 	StartSegmentNumber     int    // -hls_segment_start_number, default 0
 	FFmpegPath             string // optional explicit ffmpeg binary path
-	HWAccel                string // auto, qsv, vaapi, nvenc, none
+	HWAccel                string // auto, qsv, vaapi, nvenc, videotoolbox, none
 	HWDevice               string // e.g., /dev/dri/renderD128 (default if empty)
 	// AvoidHWDevice asks the initial multi-device allocator to prefer any other
 	// present render device. It is a process-local startup hint used after an
@@ -99,10 +99,12 @@ type TranscodeOpts struct {
 const DV7ToHDR10BitstreamFilter = "dovi_rpu=strip=1"
 
 const (
-	transcodeCodecH264 = "h264"
-	transcodeHWQSV     = "qsv"
-	transcodeHWVAAPI   = "vaapi"
-	transcodeHWNVENC   = "nvenc"
+	transcodeCodecH264      = "h264"
+	transcodeHWQSV          = "qsv"
+	transcodeHWVAAPI        = "vaapi"
+	transcodeHWNVENC        = "nvenc"
+	transcodeHWVideoToolbox = "videotoolbox"
+	transcodeHWNone         = "none"
 )
 
 // TranscodeSession manages a running ffmpeg HLS transcode process.
@@ -594,12 +596,13 @@ func appendSegmentBoundaryArgs(args []string, opts TranscodeOpts) []string {
 			fmt.Sprintf("expr:gte(t,n_forced*%d)", opts.SegmentDuration))
 	}
 
-	// Hardware encoders (QSV, VAAPI, NVENC) may not reliably honor
-	// force_key_frames expressions. Set explicit GOP size so segment
+	// Hardware encoders (QSV, VAAPI, NVENC, VideoToolbox) may not reliably
+	// honor force_key_frames expressions. Set explicit GOP size so segment
 	// boundaries always start with an intra frame. We assume 30 fps as a
 	// safe ceiling — the GOP will be at most segmentDuration * 30 frames.
 	// Matches Jellyfin's approach for hardware encoders.
-	if opts.HWAccel == transcodeHWQSV || opts.HWAccel == transcodeHWVAAPI || opts.HWAccel == transcodeHWNVENC {
+	if opts.HWAccel == transcodeHWQSV || opts.HWAccel == transcodeHWVAAPI ||
+		opts.HWAccel == transcodeHWNVENC || opts.HWAccel == transcodeHWVideoToolbox {
 		gopSize := fmt.Sprintf("%d", opts.SegmentDuration*30)
 		args = append(args, "-g", gopSize, "-keyint_min", gopSize)
 	}
@@ -656,6 +659,16 @@ func appendHWAccelArgs(args []string, opts TranscodeOpts) []string {
 		if hwDevice := strings.TrimSpace(opts.HWDevice); hwDevice != "" {
 			args = append(args, "-hwaccel_device", hwDevice)
 		}
+	case transcodeHWVideoToolbox:
+		// No -hwaccel_output_format: decoded frames land in system memory,
+		// so the software filter graph (scale, subtitle burn-in, tone paths)
+		// applies unchanged and the *_videotoolbox encoders upload
+		// internally. HW decode + HW encode carries nearly all of the win.
+		// H.264 Hi10P decodes in software (VideoToolbox cannot), same as the
+		// QSV/VAAPI paths; the encoder still runs in hardware.
+		if !opts.SoftwareVideoDecode {
+			args = append(args, "-hwaccel", "videotoolbox")
+		}
 	}
 	return args
 }
@@ -741,6 +754,32 @@ func appendVideoArgs(args []string, opts TranscodeOpts) []string {
 		} else {
 			args = append(args, "-cq:v", "28", "-b:v", "0")
 		}
+	case opts.HWAccel == transcodeHWVideoToolbox && codec == transcodeCodecH264:
+		// 8-bit output for browser MSE compatibility (mirrors the libx264
+		// path); VideoToolbox has no 10-bit H.264 encode.
+		args = append(args, "-c:v", "h264_videotoolbox",
+			"-pix_fmt", "yuv420p", "-profile:v", "high")
+		if hasBitrateCap {
+			args = append(args,
+				"-b:v", fmt.Sprintf("%dk", opts.TargetBitrateKbps),
+				"-maxrate", fmt.Sprintf("%dk", opts.TargetBitrateKbps),
+				"-bufsize", fmt.Sprintf("%dk", opts.TargetBitrateKbps*2))
+		} else {
+			// Constant-quality mode (1-100 scale, hardware-rate-controlled).
+			args = append(args, "-q:v", "65")
+		}
+	case opts.HWAccel == transcodeHWVideoToolbox && codec == "hevc":
+		// pix_fmt is left to the input: 10-bit sources encode as p010
+		// (HDR10 passthrough), matching the other hardware HEVC paths.
+		args = append(args, "-c:v", "hevc_videotoolbox")
+		if hasBitrateCap {
+			args = append(args,
+				"-b:v", fmt.Sprintf("%dk", opts.TargetBitrateKbps),
+				"-maxrate", fmt.Sprintf("%dk", opts.TargetBitrateKbps),
+				"-bufsize", fmt.Sprintf("%dk", opts.TargetBitrateKbps*2))
+		} else {
+			args = append(args, "-q:v", "60")
+		}
 	default:
 		// CPU fallback — match Jellyfin's proven browser-compatible settings.
 		// Force yuv420p to ensure 8-bit output (10-bit sources produce High 10

internal/playback/transcode_args_test.go

@@ -875,3 +875,116 @@ func TestTranscodesAudioMatchesFFmpegDefault(t *testing.T) {
 		}
 	}
 }
+
+func TestBuildFFmpegArgs_VideoToolboxH264UsesSoftwareFilters(t *testing.T) {
+	args := buildFFmpegArgs(TranscodeOpts{
+		InputPath:         "/media/movie.mkv",
+		OutputDir:         "/tmp/out",
+		SessionID:         "session-vt",
+		SourceVideoCodec:  "h264",
+		TargetCodecVideo:  "h264",
+		TargetCodecAudio:  "aac",
+		SegmentDuration:   2,
+		HWAccel:           "videotoolbox",
+		TargetResolution:  "720p",
+		TargetBitrateKbps: 2000,
+	})
+
+	joined := strings.Join(args, " ")
+	if !strings.Contains(joined, "-hwaccel videotoolbox") {
+		t.Fatalf("videotoolbox args should enable videotoolbox hwaccel: %s", joined)
+	}
+	if strings.Contains(joined, "-hwaccel_output_format") {
+		t.Fatalf("videotoolbox decode must output software frames: %s", joined)
+	}
+	if !strings.Contains(joined, "-c:v h264_videotoolbox") {
+		t.Fatalf("videotoolbox args should use h264_videotoolbox encoder: %s", joined)
+	}
+	if !strings.Contains(joined, "-pix_fmt yuv420p") {
+		t.Fatalf("videotoolbox h264 should force 8-bit output: %s", joined)
+	}
+	if !strings.Contains(joined, "-vf scale=-2:720") {
+		t.Fatalf("videotoolbox args should use the software scale filter: %s", joined)
+	}
+	if !strings.Contains(joined, "-b:v 2000k -maxrate 2000k -bufsize 4000k") {
+		t.Fatalf("videotoolbox args should include bitrate cap controls: %s", joined)
+	}
+	if !strings.Contains(joined, "-g 60 -keyint_min 60") {
+		t.Fatalf("videotoolbox args should force GOP on segment boundaries: %s", joined)
+	}
+	if strings.Contains(joined, "-preset") {
+		t.Fatalf("videotoolbox encoder does not take x264-style presets: %s", joined)
+	}
+}
+
+func TestBuildFFmpegArgs_VideoToolboxHi10PDecodesInSoftware(t *testing.T) {
+	args := buildFFmpegArgs(TranscodeOpts{
+		InputPath:          "/media/movie.mkv",
+		OutputDir:          "/tmp/out",
+		SessionID:          "session-vt-hi10p",
+		SourceVideoCodec:   "h264",
+		SourceVideoProfile: "High 10",
+		TargetCodecVideo:   "h264",
+		TargetCodecAudio:   "aac",
+		SegmentDuration:    2,
+		HWAccel:            "videotoolbox",
+		TargetBitrateKbps:  2000,
+	})
+
+	joined := strings.Join(args, " ")
+	if strings.Contains(joined, "-hwaccel videotoolbox") {
+		t.Fatalf("Hi10P source must decode in software (VideoToolbox cannot): %s", joined)
+	}
+	if !strings.Contains(joined, "-c:v h264_videotoolbox") {
+		t.Fatalf("encode should still use the hardware encoder: %s", joined)
+	}
+}
+
+func TestBuildFFmpegArgs_VideoToolboxHEVCKeepsSourceBitDepth(t *testing.T) {
+	args := buildFFmpegArgs(TranscodeOpts{
+		InputPath:        "/media/movie.mkv",
+		OutputDir:        "/tmp/out",
+		SessionID:        "session-vt-hevc",
+		SourceVideoCodec: "hevc",
+		TargetCodecVideo: "hevc",
+		TargetCodecAudio: "copy",
+		SegmentDuration:  2,
+		HWAccel:          "videotoolbox",
+	})
+
+	joined := strings.Join(args, " ")
+	if !strings.Contains(joined, "-c:v hevc_videotoolbox") {
+		t.Fatalf("videotoolbox args should use hevc_videotoolbox encoder: %s", joined)
+	}
+	if strings.Contains(joined, "-pix_fmt") {
+		t.Fatalf("videotoolbox hevc must not force a pixel format (HDR10 passthrough): %s", joined)
+	}
+	if !strings.Contains(joined, "-q:v 60") {
+		t.Fatalf("uncapped videotoolbox hevc should use constant-quality mode: %s", joined)
+	}
+}
+
+func TestBuildFFmpegArgs_VideoToolboxTextBurnInStaysOnCPUFilters(t *testing.T) {
+	args := buildFFmpegArgs(TranscodeOpts{
+		InputPath:          "/media/movie.mkv",
+		OutputDir:          "/tmp/out",
+		SessionID:          "session-vt-sub",
+		SourceVideoCodec:   "h264",
+		TargetCodecVideo:   "h264",
+		TargetCodecAudio:   "aac",
+		SegmentDuration:    2,
+		HWAccel:            "videotoolbox",
+		TargetResolution:   "720p",
+		SubtitleBurnIn:     true,
+		SubtitleTrackIndex: 0,
+		SubtitleCodec:      "subrip",
+	})
+
+	joined := strings.Join(args, " ")
+	if !strings.Contains(joined, "subtitles=") {
+		t.Fatalf("text burn-in should use the subtitles filter: %s", joined)
+	}
+	if strings.Contains(joined, "hwdownload") || strings.Contains(joined, "hwupload") {
+		t.Fatalf("videotoolbox burn-in runs on software frames, no hw round-trip: %s", joined)
+	}
+}

web/src/pages/admin-settings/PlaybackSettings.tsx

@@ -78,6 +78,7 @@ export default function PlaybackSettings() {
               { value: "qsv", label: "Intel Quick Sync (QSV)" },
               { value: "vaapi", label: "VA-API" },
               { value: "nvenc", label: "NVIDIA NVENC" },
+              { value: "videotoolbox", label: "VideoToolbox (macOS)" },
               { value: "none", label: "Software" },
             ]}
             value={form.getValue("playback.hw_accel")}
@@ -283,6 +284,8 @@ function formatResolved(resolved: string): string {
       return "VA-API";
     case "nvenc":
       return "NVIDIA NVENC";
+    case "videotoolbox":
+      return "VideoToolbox (macOS)";
     case "none":
       return "Software";
     default:

web/src/pages/setup-wizard/steps/ServerStorageStep.tsx

@@ -342,6 +342,7 @@ export function ServerStorageStep() {
                 <SelectItem value="auto">Auto</SelectItem>
                 <SelectItem value="vaapi">VAAPI</SelectItem>
                 <SelectItem value="nvenc">NVENC</SelectItem>
+                <SelectItem value="videotoolbox">VideoToolbox (macOS)</SelectItem>
                 <SelectItem value="qsv">QSV</SelectItem>
                 <SelectItem value="none">None</SelectItem>
               </SelectContent>