fix(player): shape ffmpeg demuxer audio extradata into media3 csd layouts

VP9/WebM-origin files stalled at 0ms in ExoPlayer on 2.17.0 and fell
back to MPV after ~40s. The ffmpeg demuxer published raw extradata as
the only initializationData entry; Android's Opus decoders consume
their first three input buffers as OpusHead/codec-delay/seek-pre-roll,
so they ate the first two real packets as configuration and silently
discarded every decoded sample, pinning the audio-driven clock at 0.
Vorbis (unsplit Xiph lacing) and FLAC (unmarked STREAMINFO) had the
same shape divergence.

Shape audio extradata into the layouts media3's own extractors emit
before publishing the track format.

close #2088

android/app/src/androidTest/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractorAudioCsdPlaybackTest.kt

@@ -0,0 +1,132 @@
+package com.edde746.plezy.exoplayer
+
+import android.content.Context
+import android.net.Uri
+import android.os.Handler
+import android.os.HandlerThread
+import androidx.media3.common.MediaItem
+import androidx.media3.common.PlaybackException
+import androidx.media3.common.Player
+import androidx.media3.datasource.DefaultDataSource
+import androidx.media3.exoplayer.ExoPlayer
+import androidx.media3.exoplayer.source.ProgressiveMediaSource
+import androidx.media3.extractor.ExtractorsFactory
+import androidx.media3.extractor.text.DefaultSubtitleParserFactory
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import com.edde746.plezy.libass.media.AssHandler
+import java.io.File
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicReference
+import org.junit.Assert.assertNotNull
+import org.junit.Assert.assertNull
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import org.junit.runner.RunWith
+
+/**
+ * End-to-end regression coverage for #2088: audio demuxed by FfmpegExtractor
+ * must reach MediaCodec with the per-codec csd shapes media3's own extractors
+ * emit. With raw ffmpeg extradata as the only csd buffer, Android's Opus
+ * decoders consume the first two real packets as codec delay / seek pre-roll
+ * and then silently discard every decoded sample — the player never leaves
+ * STATE_BUFFERING, which this test observes as a playback timeout.
+ *
+ * The extractors factory contains only FfmpegExtractor, so a fixture that
+ * falls back to media3's extractors fails instead of passing vacuously.
+ */
+@RunWith(AndroidJUnit4::class)
+class FfmpegExtractorAudioCsdPlaybackTest {
+
+  @Test
+  fun opusPlaysToEnd() {
+    playToEnd("ffmpeg/stereo_opus.mka")
+  }
+
+  @Test
+  fun vorbisPlaysToEnd() {
+    playToEnd("ffmpeg/stereo_vorbis.mka")
+  }
+
+  @Test
+  fun flacPlaysToEnd() {
+    playToEnd("ffmpeg/stereo.flac")
+  }
+
+  private fun playToEnd(fixture: String) {
+    val instrumentation = InstrumentationRegistry.getInstrumentation()
+    val context = instrumentation.targetContext
+    val fixtureFile = copyFixture(instrumentation.context, context, fixture)
+    val playbackThread = HandlerThread("ffmpeg-csd-test").apply { start() }
+    val handler = Handler(playbackThread.looper)
+    val completed = CountDownLatch(1)
+    val playerReference = AtomicReference<ExoPlayer>()
+    val errorReference = AtomicReference<Throwable>()
+
+    handler.post {
+      try {
+        val ffmpegOnlyExtractors = ExtractorsFactory {
+          val extractor = FfmpegExtractor.create(
+            { FfmpegDemuxerPolicy.Preference.FFMPEG },
+            DvConversionMode.DISABLED,
+            DefaultSubtitleParserFactory(),
+            AssHandler()
+          )
+          assertNotNull("FFmpeg demuxer JNI library is unavailable", extractor)
+          arrayOf(extractor!!)
+        }
+        val player = ExoPlayer.Builder(context, PlezyRenderersFactory(context))
+          .setLooper(playbackThread.looper)
+          .build()
+        playerReference.set(player)
+        player.addListener(object : Player.Listener {
+          override fun onPlayerError(error: PlaybackException) {
+            errorReference.set(error)
+            completed.countDown()
+          }
+
+          override fun onPlaybackStateChanged(playbackState: Int) {
+            if (playbackState == Player.STATE_ENDED) {
+              completed.countDown()
+            }
+          }
+        })
+        val source = ProgressiveMediaSource.Factory(
+          DefaultDataSource.Factory(context),
+          ffmpegOnlyExtractors
+        ).createMediaSource(MediaItem.fromUri(Uri.fromFile(fixtureFile)))
+        player.setMediaSource(source)
+        player.prepare()
+        player.play()
+      } catch (error: Throwable) {
+        errorReference.set(error)
+        completed.countDown()
+      }
+    }
+
+    val finished = completed.await(20, TimeUnit.SECONDS)
+    val released = CountDownLatch(1)
+    handler.post {
+      playerReference.get()?.release()
+      playbackThread.quitSafely()
+      released.countDown()
+    }
+    val teardownFinished = released.await(5, TimeUnit.SECONDS)
+    playbackThread.join(5_000)
+    val fixtureDeleted = fixtureFile.delete()
+
+    assertTrue("Player teardown timed out for $fixture", teardownFinished)
+    assertTrue("Playback timed out for $fixture", finished)
+    assertNull("Playback failed for $fixture", errorReference.get())
+    assertTrue("Fixture cleanup failed for $fixture", fixtureDeleted)
+  }
+
+  private fun copyFixture(instrumentationContext: Context, targetContext: Context, fixture: String): File {
+    val output = File.createTempFile("ffmpeg-csd-fixture-", null, targetContext.cacheDir)
+    instrumentationContext.assets.open(fixture).use { input ->
+      output.outputStream().use(input::copyTo)
+    }
+    return output
+  }
+}

android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsd.kt

@@ -0,0 +1,134 @@
+package com.edde746.plezy.exoplayer
+
+import androidx.annotation.OptIn
+import androidx.media3.common.MimeTypes
+import androidx.media3.common.ParserException
+import androidx.media3.common.util.UnstableApi
+import androidx.media3.container.OpusUtil
+
+/**
+ * Shapes ffmpeg codec extradata into the per-codec initializationData layout
+ * media3's audio decode paths consume. MediaCodecAudioRenderer maps the list
+ * entries verbatim to MediaCodec csd-0/1/2 and the bundled FfmpegAudioDecoder
+ * rebuilds avcodec extradata from the same entries, so both expect the shapes
+ * media3's own extractors emit — not raw ffmpeg extradata:
+ *
+ * - Opus: [OpusHead, codec delay ns, seek pre-roll ns]. Android's Opus
+ *   decoders consume their first three input buffers as exactly this
+ *   configuration; given only OpusHead they read the first two real packets
+ *   as delay/pre-roll and then silently discard every decoded sample, which
+ *   stalled the audio clock at 0 until the MPV fallback fired (#2088).
+ * - Vorbis: [identification header, setup header], split out of the
+ *   Xiph-laced blob the way MatroskaExtractor's parseVorbisCodecPrivate does.
+ * - FLAC: the "fLaC"-marked STREAMINFO stream MediaCodec documents; ffmpeg
+ *   surfaces a bare or block-headed STREAMINFO depending on the container.
+ *
+ * Everything else (AAC ASC, ALAC cookie, passthrough codecs) already matches
+ * media3's single-entry shape and passes through. MP3's WAVEFORMATEX blob is
+ * dropped on the JNI side before it reaches this shaping.
+ */
+@OptIn(UnstableApi::class)
+internal object FfmpegAudioCsd {
+
+  // RFC 7845 section 5.1: the identification header is at least 19 bytes;
+  // OpusUtil reads the pre-skip field at offsets 10-11.
+  private const val OPUS_HEADER_MIN_BYTES = 19
+
+  private val FLAC_STREAM_MARKER = byteArrayOf(0x66, 0x4C, 0x61, 0x43) // "fLaC"
+  private const val FLAC_STREAM_INFO_SIZE = 34
+  private const val FLAC_BLOCK_HEADER_SIZE = 4
+  private const val FLAC_LAST_BLOCK_FLAG = 0x80
+
+  fun initializationData(mime: String, extradata: ByteArray): List<ByteArray> = when (mime) {
+    MimeTypes.AUDIO_OPUS -> opus(extradata)
+    MimeTypes.AUDIO_VORBIS -> vorbis(extradata)
+    MimeTypes.AUDIO_FLAC -> flac(extradata)
+    else -> listOf(extradata)
+  }
+
+  private fun opus(extradata: ByteArray): List<ByteArray> {
+    if (extradata.size < OPUS_HEADER_MIN_BYTES) {
+      throw malformed("Opus extradata is not an identification header: ${extradata.size} bytes")
+    }
+    // [OpusHead, pre-skip ns, 80 ms default seek pre-roll ns] — the same
+    // shape media3's Matroska/Ogg/MP4 extractors publish.
+    return OpusUtil.buildInitializationData(extradata)
+  }
+
+  /**
+   * Splits the Xiph-laced header blob (identification, comment, setup) into
+   * media3's [identification header, setup header] pair, mirroring media3
+   * 1.11.0 MatroskaExtractor.parseVorbisCodecPrivate.
+   */
+  private fun vorbis(extradata: ByteArray): List<ByteArray> {
+    try {
+      if (extradata[0].toInt() != 0x02) throw malformed("Vorbis extradata is not Xiph-laced headers")
+      var offset = 1
+      var idHeaderLength = 0
+      while (extradata[offset].toInt() and 0xFF == 0xFF) {
+        idHeaderLength += 0xFF
+        offset++
+      }
+      idHeaderLength += extradata[offset++].toInt() and 0xFF
+      var commentHeaderLength = 0
+      while (extradata[offset].toInt() and 0xFF == 0xFF) {
+        commentHeaderLength += 0xFF
+        offset++
+      }
+      commentHeaderLength += extradata[offset++].toInt() and 0xFF
+
+      if (extradata[offset].toInt() != 0x01) throw malformed("Vorbis identification header not found")
+      val idHeader = extradata.copyOfRange(offset, offset + idHeaderLength)
+      offset += idHeaderLength
+      if (extradata[offset].toInt() != 0x03) throw malformed("Vorbis comment header not found")
+      offset += commentHeaderLength
+      if (extradata[offset].toInt() != 0x05) throw malformed("Vorbis setup header not found")
+      return listOf(idHeader, extradata.copyOfRange(offset, extradata.size))
+    } catch (e: IndexOutOfBoundsException) {
+      throw malformed("Vorbis extradata truncated")
+    }
+  }
+
+  /**
+   * Normalizes to the "fLaC"-marked metadata stream. MediaCodec's decoder
+   * feeds csd-0 straight into libFLAC, which requires the stream marker;
+   * ffmpeg's canonical FLAC extradata is the bare 34-byte STREAMINFO, and
+   * some demuxers keep the 4-byte metadata block header in front of it.
+   */
+  private fun flac(extradata: ByteArray): List<ByteArray> {
+    if (startsWithFlacMarker(extradata)) return listOf(extradata)
+    if (extradata.size == FLAC_STREAM_INFO_SIZE) {
+      val blockHeader = byteArrayOf(
+        (FLAC_LAST_BLOCK_FLAG or 0).toByte(), // last-metadata-block, type 0 = STREAMINFO
+        0x00,
+        0x00,
+        FLAC_STREAM_INFO_SIZE.toByte()
+      )
+      return listOf(FLAC_STREAM_MARKER + blockHeader + extradata)
+    }
+    val declaredLength = if (extradata.size >= FLAC_BLOCK_HEADER_SIZE) {
+      ((extradata[1].toInt() and 0xFF) shl 16) or
+        ((extradata[2].toInt() and 0xFF) shl 8) or
+        (extradata[3].toInt() and 0xFF)
+    } else {
+      -1
+    }
+    if (extradata[0].toInt() and 0x7F == 0 &&
+      declaredLength == FLAC_STREAM_INFO_SIZE &&
+      extradata.size >= FLAC_BLOCK_HEADER_SIZE + FLAC_STREAM_INFO_SIZE
+    ) {
+      return listOf(FLAC_STREAM_MARKER + extradata)
+    }
+    throw malformed("FLAC extradata is not a STREAMINFO block: ${extradata.size} bytes")
+  }
+
+  private fun startsWithFlacMarker(extradata: ByteArray): Boolean {
+    if (extradata.size < FLAC_STREAM_MARKER.size) return false
+    for (i in FLAC_STREAM_MARKER.indices) {
+      if (extradata[i] != FLAC_STREAM_MARKER[i]) return false
+    }
+    return true
+  }
+
+  private fun malformed(message: String): ParserException = ParserException.createForMalformedContainer(message, null)
+}

android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractor.kt

@@ -336,7 +336,13 @@ internal class FfmpegExtractor private constructor(
 
       val extradata = FfmpegDemuxerJni.nativeStreamExtradata(index)
       if (extradata != null) {
-        builder.setInitializationData(listOf(extradata))
+        // Audio csd is codec-specific in media3 (see FfmpegAudioCsd); raw
+        // extradata is only correct for video (already Annex-B via the JNI
+        // bitstream filter) and text tracks.
+        builder.setInitializationData(
+          if (trackType == C.TRACK_TYPE_AUDIO) FfmpegAudioCsd.initializationData(mime, extradata)
+          else listOf(extradata)
+        )
       }
 
       when (trackType) {

android/app/src/test/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsdTest.kt

@@ -0,0 +1,138 @@
+package com.edde746.plezy.exoplayer
+
+import androidx.media3.common.MimeTypes
+import androidx.media3.common.ParserException
+import java.nio.ByteBuffer
+import java.nio.ByteOrder
+import org.junit.Assert.assertArrayEquals
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertSame
+import org.junit.Assert.assertThrows
+import org.junit.Test
+
+/**
+ * Spec for the per-codec initializationData shapes the ffmpeg demuxer must
+ * publish so MediaCodec csd-0/1/2 and FfmpegAudioDecoder's extradata
+ * reconstruction both see what media3's own extractors emit. Regression
+ * coverage for #2088: Opus with a bare OpusHead csd made Android's decoders
+ * eat the first two real packets as configuration and discard all output.
+ */
+class FfmpegAudioCsdTest {
+
+  private fun opusHead(preSkip: Int): ByteArray {
+    val header = ByteArray(19)
+    "OpusHead".toByteArray(Charsets.US_ASCII).copyInto(header)
+    header[8] = 1 // version
+    header[9] = 2 // channel count
+    header[10] = (preSkip and 0xFF).toByte()
+    header[11] = (preSkip ushr 8).toByte()
+    return header
+  }
+
+  private fun nativeOrderLong(bytes: ByteArray): Long = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).long
+
+  @Test
+  fun opusBuildsThreeCsdBuffers() {
+    val header = opusHead(preSkip = 312)
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_OPUS, header)
+    assertEquals(3, csd.size)
+    assertArrayEquals(header, csd[0])
+    // Pre-skip converted to nanoseconds at the Opus 48 kHz clock.
+    assertEquals(312L * 1_000_000_000L / 48_000L, nativeOrderLong(csd[1]))
+    // media3's default seek pre-roll: 3840 samples = 80 ms.
+    assertEquals(80_000_000L, nativeOrderLong(csd[2]))
+  }
+
+  @Test
+  fun opusRejectsTruncatedHeader() {
+    assertThrows(ParserException::class.java) {
+      FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_OPUS, ByteArray(11))
+    }
+  }
+
+  private fun xiphLace(length: Int): ByteArray {
+    val bytes = ArrayList<Byte>()
+    var remaining = length
+    while (remaining >= 0xFF) {
+      bytes.add(0xFF.toByte())
+      remaining -= 0xFF
+    }
+    bytes.add(remaining.toByte())
+    return bytes.toByteArray()
+  }
+
+  private fun vorbisHeader(type: Int, payloadLength: Int): ByteArray =
+    byteArrayOf(type.toByte()) + "vorbis".toByteArray(Charsets.US_ASCII) + ByteArray(payloadLength)
+
+  @Test
+  fun vorbisSplitsLacedHeadersIntoIdentificationAndSetup() {
+    val id = vorbisHeader(0x01, payloadLength = 23) // 30 bytes, the real identification header size
+    val comment = vorbisHeader(0x03, payloadLength = 300) // > 255 exercises the 0xFF lacing
+    val setup = vorbisHeader(0x05, payloadLength = 40)
+    val extradata = byteArrayOf(0x02) + xiphLace(id.size) + xiphLace(comment.size) + id + comment + setup
+
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, extradata)
+
+    assertEquals(2, csd.size)
+    assertArrayEquals(id, csd[0])
+    assertArrayEquals(setup, csd[1])
+  }
+
+  @Test
+  fun vorbisRejectsNonLacedExtradata() {
+    assertThrows(ParserException::class.java) {
+      FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, byteArrayOf(0x01, 0x02, 0x03))
+    }
+  }
+
+  @Test
+  fun vorbisRejectsTruncatedExtradata() {
+    val id = vorbisHeader(0x01, payloadLength = 23)
+    val truncated = byteArrayOf(0x02) + xiphLace(id.size) + xiphLace(64) + id // headers end early
+    assertThrows(ParserException::class.java) {
+      FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, truncated)
+    }
+  }
+
+  @Test
+  fun flacWrapsBareStreamInfoInMarkedStream() {
+    val streamInfo = ByteArray(34) { it.toByte() }
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, streamInfo)
+    assertEquals(1, csd.size)
+    assertArrayEquals(
+      "fLaC".toByteArray(Charsets.US_ASCII) + byteArrayOf(0x80.toByte(), 0x00, 0x00, 34) + streamInfo,
+      csd[0]
+    )
+  }
+
+  @Test
+  fun flacPrependsMarkerToBlockHeadedStreamInfo() {
+    val block = byteArrayOf(0x00, 0x00, 0x00, 34) + ByteArray(34) { it.toByte() }
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, block)
+    assertEquals(1, csd.size)
+    assertArrayEquals("fLaC".toByteArray(Charsets.US_ASCII) + block, csd[0])
+  }
+
+  @Test
+  fun flacKeepsMarkedStreamVerbatim() {
+    val marked = "fLaC".toByteArray(Charsets.US_ASCII) + byteArrayOf(0x80.toByte(), 0x00, 0x00, 34) + ByteArray(34)
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, marked)
+    assertEquals(1, csd.size)
+    assertSame(marked, csd[0])
+  }
+
+  @Test
+  fun flacRejectsUnrecognizedExtradata() {
+    assertThrows(ParserException::class.java) {
+      FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, ByteArray(10))
+    }
+  }
+
+  @Test
+  fun otherCodecsPassExtradataThroughAsSingleEntry() {
+    val asc = byteArrayOf(0x12, 0x10) // AAC-LC 44.1 kHz stereo AudioSpecificConfig
+    val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_AAC, asc)
+    assertEquals(1, csd.size)
+    assertSame(asc, csd[0])
+  }
+}