diff --git a/android/app/src/androidTest/assets/ffmpeg/stereo_opus.mka b/android/app/src/androidTest/assets/ffmpeg/stereo_opus.mka new file mode 100644 index 000000000..a868c1006 Binary files /dev/null and b/android/app/src/androidTest/assets/ffmpeg/stereo_opus.mka differ diff --git a/android/app/src/androidTest/assets/ffmpeg/stereo_vorbis.mka b/android/app/src/androidTest/assets/ffmpeg/stereo_vorbis.mka new file mode 100644 index 000000000..c9b243417 Binary files /dev/null and b/android/app/src/androidTest/assets/ffmpeg/stereo_vorbis.mka differ diff --git a/android/app/src/androidTest/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractorAudioCsdPlaybackTest.kt b/android/app/src/androidTest/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractorAudioCsdPlaybackTest.kt new file mode 100644 index 000000000..d37792c76 --- /dev/null +++ b/android/app/src/androidTest/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractorAudioCsdPlaybackTest.kt @@ -0,0 +1,132 @@ +package com.edde746.plezy.exoplayer + +import android.content.Context +import android.net.Uri +import android.os.Handler +import android.os.HandlerThread +import androidx.media3.common.MediaItem +import androidx.media3.common.PlaybackException +import androidx.media3.common.Player +import androidx.media3.datasource.DefaultDataSource +import androidx.media3.exoplayer.ExoPlayer +import androidx.media3.exoplayer.source.ProgressiveMediaSource +import androidx.media3.extractor.ExtractorsFactory +import androidx.media3.extractor.text.DefaultSubtitleParserFactory +import androidx.test.ext.junit.runners.AndroidJUnit4 +import androidx.test.platform.app.InstrumentationRegistry +import com.edde746.plezy.libass.media.AssHandler +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicReference +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +/** + * End-to-end regression coverage for #2088: audio demuxed by FfmpegExtractor + * must reach MediaCodec with the per-codec csd shapes media3's own extractors + * emit. With raw ffmpeg extradata as the only csd buffer, Android's Opus + * decoders consume the first two real packets as codec delay / seek pre-roll + * and then silently discard every decoded sample — the player never leaves + * STATE_BUFFERING, which this test observes as a playback timeout. + * + * The extractors factory contains only FfmpegExtractor, so a fixture that + * falls back to media3's extractors fails instead of passing vacuously. + */ +@RunWith(AndroidJUnit4::class) +class FfmpegExtractorAudioCsdPlaybackTest { + + @Test + fun opusPlaysToEnd() { + playToEnd("ffmpeg/stereo_opus.mka") + } + + @Test + fun vorbisPlaysToEnd() { + playToEnd("ffmpeg/stereo_vorbis.mka") + } + + @Test + fun flacPlaysToEnd() { + playToEnd("ffmpeg/stereo.flac") + } + + private fun playToEnd(fixture: String) { + val instrumentation = InstrumentationRegistry.getInstrumentation() + val context = instrumentation.targetContext + val fixtureFile = copyFixture(instrumentation.context, context, fixture) + val playbackThread = HandlerThread("ffmpeg-csd-test").apply { start() } + val handler = Handler(playbackThread.looper) + val completed = CountDownLatch(1) + val playerReference = AtomicReference() + val errorReference = AtomicReference() + + handler.post { + try { + val ffmpegOnlyExtractors = ExtractorsFactory { + val extractor = FfmpegExtractor.create( + { FfmpegDemuxerPolicy.Preference.FFMPEG }, + DvConversionMode.DISABLED, + DefaultSubtitleParserFactory(), + AssHandler() + ) + assertNotNull("FFmpeg demuxer JNI library is unavailable", extractor) + arrayOf(extractor!!) + } + val player = ExoPlayer.Builder(context, PlezyRenderersFactory(context)) + .setLooper(playbackThread.looper) + .build() + playerReference.set(player) + player.addListener(object : Player.Listener { + override fun onPlayerError(error: PlaybackException) { + errorReference.set(error) + completed.countDown() + } + + override fun onPlaybackStateChanged(playbackState: Int) { + if (playbackState == Player.STATE_ENDED) { + completed.countDown() + } + } + }) + val source = ProgressiveMediaSource.Factory( + DefaultDataSource.Factory(context), + ffmpegOnlyExtractors + ).createMediaSource(MediaItem.fromUri(Uri.fromFile(fixtureFile))) + player.setMediaSource(source) + player.prepare() + player.play() + } catch (error: Throwable) { + errorReference.set(error) + completed.countDown() + } + } + + val finished = completed.await(20, TimeUnit.SECONDS) + val released = CountDownLatch(1) + handler.post { + playerReference.get()?.release() + playbackThread.quitSafely() + released.countDown() + } + val teardownFinished = released.await(5, TimeUnit.SECONDS) + playbackThread.join(5_000) + val fixtureDeleted = fixtureFile.delete() + + assertTrue("Player teardown timed out for $fixture", teardownFinished) + assertTrue("Playback timed out for $fixture", finished) + assertNull("Playback failed for $fixture", errorReference.get()) + assertTrue("Fixture cleanup failed for $fixture", fixtureDeleted) + } + + private fun copyFixture(instrumentationContext: Context, targetContext: Context, fixture: String): File { + val output = File.createTempFile("ffmpeg-csd-fixture-", null, targetContext.cacheDir) + instrumentationContext.assets.open(fixture).use { input -> + output.outputStream().use(input::copyTo) + } + return output + } +} diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsd.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsd.kt new file mode 100644 index 000000000..3d8084278 --- /dev/null +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsd.kt @@ -0,0 +1,134 @@ +package com.edde746.plezy.exoplayer + +import androidx.annotation.OptIn +import androidx.media3.common.MimeTypes +import androidx.media3.common.ParserException +import androidx.media3.common.util.UnstableApi +import androidx.media3.container.OpusUtil + +/** + * Shapes ffmpeg codec extradata into the per-codec initializationData layout + * media3's audio decode paths consume. MediaCodecAudioRenderer maps the list + * entries verbatim to MediaCodec csd-0/1/2 and the bundled FfmpegAudioDecoder + * rebuilds avcodec extradata from the same entries, so both expect the shapes + * media3's own extractors emit — not raw ffmpeg extradata: + * + * - Opus: [OpusHead, codec delay ns, seek pre-roll ns]. Android's Opus + * decoders consume their first three input buffers as exactly this + * configuration; given only OpusHead they read the first two real packets + * as delay/pre-roll and then silently discard every decoded sample, which + * stalled the audio clock at 0 until the MPV fallback fired (#2088). + * - Vorbis: [identification header, setup header], split out of the + * Xiph-laced blob the way MatroskaExtractor's parseVorbisCodecPrivate does. + * - FLAC: the "fLaC"-marked STREAMINFO stream MediaCodec documents; ffmpeg + * surfaces a bare or block-headed STREAMINFO depending on the container. + * + * Everything else (AAC ASC, ALAC cookie, passthrough codecs) already matches + * media3's single-entry shape and passes through. MP3's WAVEFORMATEX blob is + * dropped on the JNI side before it reaches this shaping. + */ +@OptIn(UnstableApi::class) +internal object FfmpegAudioCsd { + + // RFC 7845 section 5.1: the identification header is at least 19 bytes; + // OpusUtil reads the pre-skip field at offsets 10-11. + private const val OPUS_HEADER_MIN_BYTES = 19 + + private val FLAC_STREAM_MARKER = byteArrayOf(0x66, 0x4C, 0x61, 0x43) // "fLaC" + private const val FLAC_STREAM_INFO_SIZE = 34 + private const val FLAC_BLOCK_HEADER_SIZE = 4 + private const val FLAC_LAST_BLOCK_FLAG = 0x80 + + fun initializationData(mime: String, extradata: ByteArray): List = when (mime) { + MimeTypes.AUDIO_OPUS -> opus(extradata) + MimeTypes.AUDIO_VORBIS -> vorbis(extradata) + MimeTypes.AUDIO_FLAC -> flac(extradata) + else -> listOf(extradata) + } + + private fun opus(extradata: ByteArray): List { + if (extradata.size < OPUS_HEADER_MIN_BYTES) { + throw malformed("Opus extradata is not an identification header: ${extradata.size} bytes") + } + // [OpusHead, pre-skip ns, 80 ms default seek pre-roll ns] — the same + // shape media3's Matroska/Ogg/MP4 extractors publish. + return OpusUtil.buildInitializationData(extradata) + } + + /** + * Splits the Xiph-laced header blob (identification, comment, setup) into + * media3's [identification header, setup header] pair, mirroring media3 + * 1.11.0 MatroskaExtractor.parseVorbisCodecPrivate. + */ + private fun vorbis(extradata: ByteArray): List { + try { + if (extradata[0].toInt() != 0x02) throw malformed("Vorbis extradata is not Xiph-laced headers") + var offset = 1 + var idHeaderLength = 0 + while (extradata[offset].toInt() and 0xFF == 0xFF) { + idHeaderLength += 0xFF + offset++ + } + idHeaderLength += extradata[offset++].toInt() and 0xFF + var commentHeaderLength = 0 + while (extradata[offset].toInt() and 0xFF == 0xFF) { + commentHeaderLength += 0xFF + offset++ + } + commentHeaderLength += extradata[offset++].toInt() and 0xFF + + if (extradata[offset].toInt() != 0x01) throw malformed("Vorbis identification header not found") + val idHeader = extradata.copyOfRange(offset, offset + idHeaderLength) + offset += idHeaderLength + if (extradata[offset].toInt() != 0x03) throw malformed("Vorbis comment header not found") + offset += commentHeaderLength + if (extradata[offset].toInt() != 0x05) throw malformed("Vorbis setup header not found") + return listOf(idHeader, extradata.copyOfRange(offset, extradata.size)) + } catch (e: IndexOutOfBoundsException) { + throw malformed("Vorbis extradata truncated") + } + } + + /** + * Normalizes to the "fLaC"-marked metadata stream. MediaCodec's decoder + * feeds csd-0 straight into libFLAC, which requires the stream marker; + * ffmpeg's canonical FLAC extradata is the bare 34-byte STREAMINFO, and + * some demuxers keep the 4-byte metadata block header in front of it. + */ + private fun flac(extradata: ByteArray): List { + if (startsWithFlacMarker(extradata)) return listOf(extradata) + if (extradata.size == FLAC_STREAM_INFO_SIZE) { + val blockHeader = byteArrayOf( + (FLAC_LAST_BLOCK_FLAG or 0).toByte(), // last-metadata-block, type 0 = STREAMINFO + 0x00, + 0x00, + FLAC_STREAM_INFO_SIZE.toByte() + ) + return listOf(FLAC_STREAM_MARKER + blockHeader + extradata) + } + val declaredLength = if (extradata.size >= FLAC_BLOCK_HEADER_SIZE) { + ((extradata[1].toInt() and 0xFF) shl 16) or + ((extradata[2].toInt() and 0xFF) shl 8) or + (extradata[3].toInt() and 0xFF) + } else { + -1 + } + if (extradata[0].toInt() and 0x7F == 0 && + declaredLength == FLAC_STREAM_INFO_SIZE && + extradata.size >= FLAC_BLOCK_HEADER_SIZE + FLAC_STREAM_INFO_SIZE + ) { + return listOf(FLAC_STREAM_MARKER + extradata) + } + throw malformed("FLAC extradata is not a STREAMINFO block: ${extradata.size} bytes") + } + + private fun startsWithFlacMarker(extradata: ByteArray): Boolean { + if (extradata.size < FLAC_STREAM_MARKER.size) return false + for (i in FLAC_STREAM_MARKER.indices) { + if (extradata[i] != FLAC_STREAM_MARKER[i]) return false + } + return true + } + + private fun malformed(message: String): ParserException = ParserException.createForMalformedContainer(message, null) +} diff --git a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractor.kt b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractor.kt index bf97518c2..a8d201036 100644 --- a/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractor.kt +++ b/android/app/src/main/kotlin/com/edde746/plezy/exoplayer/FfmpegExtractor.kt @@ -336,7 +336,13 @@ internal class FfmpegExtractor private constructor( val extradata = FfmpegDemuxerJni.nativeStreamExtradata(index) if (extradata != null) { - builder.setInitializationData(listOf(extradata)) + // Audio csd is codec-specific in media3 (see FfmpegAudioCsd); raw + // extradata is only correct for video (already Annex-B via the JNI + // bitstream filter) and text tracks. + builder.setInitializationData( + if (trackType == C.TRACK_TYPE_AUDIO) FfmpegAudioCsd.initializationData(mime, extradata) + else listOf(extradata) + ) } when (trackType) { diff --git a/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsdTest.kt b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsdTest.kt new file mode 100644 index 000000000..969ae520f --- /dev/null +++ b/android/app/src/test/kotlin/com/edde746/plezy/exoplayer/FfmpegAudioCsdTest.kt @@ -0,0 +1,138 @@ +package com.edde746.plezy.exoplayer + +import androidx.media3.common.MimeTypes +import androidx.media3.common.ParserException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Assert.assertThrows +import org.junit.Test + +/** + * Spec for the per-codec initializationData shapes the ffmpeg demuxer must + * publish so MediaCodec csd-0/1/2 and FfmpegAudioDecoder's extradata + * reconstruction both see what media3's own extractors emit. Regression + * coverage for #2088: Opus with a bare OpusHead csd made Android's decoders + * eat the first two real packets as configuration and discard all output. + */ +class FfmpegAudioCsdTest { + + private fun opusHead(preSkip: Int): ByteArray { + val header = ByteArray(19) + "OpusHead".toByteArray(Charsets.US_ASCII).copyInto(header) + header[8] = 1 // version + header[9] = 2 // channel count + header[10] = (preSkip and 0xFF).toByte() + header[11] = (preSkip ushr 8).toByte() + return header + } + + private fun nativeOrderLong(bytes: ByteArray): Long = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).long + + @Test + fun opusBuildsThreeCsdBuffers() { + val header = opusHead(preSkip = 312) + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_OPUS, header) + assertEquals(3, csd.size) + assertArrayEquals(header, csd[0]) + // Pre-skip converted to nanoseconds at the Opus 48 kHz clock. + assertEquals(312L * 1_000_000_000L / 48_000L, nativeOrderLong(csd[1])) + // media3's default seek pre-roll: 3840 samples = 80 ms. + assertEquals(80_000_000L, nativeOrderLong(csd[2])) + } + + @Test + fun opusRejectsTruncatedHeader() { + assertThrows(ParserException::class.java) { + FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_OPUS, ByteArray(11)) + } + } + + private fun xiphLace(length: Int): ByteArray { + val bytes = ArrayList() + var remaining = length + while (remaining >= 0xFF) { + bytes.add(0xFF.toByte()) + remaining -= 0xFF + } + bytes.add(remaining.toByte()) + return bytes.toByteArray() + } + + private fun vorbisHeader(type: Int, payloadLength: Int): ByteArray = + byteArrayOf(type.toByte()) + "vorbis".toByteArray(Charsets.US_ASCII) + ByteArray(payloadLength) + + @Test + fun vorbisSplitsLacedHeadersIntoIdentificationAndSetup() { + val id = vorbisHeader(0x01, payloadLength = 23) // 30 bytes, the real identification header size + val comment = vorbisHeader(0x03, payloadLength = 300) // > 255 exercises the 0xFF lacing + val setup = vorbisHeader(0x05, payloadLength = 40) + val extradata = byteArrayOf(0x02) + xiphLace(id.size) + xiphLace(comment.size) + id + comment + setup + + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, extradata) + + assertEquals(2, csd.size) + assertArrayEquals(id, csd[0]) + assertArrayEquals(setup, csd[1]) + } + + @Test + fun vorbisRejectsNonLacedExtradata() { + assertThrows(ParserException::class.java) { + FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, byteArrayOf(0x01, 0x02, 0x03)) + } + } + + @Test + fun vorbisRejectsTruncatedExtradata() { + val id = vorbisHeader(0x01, payloadLength = 23) + val truncated = byteArrayOf(0x02) + xiphLace(id.size) + xiphLace(64) + id // headers end early + assertThrows(ParserException::class.java) { + FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_VORBIS, truncated) + } + } + + @Test + fun flacWrapsBareStreamInfoInMarkedStream() { + val streamInfo = ByteArray(34) { it.toByte() } + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, streamInfo) + assertEquals(1, csd.size) + assertArrayEquals( + "fLaC".toByteArray(Charsets.US_ASCII) + byteArrayOf(0x80.toByte(), 0x00, 0x00, 34) + streamInfo, + csd[0] + ) + } + + @Test + fun flacPrependsMarkerToBlockHeadedStreamInfo() { + val block = byteArrayOf(0x00, 0x00, 0x00, 34) + ByteArray(34) { it.toByte() } + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, block) + assertEquals(1, csd.size) + assertArrayEquals("fLaC".toByteArray(Charsets.US_ASCII) + block, csd[0]) + } + + @Test + fun flacKeepsMarkedStreamVerbatim() { + val marked = "fLaC".toByteArray(Charsets.US_ASCII) + byteArrayOf(0x80.toByte(), 0x00, 0x00, 34) + ByteArray(34) + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, marked) + assertEquals(1, csd.size) + assertSame(marked, csd[0]) + } + + @Test + fun flacRejectsUnrecognizedExtradata() { + assertThrows(ParserException::class.java) { + FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_FLAC, ByteArray(10)) + } + } + + @Test + fun otherCodecsPassExtradataThroughAsSingleEntry() { + val asc = byteArrayOf(0x12, 0x10) // AAC-LC 44.1 kHz stereo AudioSpecificConfig + val csd = FfmpegAudioCsd.initializationData(MimeTypes.AUDIO_AAC, asc) + assertEquals(1, csd.size) + assertSame(asc, csd[0]) + } +}