fix(subtitles): render zlib-compressed embedded MKV subtitles on the ExoPlayer path

Embedded subtitles on mkvmerge-muxed files with compressed subtitle tracks
(the anime-release convention, reported against AV1 encodes) never rendered:
the track selected fine but nothing ever appeared on screen.

Two defects compounded. ContentCompAlgo 0 (zlib) is the Matroska default
value, so mkvmerge omits the element and the explicit-value detection never
engaged - media3 ignores an empty ContentCompression and silently emits
compressed samples. And for text subtitle tracks TrackOutput-level inflation
can never work: MatroskaExtractor prepends the plaintext timecode prefix to
the still-compressed payload and truncates the sample at the first NUL byte
before any TrackOutput runs, while AssTrackOutput feeds that same internal
buffer to libass, which then parses zero events and reports every frame as a
changed empty render.

Treat the presence of ContentCompression as zlib until an explicit
ContentCompAlgo says otherwise, and for SRT/ASS/SSA/VTT tracks inflate the
block frame payload before the parent's subtitle sample assembly, leaving the
TrackOutput wrapper to the formats it can actually handle. Blocks that are
laced, corrupt, or over-bound pass through byte-identical. The detection fix
also makes the existing whole-sample TrackOutput inflation engage for
PGS/VobSub zlib tracks, which media3 does not truncate.

Verified with mkvmerge zlib/plain fixture extraction tests and end-to-end on
an API 36 emulator against a Jellyfin direct-play mux mirroring the reported
file.

close #2023
close #2022
This commit is contained in:
edde746
2026-08-19 12:34:40 +02:00
parent 4a1111f435
commit 761303de11
5 changed files with 436 additions and 11 deletions
@@ -1,6 +1,9 @@
package com.edde746.plezy.exoplayer
import android.util.Log
import androidx.media3.common.C
import androidx.media3.common.DataReader
import androidx.media3.extractor.DefaultExtractorInput
import androidx.media3.extractor.ExtractorInput
import androidx.media3.extractor.ExtractorOutput
import androidx.media3.extractor.SeekMap
@@ -8,6 +11,9 @@ import androidx.media3.extractor.TrackOutput
import androidx.media3.extractor.text.SubtitleParser
import com.edde746.plezy.libass.media.AssHandler
import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
import java.io.EOFException
import java.util.zip.DataFormatException
import java.util.zip.Inflater
/**
* Extends AssMatroskaExtractor to add support for MKV quirks media3 rejects:
@@ -16,8 +22,19 @@ import com.edde746.plezy.libass.media.extractor.AssMatroskaExtractor
* stripping). This subclass intercepts the compression algorithm during track
* header parsing:
* - Tells the parent it's header stripping (algo 3) to avoid the ParserException
* - Wraps TrackOutputs with ZlibInflatingTrackOutput to decompress per-sample data
* - Skips ContentCompSettings for zlib tracks (not applicable)
* - For text subtitle tracks, inflates the block payload *before* the parent
* parses it (see below). For every other zlib track, wraps TrackOutputs with
* ZlibInflatingTrackOutput to decompress per-sample data.
*
* Text subtitle tracks (SRT/ASS/SSA/VTT) cannot be inflated at the TrackOutput
* level: MatroskaExtractor rewrites their samples in-place before any TrackOutput
* runs — it prepends a plaintext timecode prefix ("Dialogue: 0:00:00:00,…,") to
* the still-compressed payload and truncates the sample at the first NUL byte,
* which deflate streams routinely contain (#2023). AssTrackOutput additionally
* feeds that same internal buffer straight to libass. So for zlib text tracks the
* frame payload is inflated at the block level, before the parent's sample
* assembly, and the TrackOutput wrapper is left inactive.
*
* LOAS/LATM AAC as A_MS/ACM — media3 sets audio/x-unknown for non-PCM ACM
* tracks (silent playback). Detected tracks are wrapped with LatmTrackOutput,
@@ -36,15 +53,43 @@ class ZlibMatroskaExtractor(
private const val ID_TRACK_ENTRY = 0xAE
private const val ID_CONTENT_COMPRESSION_ALGORITHM = 0x4254
private const val ID_CONTENT_COMPRESSION_SETTINGS = 0x4255
private const val ID_CONTENT_COMPRESSION = 0x5034
private const val ID_SIMPLE_BLOCK = 0xA3
private const val ID_BLOCK = 0xA1
/**
* Codec IDs whose samples MatroskaExtractor rewrites in-place (timecode prefix
* plus NUL truncation) before any TrackOutput runs — the authority is the
* prefix list in MatroskaExtractor.writeSampleData.
*/
private val TEXT_SUBTITLE_CODEC_IDS = setOf("S_TEXT/UTF8", "S_TEXT/ASS", "S_TEXT/SSA", "S_TEXT/WEBVTT")
/** Subtitle blocks are tiny; anything above this passes through untouched. */
private const val MAX_TEXT_BLOCK_BYTES = 16 * 1024 * 1024
}
private var zlibOutput: ZlibExtractorOutputWrapper? = null
private var latmOutput: LatmExtractorOutputWrapper? = null
private var currentTrackUsesZlib = false
/** Track numbers whose blocks are inflated before the parent parses them. */
private val zlibTextTrackNumbers = mutableSetOf<Int>()
private val blockInflater = Inflater()
private var blockBuf = ByteArray(0)
private val peekBuf = ByteArray(8)
override fun startMasterElement(id: Int, contentPosition: Long, contentSize: Long) {
super.startMasterElement(id, contentPosition, contentSize)
// ContentCompAlgo DEFAULTS to 0 (zlib), so mkvmerge omits the element for
// zlib tracks entirely — the mere presence of ContentCompression means zlib
// until an explicit ContentCompAlgo says otherwise. media3 ignores an empty
// ContentCompression and would silently emit compressed samples.
if (id == ID_CONTENT_COMPRESSION) {
currentTrackUsesZlib = true
Log.i(TAG, "Track has ContentCompression, assuming ContentCompAlgo 0 (zlib) until told otherwise")
}
// After super installs AssSubtitleExtractorOutput, wrap it with our zlib +
// LATM layers (zlib outermost so inflation runs before LATM parsing).
if (id == ID_SEGMENT && zlibOutput == null) {
@@ -59,12 +104,17 @@ class ZlibMatroskaExtractor(
}
override fun integerElement(id: Int, value: Long) {
if (id == ID_CONTENT_COMPRESSION_ALGORITHM && value == 0L) {
currentTrackUsesZlib = true
Log.i(TAG, "Track uses ContentCompAlgo 0 (zlib), will inflate samples")
// Tell parent it's header stripping (algo 3) to avoid ParserException
super.integerElement(id, 3)
return
if (id == ID_CONTENT_COMPRESSION_ALGORITHM) {
if (value == 0L) {
currentTrackUsesZlib = true
Log.i(TAG, "Track uses explicit ContentCompAlgo 0 (zlib), will inflate samples")
// Tell parent it's header stripping (algo 3) to avoid ParserException
super.integerElement(id, 3)
return
}
// Explicit non-zlib algorithm: header stripping (3) is handled by the
// parent; anything else makes the parent throw, matching stock behavior.
currentTrackUsesZlib = false
}
super.integerElement(id, value)
}
@@ -76,27 +126,47 @@ class ZlibMatroskaExtractor(
input.skipFully(contentSize)
return
}
if ((id == ID_SIMPLE_BLOCK || id == ID_BLOCK) &&
zlibTextTrackNumbers.isNotEmpty() &&
contentSize in MIN_BLOCK_BYTES..MAX_TEXT_BLOCK_BYTES &&
peekBlockTrackNumber(input) in zlibTextTrackNumbers
) {
inflateTextBlock(id, contentSize, input)
return
}
super.binaryElement(id, contentSize, input)
}
override fun endMasterElement(id: Int) {
var zlibTextTrackNumber: Int? = null
if (id == ID_TRACK_ENTRY) {
// Must mark before super — the track output is created inside super's
// endMasterElement, and the x-unknown format must never reach the queue.
// Must inspect before super — the track output is created inside super's
// endMasterElement, and the current track is cleared afterwards.
val track = getCurrentTrack(id)
if (isLoasAcmTrack(track.codecId, track.codecPrivate)) {
Log.i(TAG, "Track ${track.number} is LOAS/LATM AAC, unwrapping to raw AAC")
latmOutput?.markNextTrackLatm()
}
if (currentTrackUsesZlib && track.codecId in TEXT_SUBTITLE_CODEC_IDS) {
zlibTextTrackNumber = track.number
}
}
val wasZlib = currentTrackUsesZlib
super.endMasterElement(id)
if (id == ID_TRACK_ENTRY && wasZlib) {
zlibOutput?.activateLast()
currentTrackUsesZlib = false
Log.i(TAG, "Activated zlib inflation for track")
if (zlibTextTrackNumber != null) {
// Text subtitle samples are rewritten inside the parent before any
// TrackOutput runs, so the TrackOutput wrapper stays inactive and the
// block payload is inflated in binaryElement instead.
zlibTextTrackNumbers.add(zlibTextTrackNumber)
Log.i(TAG, "Track $zlibTextTrackNumber is a zlib text subtitle track, inflating at block level")
} else {
zlibOutput?.activateLast()
Log.i(TAG, "Activated zlib inflation for track")
}
}
}
@@ -106,6 +176,64 @@ class ZlibMatroskaExtractor(
super.seek(position, timeUs)
}
/**
* Peeks the EBML varint at the block start — the block's track number — without
* consuming input. Returns null for malformed varints or truncated input; the
* parent then produces the canonical failure for the untouched stream.
*/
private fun peekBlockTrackNumber(input: ExtractorInput): Int? {
try {
input.peekFully(peekBuf, 0, 1)
val first = peekBuf[0].toInt() and 0xFF
if (first == 0) return null
val length = Integer.numberOfLeadingZeros(first) - 23
var value = (first and (0xFF ushr length)).toLong()
if (length > 1) {
input.peekFully(peekBuf, 1, length - 1)
for (i in 1 until length) {
value = (value shl 8) or (peekBuf[i].toLong() and 0xFF)
}
}
return if (value <= Int.MAX_VALUE) value.toInt() else null
} catch (_: EOFException) {
return null
} finally {
input.resetPeekPosition()
}
}
/**
* Buffers one text-subtitle block, inflates its frame payload, and hands the
* parent a block whose payload is plaintext. A block that cannot be rewritten
* (laced, corrupt, or over-bound) passes through byte-identical.
*/
private fun inflateTextBlock(id: Int, contentSize: Int, input: ExtractorInput) {
val basePosition = input.position
if (blockBuf.size < contentSize) blockBuf = ByteArray(maxOf(contentSize, blockBuf.size * 2))
input.readFully(blockBuf, 0, contentSize)
val rewritten = rewriteZlibTextBlock(blockBuf, contentSize, blockInflater)
if (rewritten == null) {
Log.w(TAG, "Passing zlib text block through uninflated (laced, corrupt, or over-bound)")
super.binaryElement(id, contentSize, bufferedInput(blockBuf, contentSize, basePosition))
} else {
super.binaryElement(id, rewritten.size, bufferedInput(rewritten, rewritten.size, basePosition))
}
}
private fun bufferedInput(data: ByteArray, limit: Int, position: Long): ExtractorInput = DefaultExtractorInput(ByteRangeDataReader(data, limit), position, position + limit)
private class ByteRangeDataReader(private val data: ByteArray, private val limit: Int) : DataReader {
private var position = 0
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
if (position == limit) return C.RESULT_END_OF_INPUT
val count = minOf(length, limit - position)
System.arraycopy(data, position, buffer, offset, count)
position += count
return count
}
}
/**
* ExtractorOutput wrapper that wraps all TrackOutputs with ZlibInflatingTrackOutput.
* Tracks are created inactive; activateLast() enables inflation for the most recently
@@ -138,3 +266,50 @@ class ZlibMatroskaExtractor(
override fun seekMap(seekMap: SeekMap) = delegate.seekMap(seekMap)
}
}
/** Smallest well-formed unlaced block: 1-byte varint + 2-byte timecode + flags + 1 payload byte. */
internal const val MIN_BLOCK_BYTES = 5
/**
* Rewrites one unlaced Matroska block whose frame payload is zlib-compressed:
* returns the unchanged block header followed by the inflated payload, or null
* when the block must pass through unchanged — lacing (never produced for text
* subtitle tracks), a truncated or corrupt deflate stream, or an inflated size
* beyond the bounds shared with [ZlibInflatingTrackOutput]'s hardening.
*/
internal fun rewriteZlibTextBlock(data: ByteArray, limit: Int, inflater: Inflater): ByteArray? {
if (limit < MIN_BLOCK_BYTES) return null
val first = data[0].toInt() and 0xFF
if (first == 0) return null // track-number varint longer than 8 bytes
val varintLength = Integer.numberOfLeadingZeros(first) - 23
val headerLength = varintLength + 3 // varint + 2-byte timecode + flags
if (limit <= headerLength) return null
if (data[headerLength - 1].toInt() and 0x06 != 0) return null // laced
val payloadLength = limit - headerLength
val ratioBound = maxOf(1024L * 1024, payloadLength.toLong() * 1024)
val maxInflatedBytes = 16 * 1024 * 1024
inflater.reset()
inflater.setInput(data, headerLength, payloadLength)
var buf = ByteArray(maxOf(4096, payloadLength * 4))
var written = 0
try {
while (true) {
if (written == buf.size) {
if (buf.size >= maxInflatedBytes) return null
buf = buf.copyOf(minOf(maxInflatedBytes, buf.size * 2))
}
val count = inflater.inflate(buf, written, buf.size - written)
written += count
if (written > ratioBound) return null
if (inflater.finished()) break
if (count == 0) return null // truncated stream or preset-dictionary request
}
} catch (_: DataFormatException) {
return null
}
return ByteArray(headerLength + written).also {
System.arraycopy(data, 0, it, 0, headerLength)
System.arraycopy(buf, 0, it, headerLength, written)
}
}
@@ -0,0 +1,80 @@
package com.edde746.plezy.exoplayer
import java.io.ByteArrayOutputStream
import java.util.zip.Deflater
import java.util.zip.DeflaterOutputStream
import java.util.zip.Inflater
import org.junit.Assert.assertArrayEquals
import org.junit.Assert.assertNull
import org.junit.Test
/**
* Unit spec for [rewriteZlibTextBlock]: the block header (track-number varint,
* timecode, flags) must survive byte-identically, the payload must inflate, and
* every block the rewriter cannot prove safe must pass through unchanged (null).
*/
class ZlibTextBlockTest {
private val inflater = Inflater()
private fun block(header: ByteArray, payload: ByteArray): ByteArray = header + payload
private fun deflate(bytes: ByteArray): ByteArray {
val target = ByteArrayOutputStream()
DeflaterOutputStream(target, Deflater()).use { it.write(bytes) }
return target.toByteArray()
}
// Track 3 (varint 0x83), relative timecode 0x0102, no lacing.
private val unlacedHeader = byteArrayOf(0x83.toByte(), 0x01, 0x02, 0x00)
@Test
fun inflatesUnlacedBlockPayloadAndPreservesHeader() {
val dialogue = "42,,Default,,0,0,0,,SUBTITLE LINE 1 (0:00:00.00)".toByteArray()
val data = block(unlacedHeader, deflate(dialogue))
val rewritten = rewriteZlibTextBlock(data, data.size, inflater)!!
assertArrayEquals(unlacedHeader + dialogue, rewritten)
}
@Test
fun twoByteTrackNumberVarintIsPreserved() {
val dialogue = "1,,Default,,0,0,0,,hello".toByteArray()
val header = byteArrayOf(0x41, 0x2A, 0x01, 0x02, 0x00) // varint 0x412A = track 298
val data = block(header, deflate(dialogue))
val rewritten = rewriteZlibTextBlock(data, data.size, inflater)!!
assertArrayEquals(header + dialogue, rewritten)
}
@Test
fun lacedBlockPassesThrough() {
val laced = byteArrayOf(0x83.toByte(), 0x01, 0x02, 0x06) // EBML lacing bits set
val data = block(laced, deflate("payload".toByteArray()))
assertNull(rewriteZlibTextBlock(data, data.size, inflater))
}
@Test
fun corruptStreamPassesThrough() {
val data = block(unlacedHeader, byteArrayOf(0x44, 0x69, 0x61, 0x6C)) // "Dial", not zlib
assertNull(rewriteZlibTextBlock(data, data.size, inflater))
}
@Test
fun truncatedStreamPassesThrough() {
val compressed = deflate("a longer payload that spans several deflate symbols".toByteArray())
val truncated = compressed.copyOf(compressed.size - 4)
val data = block(unlacedHeader, truncated)
assertNull(rewriteZlibTextBlock(data, data.size, inflater))
}
@Test
fun headerOnlyBlockPassesThrough() {
assertNull(rewriteZlibTextBlock(unlacedHeader, unlacedHeader.size, inflater))
}
}
@@ -0,0 +1,170 @@
package com.edde746.plezy.exoplayer
import androidx.media3.common.C
import androidx.media3.common.DataReader
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.ParsableByteArray
import androidx.media3.extractor.DefaultExtractorInput
import androidx.media3.extractor.Extractor
import androidx.media3.extractor.ExtractorOutput
import androidx.media3.extractor.PositionHolder
import androidx.media3.extractor.SeekMap
import androidx.media3.extractor.TrackOutput
import androidx.media3.extractor.text.CueDecoder
import androidx.media3.extractor.text.DefaultSubtitleParserFactory
import com.edde746.plezy.libass.media.AssHandler
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
import org.junit.runner.RunWith
import org.robolectric.RobolectricTestRunner
import org.robolectric.annotation.Config
/**
* Extracts the committed fixtures — one ASS subtitle track muxed by mkvmerge with
* `--compression 0:zlib` (ContentCompAlgo 0, the anime-release convention that
* broke #2023) and an identical uncompressed mux — and verifies the zlib block
* payloads are inflated *before* MatroskaExtractor's subtitle sample assembly.
*
* Regenerate with:
* mkvmerge -o zlib_ssa.mkv --compression 0:zlib repro.ass
* mkvmerge -o plain_ssa.mkv --compression 0:none repro.ass
* where repro.ass holds 45 two-second "SUBTITLE LINE N (H:MM:SS.00)" events.
*
* Robolectric provides real android.util/android.os implementations —
* MatroskaExtractor uses SparseArray and CueDecoder uses Parcel, both of which
* are no-op stubs on plain JVM.
*/
@RunWith(RobolectricTestRunner::class)
@Config(sdk = [34])
class ZlibTextSubtitleExtractionTest {
private class CapturedSample(val timeUs: Long, val data: ByteArray)
private class FakeTrackOutput : TrackOutput {
val formats = mutableListOf<androidx.media3.common.Format>()
val samples = mutableListOf<CapturedSample>()
private var buf = ByteArray(64 * 1024)
private var bufLen = 0
override fun format(format: androidx.media3.common.Format) {
formats.add(format)
}
override fun sampleData(input: DataReader, length: Int, allowEndOfInput: Boolean, sampleDataPart: Int): Int {
ensureCapacity(bufLen + length)
val read = input.read(buf, bufLen, length)
if (read > 0) bufLen += read
return read
}
override fun sampleData(data: ParsableByteArray, length: Int, sampleDataPart: Int) {
ensureCapacity(bufLen + length)
data.readBytes(buf, bufLen, length)
bufLen += length
}
override fun sampleMetadata(timeUs: Long, flags: Int, size: Int, offset: Int, cryptoData: TrackOutput.CryptoData?) {
val start = bufLen - offset - size
samples.add(CapturedSample(timeUs, buf.copyOfRange(start, start + size)))
if (offset == 0) bufLen = 0
}
private fun ensureCapacity(needed: Int) {
if (buf.size < needed) buf = buf.copyOf(maxOf(needed, buf.size * 2))
}
}
private class FakeExtractorOutput : ExtractorOutput {
val tracks = mutableMapOf<Int, FakeTrackOutput>()
override fun track(id: Int, type: Int): TrackOutput = tracks.getOrPut(id) { FakeTrackOutput() }
override fun endTracks() {}
override fun seekMap(seekMap: SeekMap) {}
}
private class ByteArrayDataReader(private val data: ByteArray) : DataReader {
var position = 0L
override fun read(buffer: ByteArray, offset: Int, length: Int): Int {
if (position >= data.size) return C.RESULT_END_OF_INPUT
val toRead = minOf(length, data.size - position.toInt())
System.arraycopy(data, position.toInt(), buffer, offset, toRead)
position += toRead
return toRead
}
}
private fun extract(resource: String): FakeExtractorOutput {
val data = checkNotNull(javaClass.getResourceAsStream(resource)) {
"fixture $resource missing from test resources"
}.use { it.readBytes() }
// DefaultSubtitleParserFactory (not AssSubtitleParserFactory) so the SSA track
// transcodes to decodable media3 cues instead of loading native libass.
val extractor = ZlibMatroskaExtractor(DefaultSubtitleParserFactory(), AssHandler())
val output = FakeExtractorOutput()
extractor.init(output)
val reader = ByteArrayDataReader(data)
var input = DefaultExtractorInput(reader, 0, data.size.toLong())
val seekPosition = PositionHolder()
while (true) {
when (extractor.read(input, seekPosition)) {
Extractor.RESULT_END_OF_INPUT -> return output
Extractor.RESULT_SEEK -> {
reader.position = seekPosition.position
input = DefaultExtractorInput(reader, seekPosition.position, data.size.toLong())
}
else -> {}
}
}
}
private class DecodedCue(val startTimeUs: Long, val durationUs: Long, val text: String)
private fun decodeCues(output: FakeExtractorOutput): List<DecodedCue> {
val track = output.tracks.values.single()
val format = track.formats.last()
assertEquals(MimeTypes.APPLICATION_MEDIA3_CUES, format.sampleMimeType)
assertEquals(MimeTypes.TEXT_SSA, format.codecs)
val decoder = CueDecoder()
return track.samples.map { sample ->
val cues = decoder.decode(sample.timeUs, sample.data, 0, sample.data.size)
DecodedCue(
cues.startTimeUs,
cues.durationUs,
cues.cues.joinToString("\n") { it.text.toString() }
)
}
}
@Test
fun zlibCompressedAssDialogueIsInflatedToParseableCues() {
val cues = decodeCues(extract("/zlib_ssa.mkv"))
assertEquals(45, cues.size)
assertEquals("SUBTITLE LINE 1 (0:00:00.00)", cues.first().text)
assertEquals(0L, cues.first().startTimeUs)
assertEquals(2_000_000L, cues.first().durationUs)
assertEquals("SUBTITLE LINE 45 (0:01:28.00)", cues.last().text)
assertEquals(88_000_000L, cues.last().startTimeUs)
cues.forEachIndexed { index, cue ->
assertEquals(index * 2_000_000L, cue.startTimeUs)
assertTrue("cue $index text: ${cue.text}", cue.text.startsWith("SUBTITLE LINE ${index + 1} "))
}
}
@Test
fun zlibAndPlainMuxesProduceIdenticalCues() {
val zlib = decodeCues(extract("/zlib_ssa.mkv"))
val plain = decodeCues(extract("/plain_ssa.mkv"))
assertEquals(plain.size, zlib.size)
plain.zip(zlib).forEachIndexed { index, (expected, actual) ->
assertEquals("startTimeUs of cue $index", expected.startTimeUs, actual.startTimeUs)
assertEquals("durationUs of cue $index", expected.durationUs, actual.durationUs)
assertEquals("text of cue $index", expected.text, actual.text)
}
}
}
Binary file not shown.
Binary file not shown.