fix(android): consolidate native media decoders

This commit is contained in:
edde746
2026-07-24 03:56:40 +02:00
parent fb45ff44f3
commit 54a002f9b7
26 changed files with 2101 additions and 65 deletions
@@ -0,0 +1,294 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.ffmpeg;
import static com.google.common.base.Preconditions.checkNotNull;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.decoder.SimpleDecoder;
import androidx.media3.decoder.SimpleDecoderOutputBuffer;
import java.nio.ByteBuffer;
import java.util.List;
/** Media3 audio decoder backed by the FFmpeg libraries packaged by libmpv. */
final class FfmpegAudioDecoder
extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, FfmpegDecoderException> {
private static final int INITIAL_OUTPUT_BUFFER_SIZE_16BIT = 65535;
private static final int INITIAL_OUTPUT_BUFFER_SIZE_32BIT = INITIAL_OUTPUT_BUFFER_SIZE_16BIT * 2;
private static final int AUDIO_DECODER_ERROR_INVALID_DATA = -1;
private static final int AUDIO_DECODER_ERROR_OTHER = -2;
private static final byte[] FLAC_STREAM_MARKER = {'f', 'L', 'a', 'C'};
private static final int FLAC_METADATA_TYPE_STREAM_INFO = 0;
private static final int FLAC_METADATA_BLOCK_HEADER_SIZE = 4;
private static final int FLAC_STREAM_INFO_DATA_SIZE = 34;
private final String codecName;
@Nullable private final byte[] extraData;
private final @C.PcmEncoding int encoding;
private int outputBufferSize;
private long nativeContext;
private boolean hasOutputFormat;
private volatile int channelCount;
private volatile int sampleRate;
FfmpegAudioDecoder(
Format format,
int numInputBuffers,
int numOutputBuffers,
int initialInputBufferSize,
boolean outputFloat)
throws FfmpegDecoderException {
super(new DecoderInputBuffer[numInputBuffers], new SimpleDecoderOutputBuffer[numOutputBuffers]);
if (!FfmpegLibrary.isAvailable()) {
throw new FfmpegDecoderException("Failed to load decoder native libraries.");
}
String mimeType = checkNotNull(format.sampleMimeType);
codecName = checkNotNull(FfmpegLibrary.getCodecName(mimeType));
extraData = getExtraData(mimeType, format.initializationData);
encoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT;
outputBufferSize =
outputFloat ? INITIAL_OUTPUT_BUFFER_SIZE_32BIT : INITIAL_OUTPUT_BUFFER_SIZE_16BIT;
nativeContext =
ffmpegInitialize(codecName, extraData, outputFloat, format.sampleRate, format.channelCount);
if (nativeContext == 0) {
throw new FfmpegDecoderException("Initialization failed.");
}
setInitialInputBufferSize(initialInputBufferSize);
}
@Override
public String getName() {
return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName;
}
@Override
protected DecoderInputBuffer createInputBuffer() {
return new DecoderInputBuffer(
DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT,
FfmpegLibrary.getInputBufferPaddingSize());
}
@Override
protected SimpleDecoderOutputBuffer createOutputBuffer() {
return new SimpleDecoderOutputBuffer(this::releaseOutputBuffer);
}
@Override
protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) {
return new FfmpegDecoderException("Unexpected decode error", error);
}
@Override
@Nullable
protected FfmpegDecoderException decode(
DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) {
if (reset) {
nativeContext = ffmpegReset(nativeContext, extraData);
if (nativeContext == 0) {
return new FfmpegDecoderException("Error resetting (see logcat).");
}
}
ByteBuffer inputData = Util.castNonNull(inputBuffer.data);
int inputSize = inputData.limit();
ByteBuffer outputData = outputBuffer.init(inputBuffer.timeUs, outputBufferSize);
int result =
ffmpegDecode(
nativeContext, inputData, inputSize, outputBuffer, outputData, outputBufferSize);
if (result == AUDIO_DECODER_ERROR_OTHER) {
return new FfmpegDecoderException("Error decoding (see logcat).");
} else if (result == AUDIO_DECODER_ERROR_INVALID_DATA) {
outputBuffer.shouldBeSkipped = true;
return null;
} else if (result == 0) {
outputBuffer.shouldBeSkipped = true;
return null;
}
if (!hasOutputFormat) {
channelCount = ffmpegGetChannelCount(nativeContext);
sampleRate = ffmpegGetSampleRate(nativeContext);
if (sampleRate == 0 && "alac".equals(codecName)) {
checkNotNull(extraData);
ParsableByteArray parsableExtraData = new ParsableByteArray(extraData);
parsableExtraData.setPosition(extraData.length - 4);
sampleRate = parsableExtraData.readUnsignedIntToInt();
}
hasOutputFormat = true;
}
outputData = checkNotNull(outputBuffer.data);
outputData.position(0);
outputData.limit(result);
return null;
}
@SuppressWarnings("unused")
private ByteBuffer growOutputBuffer(SimpleDecoderOutputBuffer outputBuffer, int requiredSize) {
outputBufferSize = requiredSize;
return outputBuffer.grow(requiredSize);
}
@Override
public void release() {
super.release();
ffmpegRelease(nativeContext);
nativeContext = 0;
}
int getChannelCount() {
return channelCount;
}
int getSampleRate() {
return sampleRate;
}
@C.PcmEncoding
int getEncoding() {
return encoding;
}
@Nullable
private static byte[] getExtraData(String mimeType, List<byte[]> initializationData) {
switch (mimeType) {
case MimeTypes.AUDIO_AAC:
case MimeTypes.AUDIO_OPUS:
return initializationData.get(0);
case MimeTypes.AUDIO_ALAC:
return getAlacExtraData(initializationData);
case MimeTypes.AUDIO_VORBIS:
return getVorbisExtraData(initializationData);
case MimeTypes.AUDIO_FLAC:
return getFlacExtraData(initializationData);
default:
return null;
}
}
private static byte[] getAlacExtraData(List<byte[]> initializationData) {
byte[] magicCookie = initializationData.get(0);
int alacAtomLength = 12 + magicCookie.length;
ByteBuffer alacAtom = ByteBuffer.allocate(alacAtomLength);
alacAtom.putInt(alacAtomLength);
alacAtom.putInt(0x616c6163);
alacAtom.putInt(0);
alacAtom.put(magicCookie, 0, magicCookie.length);
return alacAtom.array();
}
private static byte[] getVorbisExtraData(List<byte[]> initializationData) {
byte[] header0 = initializationData.get(0);
byte[] header1 = initializationData.get(1);
byte[] extraData = new byte[header0.length + header1.length + 6];
extraData[0] = (byte) (header0.length >> 8);
extraData[1] = (byte) (header0.length & 0xFF);
System.arraycopy(header0, 0, extraData, 2, header0.length);
extraData[header0.length + 2] = 0;
extraData[header0.length + 3] = 0;
extraData[header0.length + 4] = (byte) (header1.length >> 8);
extraData[header0.length + 5] = (byte) (header1.length & 0xFF);
System.arraycopy(header1, 0, extraData, header0.length + 6, header1.length);
return extraData;
}
@Nullable
private static byte[] getFlacExtraData(List<byte[]> initializationData) {
for (int i = 0; i < initializationData.size(); i++) {
@Nullable byte[] streamInfo = extractFlacStreamInfo(initializationData.get(i));
if (streamInfo != null) {
return streamInfo;
}
}
return null;
}
@Nullable
private static byte[] extractFlacStreamInfo(byte[] data) {
int offset = 0;
if (arrayStartsWith(data, FLAC_STREAM_MARKER)) {
offset = FLAC_STREAM_MARKER.length;
}
if (data.length - offset == FLAC_STREAM_INFO_DATA_SIZE) {
byte[] streamInfo = new byte[FLAC_STREAM_INFO_DATA_SIZE];
System.arraycopy(data, offset, streamInfo, 0, FLAC_STREAM_INFO_DATA_SIZE);
return streamInfo;
}
if (data.length >= offset + FLAC_METADATA_BLOCK_HEADER_SIZE) {
int type = data[offset] & 0x7F;
int length =
((data[offset + 1] & 0xFF) << 16)
| ((data[offset + 2] & 0xFF) << 8)
| (data[offset + 3] & 0xFF);
if (type == FLAC_METADATA_TYPE_STREAM_INFO
&& length == FLAC_STREAM_INFO_DATA_SIZE
&& data.length >= offset + FLAC_METADATA_BLOCK_HEADER_SIZE + FLAC_STREAM_INFO_DATA_SIZE) {
byte[] streamInfo = new byte[FLAC_STREAM_INFO_DATA_SIZE];
System.arraycopy(
data,
offset + FLAC_METADATA_BLOCK_HEADER_SIZE,
streamInfo,
0,
FLAC_STREAM_INFO_DATA_SIZE);
return streamInfo;
}
}
return null;
}
private static boolean arrayStartsWith(byte[] data, byte[] prefix) {
if (data.length < prefix.length) {
return false;
}
for (int i = 0; i < prefix.length; i++) {
if (data[i] != prefix[i]) {
return false;
}
}
return true;
}
private native long ffmpegInitialize(
String codecName,
@Nullable byte[] extraData,
boolean outputFloat,
int rawSampleRate,
int rawChannelCount);
private native int ffmpegDecode(
long context,
ByteBuffer inputData,
int inputSize,
SimpleDecoderOutputBuffer decoderOutputBuffer,
ByteBuffer outputData,
int outputSize);
private native int ffmpegGetChannelCount(long context);
private native int ffmpegGetSampleRate(long context);
private native long ffmpegReset(long context, @Nullable byte[] extraData);
private native void ffmpegRelease(long context);
}
@@ -0,0 +1,126 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.ffmpeg;
import static androidx.media3.exoplayer.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY;
import static androidx.media3.exoplayer.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING;
import static androidx.media3.exoplayer.audio.AudioSink.SINK_FORMAT_UNSUPPORTED;
import static com.google.common.base.Preconditions.checkNotNull;
import android.os.Handler;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.TraceUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.CryptoConfig;
import androidx.media3.exoplayer.audio.AudioRendererEventListener;
import androidx.media3.exoplayer.audio.AudioSink;
import androidx.media3.exoplayer.audio.AudioSink.SinkFormatSupport;
import androidx.media3.exoplayer.audio.DecoderAudioRenderer;
/** Decodes and renders audio using the FFmpeg libraries shared with libmpv. */
@UnstableApi
public final class FfmpegAudioRenderer extends DecoderAudioRenderer<FfmpegAudioDecoder> {
private static final String TAG = "FfmpegAudioRenderer";
private static final int NUM_BUFFERS = 16;
private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6;
public FfmpegAudioRenderer(
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioSink audioSink) {
super(eventHandler, eventListener, audioSink);
}
@Override
public String getName() {
return TAG;
}
@Override
protected @C.FormatSupport int supportsFormatInternal(Format format) {
String mimeType = checkNotNull(format.sampleMimeType);
if (!FfmpegLibrary.isAvailable() || !MimeTypes.isAudio(mimeType)) {
return C.FORMAT_UNSUPPORTED_TYPE;
} else if (!FfmpegLibrary.supportsFormat(mimeType)
|| (!sinkSupportsFormat(format, C.ENCODING_PCM_16BIT)
&& !sinkSupportsFormat(format, C.ENCODING_PCM_FLOAT))) {
return C.FORMAT_UNSUPPORTED_SUBTYPE;
} else if (format.cryptoType != C.CRYPTO_TYPE_NONE) {
return C.FORMAT_UNSUPPORTED_DRM;
} else {
return C.FORMAT_HANDLED;
}
}
@Override
public @AdaptiveSupport int supportsMixedMimeTypeAdaptation() {
return ADAPTIVE_NOT_SEAMLESS;
}
@Override
protected FfmpegAudioDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig)
throws FfmpegDecoderException {
TraceUtil.beginSection("createFfmpegAudioDecoder");
int initialInputBufferSize =
format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE;
FfmpegAudioDecoder decoder =
new FfmpegAudioDecoder(
format, NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, shouldOutputFloat(format));
TraceUtil.endSection();
return decoder;
}
@Override
protected Format getOutputFormat(FfmpegAudioDecoder decoder) {
checkNotNull(decoder);
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(decoder.getChannelCount())
.setSampleRate(decoder.getSampleRate())
.setPcmEncoding(decoder.getEncoding())
.build();
}
private boolean sinkSupportsFormat(Format inputFormat, @C.PcmEncoding int pcmEncoding) {
return sinkSupportsFormat(
Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate));
}
private boolean shouldOutputFloat(Format inputFormat) {
if (!sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT)) {
return true;
}
@SinkFormatSupport
int formatSupport =
getSinkFormatSupport(
Util.getPcmFormat(
C.ENCODING_PCM_FLOAT, inputFormat.channelCount, inputFormat.sampleRate));
switch (formatSupport) {
case SINK_FORMAT_SUPPORTED_DIRECTLY:
return !MimeTypes.AUDIO_AC3.equals(inputFormat.sampleMimeType);
case SINK_FORMAT_UNSUPPORTED:
case SINK_FORMAT_SUPPORTED_WITH_TRANSCODING:
default:
return false;
}
}
}
@@ -0,0 +1,30 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.ffmpeg;
import androidx.media3.decoder.DecoderException;
/** Thrown when an FFmpeg decoder error occurs. */
final class FfmpegDecoderException extends DecoderException {
FfmpegDecoderException(String message) {
super(message);
}
FfmpegDecoderException(String message, Throwable cause) {
super(message, cause);
}
}
@@ -0,0 +1,126 @@
/*
* Copyright (C) 2016 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.decoder.ffmpeg;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.LibraryLoader;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.UnstableApi;
/** Configures and queries the FFmpeg libraries shared with libmpv. */
@UnstableApi
final class FfmpegLibrary {
private static final String TAG = "FfmpegLibrary";
private static final LibraryLoader LOADER =
new LibraryLoader("ffmpegJNI") {
@Override
protected void loadLibrary(String name) {
System.loadLibrary(name);
}
};
@Nullable private static String version;
private static int inputBufferPaddingSize = C.LENGTH_UNSET;
private FfmpegLibrary() {}
/** Returns whether the JNI adapter and libmpv's FFmpeg libraries can be loaded. */
static boolean isAvailable() {
return LOADER.isAvailable();
}
/** Returns the linked FFmpeg version. The native libraries must be available. */
static String getVersion() {
String cachedVersion = version;
if (cachedVersion == null) {
cachedVersion = ffmpegGetVersion();
version = cachedVersion;
}
return cachedVersion;
}
/** Returns the required FFmpeg input-buffer padding. The native libraries must be available. */
static int getInputBufferPaddingSize() {
if (inputBufferPaddingSize == C.LENGTH_UNSET) {
inputBufferPaddingSize = ffmpegGetInputBufferPaddingSize();
}
return inputBufferPaddingSize;
}
/** Returns whether the linked FFmpeg build supports the MIME type. */
static boolean supportsFormat(String mimeType) {
@Nullable String codecName = getCodecName(mimeType);
if (codecName == null) {
return false;
}
if (!ffmpegHasDecoder(codecName)) {
Log.w(TAG, "No " + codecName + " decoder available in libmpv's FFmpeg build.");
return false;
}
return true;
}
/** Returns the FFmpeg decoder name for a supported MIME type. */
@Nullable
static String getCodecName(String mimeType) {
switch (mimeType) {
case MimeTypes.AUDIO_AAC:
return "aac";
case MimeTypes.AUDIO_MPEG:
case MimeTypes.AUDIO_MPEG_L1:
case MimeTypes.AUDIO_MPEG_L2:
return "mp3";
case MimeTypes.AUDIO_AC3:
return "ac3";
case MimeTypes.AUDIO_E_AC3:
case MimeTypes.AUDIO_E_AC3_JOC:
return "eac3";
case MimeTypes.AUDIO_TRUEHD:
return "truehd";
case MimeTypes.AUDIO_DTS:
case MimeTypes.AUDIO_DTS_HD:
return "dca";
case MimeTypes.AUDIO_VORBIS:
return "vorbis";
case MimeTypes.AUDIO_OPUS:
return "opus";
case MimeTypes.AUDIO_AMR_NB:
return "amrnb";
case MimeTypes.AUDIO_AMR_WB:
return "amrwb";
case MimeTypes.AUDIO_FLAC:
return "flac";
case MimeTypes.AUDIO_ALAC:
return "alac";
case MimeTypes.AUDIO_MLAW:
return "pcm_mulaw";
case MimeTypes.AUDIO_ALAW:
return "pcm_alaw";
default:
return null;
}
}
private static native String ffmpegGetVersion();
private static native int ffmpegGetInputBufferPaddingSize();
private static native boolean ffmpegHasDecoder(String codecName);
}