From dcfcdc25ed26cf2dd7b788593da604156568777c Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Tue, 12 May 2026 13:42:08 +0200 Subject: [PATCH 1/3] Implementation of MPEG-H offload playback --- .../main/java/androidx/media3/common/C.java | 16 +++ .../androidx/media3/common/MimeTypes.java | 2 + .../androidx/media3/common/util/Util.java | 18 +++ libraries/decoder_mpegh/README.md | 2 +- .../decoder/mpegh/MpeghAudioRenderer.java | 33 +++++- .../media3/decoder/mpegh/MpeghDecoder.java | 104 +++++++++++------- .../exoplayer/audio/AudioCapabilities.java | 4 + .../audio/AudioTrackAudioOutput.java | 9 +- .../exoplayer/audio/DefaultAudioSink.java | 31 +++++- .../media3/extractor/ExtractorUtil.java | 6 + .../media3/extractor/ts/MpeghUtil.java | 76 ++++++++++++- 11 files changed, 247 insertions(+), 54 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/C.java b/libraries/common/src/main/java/androidx/media3/common/C.java index 441628474ac..3c1d582062d 100644 --- a/libraries/common/src/main/java/androidx/media3/common/C.java +++ b/libraries/common/src/main/java/androidx/media3/common/C.java @@ -221,6 +221,10 @@ private C() {} ENCODING_OPUS, ENCODING_DTS_UHD_P2, ENCODING_DSD, + ENCODING_MPEGH_BL_L3, + ENCODING_MPEGH_BL_L4, + ENCODING_MPEGH_LC_L3, + ENCODING_MPEGH_LC_L4 }) public @interface Encoding {} @@ -340,6 +344,18 @@ private C() {} /** See {@link AudioFormat#ENCODING_DSD}. */ @UnstableApi public static final int ENCODING_DSD = AudioFormat.ENCODING_DSD; + /** See {@link AudioFormat#ENCODING_MPEGH_BL_L3}. */ + @UnstableApi public static final int ENCODING_MPEGH_BL_L3 = AudioFormat.ENCODING_MPEGH_BL_L3; + + /** See {@link AudioFormat#ENCODING_MPEGH_BL_L4}. */ + @UnstableApi public static final int ENCODING_MPEGH_BL_L4 = AudioFormat.ENCODING_MPEGH_BL_L4; + + /** See {@link AudioFormat#ENCODING_MPEGH_LC_L3}. */ + @UnstableApi public static final int ENCODING_MPEGH_LC_L3 = AudioFormat.ENCODING_MPEGH_LC_L3; + + /** See {@link AudioFormat#ENCODING_MPEGH_LC_L4}. */ + @UnstableApi public static final int ENCODING_MPEGH_LC_L4 = AudioFormat.ENCODING_MPEGH_LC_L4; + /** * Represents the behavior affecting whether spatialization will be used. One of {@link * #SPATIALIZATION_BEHAVIOR_AUTO} or {@link #SPATIALIZATION_BEHAVIOR_NEVER}. diff --git a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java index 134bafc85fd..f161969e905 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java +++ b/libraries/common/src/main/java/androidx/media3/common/MimeTypes.java @@ -726,6 +726,8 @@ public static boolean isDolbyVisionCodec( return C.ENCODING_OPUS; case MimeTypes.AUDIO_DSD: return C.ENCODING_DSD; + case MimeTypes.AUDIO_MPEGH_MHM1: + return C.ENCODING_MPEGH_BL_L3; default: return C.ENCODING_INVALID; } diff --git a/libraries/common/src/main/java/androidx/media3/common/util/Util.java b/libraries/common/src/main/java/androidx/media3/common/util/Util.java index 1456fdef55e..c226070df2e 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/Util.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/Util.java @@ -2629,6 +2629,20 @@ public static boolean isEncodingLinearPcm(@C.Encoding int encoding) { || encoding == C.ENCODING_PCM_DOUBLE_BIG_ENDIAN; } + /** + * Returns whether {@code encoding} is an MPEG-H encoding. + * + * @param encoding The encoding of the audio data. + * @return Whether the encoding is an MPEG-H encoding. + */ + @UnstableApi + public static boolean isMpegh(@C.Encoding int encoding) { + return encoding == C.ENCODING_MPEGH_BL_L3 + || encoding == C.ENCODING_MPEGH_BL_L4 + || encoding == C.ENCODING_MPEGH_LC_L3 + || encoding == C.ENCODING_MPEGH_LC_L4; + } + /** * Returns whether {@code encoding} is high resolution (> 16-bit) PCM. * @@ -2801,6 +2815,10 @@ public static int getApiLevelThatAudioFormatIntroducedAudioEncoding(int encoding return 30; case C.ENCODING_PCM_24BIT: case C.ENCODING_PCM_32BIT: + case C.ENCODING_MPEGH_BL_L3: + case C.ENCODING_MPEGH_BL_L4: + case C.ENCODING_MPEGH_LC_L3: + case C.ENCODING_MPEGH_LC_L4: return 31; case C.ENCODING_DTS_UHD_P2: case C.ENCODING_DSD: diff --git a/libraries/decoder_mpegh/README.md b/libraries/decoder_mpegh/README.md index c63a5e15409..d7d479ea8d6 100644 --- a/libraries/decoder_mpegh/README.md +++ b/libraries/decoder_mpegh/README.md @@ -25,7 +25,7 @@ the root of your media3 check-out): ```shell cd libraries/decoder_mpegh/src/main/jni && \ git clone https://github.com/Fraunhofer-IIS/mpeghdec.git \ - --branch r3.0.2 \ + --branch r3.0.3 \ --depth=1 \ libmpegh ``` diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java index 191f0d0ee68..82c35602ad1 100644 --- a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java @@ -61,6 +61,9 @@ public final class MpeghAudioRenderer extends DecoderAudioRenderer /** Helper for handling MPEG-H UI commands and system settings. */ private final MpeghUiCommandHelper uiHelper; + /** Whether the decoder is configured for direct (hardware offload) playback. **/ + private boolean isDirectPlayback; + /* Creates a new instance. */ public MpeghAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -95,6 +98,7 @@ public MpeghAudioRenderer( public MpeghAudioRenderer( Handler eventHandler, AudioRendererEventListener eventListener, AudioSink audioSink) { super(eventHandler, eventListener, audioSink); + audioSink.setOffloadMode(AudioSink.OFFLOAD_MODE_ENABLED_GAPLESS_NOT_REQUIRED); uiHelper = new MpeghUiCommandHelper(); uiHelper.setEventDispatcher( new AudioRendererEventListener.EventDispatcher(eventHandler, eventListener)); @@ -139,19 +143,38 @@ protected DecoderReuseEvaluation canReuseDecoder( protected MpeghDecoder createDecoder(Format format, CryptoConfig cryptoConfig) throws MpeghDecoderException { TraceUtil.beginSection("createMpeghDecoder"); - MpeghDecoder decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS, uiHelper); + Format.Builder checkFormatBuilder = format.buildUpon(); + checkFormatBuilder.setChannelCount(2); + format = checkFormatBuilder.build(); + @AudioSink.SinkFormatSupport + int formatSupport = getSinkFormatSupport(format); + isDirectPlayback = formatSupport == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; + MpeghDecoder decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS, uiHelper, isDirectPlayback); TraceUtil.endSection(); return decoder; } @Override protected Format getOutputFormat(MpeghDecoder decoder) { - return new Format.Builder() + String sampleMimeType; + @C.Encoding int pcmEncoding; + if (isDirectPlayback) { + sampleMimeType = MimeTypes.AUDIO_MPEGH_MHM1; + pcmEncoding = C.ENCODING_INVALID; + } else { + sampleMimeType = MimeTypes.AUDIO_RAW; + pcmEncoding = C.ENCODING_PCM_16BIT; + } + Format.Builder builder = new Format.Builder(); + builder .setChannelCount(decoder.getChannelCount()) .setSampleRate(decoder.getSampleRate()) - .setSampleMimeType(MimeTypes.AUDIO_RAW) - .setPcmEncoding(C.ENCODING_PCM_16BIT) - .build(); + .setSampleMimeType(sampleMimeType); + if (pcmEncoding != C.ENCODING_INVALID) { + builder.setPcmEncoding(pcmEncoding); + } + + return builder.build(); } @Override diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java index d29e6a1ef1e..ff4cfbd8e0f 100644 --- a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java @@ -51,7 +51,7 @@ public final class MpeghDecoder private final ByteBuffer tmpOutputBuffer; private final MpeghUiCommandHelper uiHelper; - private MpeghDecoderJni decoder; + private @Nullable MpeghDecoderJni decoder; private long outPtsUs; private int outChannels; private int outSampleRate; @@ -68,22 +68,15 @@ public final class MpeghDecoder * @throws MpeghDecoderException If an exception occurs when initializing the decoder. */ public MpeghDecoder( - Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper) + Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper, boolean directPlayback) throws MpeghDecoderException { super(new DecoderInputBuffer[numInputBuffers], new SimpleDecoderOutputBuffer[numOutputBuffers]); if (!MpeghLibrary.isAvailable()) { throw new MpeghDecoderException("Failed to load decoder native libraries."); } - byte[] configData = new byte[0]; - if (!format.initializationData.isEmpty() - && MimeTypes.AUDIO_MPEGH_MHA1.equals(format.sampleMimeType)) { - configData = format.initializationData.get(0); - } - - // Initialize the native MPEG-H decoder. - decoder = new MpeghDecoderJni(); - decoder.init(TARGET_LAYOUT_CICP, configData, configData.length); + outChannels = 2; + outSampleRate = 48000; int initialInputBufferSize = format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; @@ -97,6 +90,20 @@ public MpeghDecoder( * 2); // MAX_FRAME_LENGTH * MAX_NUM_CHANNELS * MAX_NUM_FRAMES * BYTES_PER_SAMPLE this.uiHelper = uiHelper; + + if (directPlayback) { + return; + } + + byte[] configData = new byte[0]; + if (!format.initializationData.isEmpty() + && MimeTypes.AUDIO_MPEGH_MHA1.equals(format.sampleMimeType)) { + configData = format.initializationData.get(0); + } + + // Initialize the native MPEG-H decoder. + decoder = new MpeghDecoderJni(); + decoder.init(TARGET_LAYOUT_CICP, configData, configData.length); } @Override @@ -123,11 +130,13 @@ protected MpeghDecoderException createUnexpectedDecodeException(Throwable error) @Nullable protected MpeghDecoderException decode( DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) { - if (reset) { - try { - decoder.flush(); - } catch (MpeghDecoderException e) { - return e; + if (decoder != null) { + if (reset) { + try { + decoder.flush(); + } catch (MpeghDecoderException e) { + return e; + } } } @@ -202,38 +211,45 @@ protected MpeghDecoderException decode( } } - long inputPtsUs = inputBuffer.timeUs; - - // Process/decode the incoming data. - try { - decoder.process(inputData, inputSize, inputPtsUs); - } catch (MpeghDecoderException e) { - return e; - } - - // Get as many decoded samples as possible. - int outputSize; int numBytes = 0; - int cnt = 0; tmpOutputBuffer.clear(); - do { + if (decoder != null) { + long inputPtsUs = inputBuffer.timeUs; + + // Process/decode the incoming data. try { - outputSize = decoder.getSamples(tmpOutputBuffer, numBytes); + decoder.process(inputData, inputSize, inputPtsUs); } catch (MpeghDecoderException e) { return e; } - // To concatenate possible additional audio frames, increase the write position. - numBytes += outputSize; - - if (cnt == 0 && outputSize > 0) { - // Only use the first frame for info about PTS, number of channels and sample rate. - outPtsUs = decoder.getPts(); - outChannels = decoder.getNumChannels(); - outSampleRate = decoder.getSamplerate(); - } - cnt++; - } while (outputSize > 0); + // Get as many decoded samples as possible. + int outputSize; + int cnt = 0; + do { + try { + outputSize = decoder.getSamples(tmpOutputBuffer, numBytes); + } catch (MpeghDecoderException e) { + return e; + } + // To concatenate possible additional audio frames, increase the write position. + numBytes += outputSize; + + if (cnt == 0 && outputSize > 0) { + // Only use the first frame for info about PTS, number of channels and sample rate. + outPtsUs = decoder.getPts(); + outChannels = decoder.getNumChannels(); + outSampleRate = decoder.getSamplerate(); + } + + cnt++; + } while (outputSize > 0); + } else { + outPtsUs = inputBuffer.timeUs; + numBytes = inputData.remaining(); + tmpOutputBuffer.put(inputData); + tmpOutputBuffer.rewind(); + } int outputSizeTotal = numBytes; tmpOutputBuffer.limit(outputSizeTotal); @@ -246,7 +262,11 @@ protected MpeghDecoderException decode( outputBuffer.init(outPtsUs, outputSizeTotal); // copy temporary output to output buffer - outputBuffer.data.asShortBuffer().put(tmpOutputBuffer.asShortBuffer()); + if (decoder != null) { + outputBuffer.data.asShortBuffer().put(tmpOutputBuffer.asShortBuffer()); + } else { + outputBuffer.data.put(tmpOutputBuffer); + } outputBuffer.data.rewind(); } else { // if no output data is available signalize that only decoding/processing was possible diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioCapabilities.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioCapabilities.java index 7824bb45241..f339264ece6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioCapabilities.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioCapabilities.java @@ -100,6 +100,10 @@ public final class AudioCapabilities { .put(C.ENCODING_E_AC3, 8) .put(C.ENCODING_DTS_HD, 8) .put(C.ENCODING_DOLBY_TRUEHD, 8) + .put(C.ENCODING_MPEGH_BL_L3, 8) + .put(C.ENCODING_MPEGH_BL_L4, 8) + .put(C.ENCODING_MPEGH_LC_L3, 8) + .put(C.ENCODING_MPEGH_LC_L4, 8) .buildOrThrow(); /** Global settings key for devices that can specify external surround sound. */ diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java index dd7662f827b..19aad7639b0 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java @@ -44,6 +44,7 @@ import androidx.media3.common.util.Util; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.audio.AudioOutputProvider.OutputConfig; +import androidx.media3.extractor.ts.MpeghUtil; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.concurrent.Future; @@ -97,6 +98,7 @@ public interface CapabilityChangeListener { @Nullable private OnRoutingChangedListenerApi24 onRoutingChangedListener; private final AudioTrackPositionTracker audioTrackPositionTracker; private final boolean isOutputPcm; + private boolean isMpegh; private final int pcmFrameSize; @Nullable private final StreamEventCallbackV29 offloadStreamEventCallbackV29; private final ListenerSet listeners; @@ -155,6 +157,7 @@ public AudioTrackAudioOutput( int channelCount = Integer.bitCount(config.channelMask); pcmFrameSize = Util.getPcmFrameSize(config.encoding, channelCount); } else { + isMpegh = Util.isMpegh(config.encoding); pcmFrameSize = C.LENGTH_UNSET; } @@ -243,6 +246,10 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present // If this is the first encoded sample, calculate the sample size in frames. framesPerEncodedSample = DefaultAudioSink.getFramesPerEncodedSample(config.encoding, buffer); } + int truncationSamples = 0; + if (isMpegh && buffer.remaining() == buffer.limit()) { + truncationSamples = MpeghUtil.getTruncationSampleCount(buffer); + } maybeReportUnderrun(); int bytesRemaining = buffer.remaining(); int bytesWrittenOrError; @@ -277,7 +284,7 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present } else if (fullyHandled) { // For non-PCM we can only be sure about the number of written frames once the entire buffer // is submitted. - writtenEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount; + writtenEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; } return fullyHandled; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index e6007c4afd5..3fa50768cca 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -66,6 +66,7 @@ import androidx.media3.extractor.DtsUtil; import androidx.media3.extractor.ExtractorUtil; import androidx.media3.extractor.MpegAudioUtil; +import androidx.media3.extractor.ts.MpeghUtil; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.lang.annotation.Documented; @@ -1022,6 +1023,11 @@ public boolean handleBuffer( } } + int truncationSamples = 0; + if (Util.isMpegh(configuration.outputConfig.encoding)) { + truncationSamples += MpeghUtil.getTruncationSampleCount(buffer); + } + if (afterDrainParameters != null) { if (!drainToEndOfStream()) { // Don't process any more input until draining completes. @@ -1065,7 +1071,7 @@ public boolean handleBuffer( if (configuration.isPcm()) { submittedPcmBytes += buffer.remaining(); } else { - submittedEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount; + submittedEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; } inputBuffer = buffer; @@ -1263,6 +1269,10 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept if (outputBuffer == null) { return; } + int truncationSamples = 0; + if (Util.isMpegh(configuration.outputConfig.encoding) && outputBuffer.remaining() == outputBuffer.limit()) { + truncationSamples = MpeghUtil.getTruncationSampleCount(outputBuffer); + } if (writeExceptionPendingExceptionHolder.shouldWaitBeforeRetry()) { return; } @@ -1326,12 +1336,20 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept // When playing non-PCM, the inputBuffer is never processed, thus the last inputBuffer // must be the current input buffer. checkState(outputBuffer == inputBuffer); - // Add only the remaining unreconciled frames to ensure exact sample accuracy. + int truncationSamples = 0; + if (Util.isMpegh(configuration.outputConfig.encoding)) { + ParsableByteArray byteBuffer = new ParsableByteArray(); + int bufferLimit = outputBuffer.limit(); + byteBuffer.reset(bufferLimit); + outputBuffer.get(byteBuffer.getData(), 0, bufferLimit); + outputBuffer.position(0); + truncationSamples = MpeghUtil.getTruncationSampleCount(byteBuffer); + } writtenEncodedFrames += ((long) framesPerEncodedSample * inputBufferAccessUnitCount) + - truncationSamples - currentBufferFramesWritten; - currentBufferFramesWritten = 0; - } + currentBufferFramesWritten = 0; } outputBuffer = null; } else { if (!configuration.isPcm() && bytesWritten > 0) { @@ -1902,6 +1920,11 @@ private FormatConfig getFormatConfig(Format format, int preferredBufferSize) { * Ac3Util.TRUEHD_RECHUNK_SAMPLE_COUNT); case C.ENCODING_OPUS: return OpusUtil.parseOggPacketAudioSampleCount(buffer); + case C.ENCODING_MPEGH_BL_L3: + case C.ENCODING_MPEGH_BL_L4: + case C.ENCODING_MPEGH_LC_L3: + case C.ENCODING_MPEGH_LC_L4: + return MpeghUtil.getStandardFrameLength(buffer); case C.ENCODING_PCM_16BIT: case C.ENCODING_PCM_16BIT_BIG_ENDIAN: case C.ENCODING_PCM_24BIT: diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java index a411b110982..2746f3e5d2e 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java @@ -21,6 +21,7 @@ import androidx.media3.common.ParserException; import androidx.media3.common.util.UnstableApi; import androidx.media3.container.OpusUtil; +import androidx.media3.extractor.ts.MpeghUtil; import java.io.EOFException; import java.io.IOException; import org.checkerframework.dataflow.qual.Pure; @@ -161,6 +162,11 @@ public static int getMaximumEncodedRateBytesPerSecond(@C.Encoding int encoding) return Ac3Util.TRUEHD_MAX_RATE_BYTES_PER_SECOND; case C.ENCODING_OPUS: return OpusUtil.MAX_BYTES_PER_SECOND; + case C.ENCODING_MPEGH_BL_L3: + case C.ENCODING_MPEGH_BL_L4: + case C.ENCODING_MPEGH_LC_L3: + case C.ENCODING_MPEGH_LC_L4: + return MpeghUtil.MAX_RATE_BYTES_PER_SECOND; case C.ENCODING_PCM_16BIT: case C.ENCODING_PCM_16BIT_BIG_ENDIAN: case C.ENCODING_PCM_24BIT: diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java index 4bdef07a8e5..3b6042a2815 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java @@ -23,15 +23,25 @@ import androidx.media3.common.C; import androidx.media3.common.ParserException; import androidx.media3.common.util.ParsableBitArray; +import androidx.media3.common.util.UnstableApi; import com.google.common.math.IntMath; import com.google.common.math.LongMath; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; +import java.nio.ByteBuffer; /** Utility methods for parsing MPEG-H frames, which are access units in MPEG-H bitstreams. */ -/* package */ final class MpeghUtil { +@UnstableApi +public final class MpeghUtil { + + /** + * Maximum rate for an MPEG-H audio stream, in bytes per second. + * 56 channels × 288 kbps/channel = 16128 kbps -> 2016000 bytes/s + * assumes 1024-sample frame length at 48000 Hz for MPEG-H BL L4 maximum + */ + public static final int MAX_RATE_BYTES_PER_SECOND = 16128 * 1000 / 8; /** See ISO_IEC_23003-8;2022, 14.4.4. */ private static final int MHAS_SYNC_WORD = 0xC001A5; @@ -736,4 +746,68 @@ private Mpegh3daConfig( this.compatibleProfileLevelSet = compatibleProfileLevelSet; } } + + /** + * Obtains the number of truncated samples from an MPEG-H bit stream. + * + * @param buffer The data to parse, containing complete MPEG-H access units + * @return The number of truncated samples. + */ + public static int getTruncationSampleCount(ByteBuffer buffer) { + int truncationSamples = 0; + int bufferPos = buffer.position(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + buffer.position(bufferPos); + ParsableBitArray bitArray = new ParsableBitArray(bytes); + + MhasPacketHeader header = new MhasPacketHeader(); + while (bitArray.bitsLeft() > 0) { + try { + if (!parseMhasPacketHeader(bitArray, header)) { + return 0; + } + if (header.packetType == MhasPacketHeader.PACTYP_AUDIOTRUNCATION) { + truncationSamples += parseAudioTruncationInfo(bitArray); + } else { + bitArray.skipBytes(header.packetLength); + } + } catch (ParserException e) { + return 0; + } + } + return truncationSamples; + } + + /** + * Obtains the standard audio frame length from an MPEG-H bit stream. + * + * @param buffer The data to parse, containing complete MPEG-H access units + * @return The standard audio frame length. + */ + public static int getStandardFrameLength(ByteBuffer buffer) { + int bufferPos = buffer.position(); + byte[] bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + buffer.position(bufferPos); + ParsableBitArray bitArray = new ParsableBitArray(bytes); + + MhasPacketHeader header = new MhasPacketHeader(); + while (bitArray.bitsLeft() > 0) { + try { + if (!parseMhasPacketHeader(bitArray, header)) { + return 0; + } + if (header.packetType == MhasPacketHeader.PACTYP_MPEGH3DACFG) { + Mpegh3daConfig config = parseMpegh3daConfig(bitArray); + return config.standardFrameLength; + } else { + bitArray.skipBytes(header.packetLength); + } + } catch (ParserException e) { + return 0; + } + } + return 0; + } } From ebb61c5c69b7756ddeaa1fe773089c892095f37f Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Thu, 28 May 2026 15:57:28 +0200 Subject: [PATCH 2/3] - moving MpeghUtil class to androidx.media3.extractor - implementing MpeghBaseDecoder class and extending MpeghDecoder and MpeghPassThroughDecoder from it - allocating resuable ParseableByteArray in DefaultAudioSink and AudioTrackAudioOutput for MPEG-H frame parsing - implementing unit tests for MpeghUtil and tests for MpeghAudioRenderer and DefaultAudioSink --- .../decoder/mpegh/MpeghAudioRenderer.java | 52 ++-- .../decoder/mpegh/MpeghBaseDecoder.java | 230 ++++++++++++++++ .../media3/decoder/mpegh/MpeghDecoder.java | 246 ++++------------- .../mpegh/MpeghPassThroughDecoder.java | 69 +++++ .../decoder/mpegh/MpeghAudioRendererTest.java | 213 +++++++++++++++ .../audio/AudioTrackAudioOutput.java | 16 +- .../exoplayer/audio/DefaultAudioSink.java | 32 ++- .../exoplayer/audio/DefaultAudioSinkTest.java | 88 ++++++ .../media3/extractor/ExtractorUtil.java | 1 - .../media3/extractor/{ts => }/MpeghUtil.java | 14 +- .../media3/extractor/ts/MpeghReader.java | 1 + .../media3/extractor/MpeghUtilTest.java | 250 ++++++++++++++++++ 12 files changed, 961 insertions(+), 251 deletions(-) create mode 100644 libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghBaseDecoder.java create mode 100644 libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghPassThroughDecoder.java create mode 100644 libraries/decoder_mpegh/src/test/java/androidx/media3/decoder/mpegh/MpeghAudioRendererTest.java rename libraries/extractor/src/main/java/androidx/media3/extractor/{ts => }/MpeghUtil.java (98%) create mode 100644 libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java index 82c35602ad1..810bbf4e056 100644 --- a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java @@ -38,7 +38,7 @@ /** Decodes and renders audio using the native MPEG-H decoder. */ @UnstableApi -public final class MpeghAudioRenderer extends DecoderAudioRenderer { +public final class MpeghAudioRenderer extends DecoderAudioRenderer { private static final String TAG = "MpeghAudioRenderer"; @@ -111,9 +111,19 @@ public String getName() { @Override protected @C.FormatSupport int supportsFormatInternal(Format format) { - // Check if JNI library is available. - if (!MpeghLibrary.isAvailable()) { - return C.FORMAT_UNSUPPORTED_TYPE; + if (format.channelCount <= 0) { + Format.Builder checkFormatBuilder = format.buildUpon(); + checkFormatBuilder.setChannelCount(2); + format = checkFormatBuilder.build(); + } + @AudioSink.SinkFormatSupport + int formatSupport = getSinkFormatSupport(format); + isDirectPlayback = formatSupport == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; + + if (isDirectPlayback) { + if (!Objects.equals(format.sampleMimeType, MimeTypes.AUDIO_MPEGH_MHM1)) { + return C.FORMAT_UNSUPPORTED_TYPE; + } } // Check if MIME type is supported. @@ -140,35 +150,29 @@ protected DecoderReuseEvaluation canReuseDecoder( } @Override - protected MpeghDecoder createDecoder(Format format, CryptoConfig cryptoConfig) + protected MpeghBaseDecoder createDecoder(Format format, CryptoConfig cryptoConfig) throws MpeghDecoderException { TraceUtil.beginSection("createMpeghDecoder"); - Format.Builder checkFormatBuilder = format.buildUpon(); - checkFormatBuilder.setChannelCount(2); - format = checkFormatBuilder.build(); - @AudioSink.SinkFormatSupport - int formatSupport = getSinkFormatSupport(format); - isDirectPlayback = formatSupport == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; - MpeghDecoder decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS, uiHelper, isDirectPlayback); + MpeghBaseDecoder decoder; + if (isDirectPlayback) { + decoder = new MpeghPassThroughDecoder(format, NUM_BUFFERS, NUM_BUFFERS, uiHelper); + } else { + decoder = new MpeghDecoder(format, NUM_BUFFERS, NUM_BUFFERS, uiHelper); + } TraceUtil.endSection(); return decoder; } @Override - protected Format getOutputFormat(MpeghDecoder decoder) { - String sampleMimeType; - @C.Encoding int pcmEncoding; - if (isDirectPlayback) { - sampleMimeType = MimeTypes.AUDIO_MPEGH_MHM1; - pcmEncoding = C.ENCODING_INVALID; - } else { - sampleMimeType = MimeTypes.AUDIO_RAW; - pcmEncoding = C.ENCODING_PCM_16BIT; - } + protected Format getOutputFormat(MpeghBaseDecoder decoder) { + int channelCount = decoder.getChannelCount(); + int sampleRate = decoder.getSampleRate(); + String sampleMimeType = decoder.getSampleMimeType(); + @C.Encoding int pcmEncoding = decoder.getPcmEncoding(); Format.Builder builder = new Format.Builder(); builder - .setChannelCount(decoder.getChannelCount()) - .setSampleRate(decoder.getSampleRate()) + .setChannelCount(channelCount) + .setSampleRate(sampleRate) .setSampleMimeType(sampleMimeType); if (pcmEncoding != C.ENCODING_INVALID) { builder.setPcmEncoding(pcmEncoding); diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghBaseDecoder.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghBaseDecoder.java new file mode 100644 index 00000000000..b68597764d0 --- /dev/null +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghBaseDecoder.java @@ -0,0 +1,230 @@ +package androidx.media3.decoder.mpegh; + +import static androidx.media3.decoder.mpegh.MpeghAudioRenderer.CODEC_PARAM_MPEGH_UI_CONFIG; +import static androidx.media3.decoder.mpegh.MpeghAudioRenderer.CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER; +import static com.google.common.base.Preconditions.checkNotNull; + +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.decoder.SimpleDecoder; +import androidx.media3.decoder.SimpleDecoderOutputBuffer; +import androidx.media3.exoplayer.CodecParameters; +import androidx.media3.exoplayer.audio.AudioRendererEventListener; +import java.nio.ByteBuffer; +import java.util.Set; + +public class MpeghBaseDecoder extends + SimpleDecoder { + + private static final String TAG = "MpeghBaseDecoder"; + + /** + * The default input buffer size. + */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = 2048 * 6; + + /** + * The maximum padding required for MPEG-H UI manager data added to the input buffer. This is + * based on the size of a single MPEG-H AU. + */ + private static final int UI_MANAGER_PADDING_SIZE = 2048; + + private final MpeghUiCommandHelper uiHelper; + + private @Nullable MpeghUiManagerJni uiManager; + + protected int outChannels; + protected int outSampleRate; + protected String outSampleMimeType; + protected @C.Encoding int outPcmEncoding; + + /** + * Creates an MPEG-H base decoder. + * + * @param format The input {@link Format}. + * @param numInputBuffers The number of input buffers. + * @param numOutputBuffers The number of output buffers. + * @param uiHelper A helper class to hold variables/commands which are obtained in the {@link + * MpeghAudioRenderer} and are needed to perform the UI handling. + * @throws MpeghDecoderException If an exception occurs when initializing the decoder. + */ + public MpeghBaseDecoder( + Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper) + throws MpeghDecoderException { + super(new DecoderInputBuffer[numInputBuffers], new SimpleDecoderOutputBuffer[numOutputBuffers]); + if (!MpeghLibrary.isAvailable()) { + throw new MpeghDecoderException("Failed to load decoder native libraries."); + } + + outChannels = 2; + outSampleRate = 48000; + outPcmEncoding = C.ENCODING_INVALID; + outSampleMimeType = MimeTypes.AUDIO_MPEGH_MHM1; + String sampleMimeType = format.sampleMimeType; + if (sampleMimeType != null) { + outSampleMimeType = sampleMimeType; + } + + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + // Add padding for MPEG-H UI manager data so we don't need to reallocate at runtime. + setInitialInputBufferSize(initialInputBufferSize + UI_MANAGER_PADDING_SIZE); + + this.uiHelper = uiHelper; + } + + @Override + protected DecoderInputBuffer createInputBuffer() { + return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); + } + + @Override + protected SimpleDecoderOutputBuffer createOutputBuffer() { + return new SimpleDecoderOutputBuffer(this::releaseOutputBuffer); + } + + @Override + protected MpeghDecoderException createUnexpectedDecodeException(Throwable error) { + return new MpeghDecoderException("Unexpected decode error", error); + } + + @Nullable + @Override + protected MpeghDecoderException decode(DecoderInputBuffer inputBuffer, + SimpleDecoderOutputBuffer outputBuffer, boolean reset) { + + // lazy initialization of UI manager + if (uiManager == null + && MimeTypes.AUDIO_MPEGH_MHM1.equals(checkNotNull(inputBuffer.format).sampleMimeType)) { + + ByteBuffer persistenceBuffer = uiHelper.getPersistenceStorage(); + + int persistenceBufferSize = 0; + if (persistenceBuffer != null) { + persistenceBufferSize = persistenceBuffer.capacity(); + } + + uiManager = new MpeghUiManagerJni(); + try { + uiManager.init(persistenceBuffer, persistenceBufferSize); + } catch (MpeghDecoderException e) { + return e; + } + + // apply MPEG-H system settings + for (String command : uiHelper.getCommands(/* includeSystemSettings= */ true)) { + uiManager.command(command); + } + } + + // Get the data from the input buffer. + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int inputSize = inputData.limit(); + + if (uiManager != null) { + // Allow the UI manager to access the whole buffer (including padding). + inputData.limit(inputData.capacity()); + + boolean feedSuccess = uiManager.feed(inputData, inputSize); + if (feedSuccess) { + for (String command : uiHelper.getCommands(/* includeSystemSettings= */ false)) { + uiManager.command(command); + } + + // process the access unit with the UI manager + boolean forceUiUpdate = uiHelper.getForceUiUpdate(); + inputSize = uiManager.update(inputData, inputSize, forceUiUpdate); + inputData.limit(inputSize); + uiHelper.setForceUiUpdate(false); + + boolean newOsdAvailable = uiManager.newOsdAvailable(); + if (newOsdAvailable) { + String osdXml = uiManager.getOsd(); + + @Nullable Set subscribedKeys = uiHelper.getSubscribedCodecParameterKeys(); + @Nullable + AudioRendererEventListener.EventDispatcher dispatcher = uiHelper.getEventDispatcher(); + if (subscribedKeys != null && dispatcher != null) { + if (subscribedKeys.contains(CODEC_PARAM_MPEGH_UI_CONFIG)) { + // reset CodecParameter with KEY_MPEGH_UI_CONFIG to null as it is possible that the + // last config needs to be resent, because only 'real' changes are propagated + // further on by audioCodecParametersChanged + dispatcher.audioCodecParametersChanged( + new CodecParameters.Builder() + .setString(CODEC_PARAM_MPEGH_UI_CONFIG, null) + .build()); + // actually send the current MPEG-H UI config + dispatcher.audioCodecParametersChanged( + new CodecParameters.Builder() + .setString(CODEC_PARAM_MPEGH_UI_CONFIG, osdXml) + .build()); + } + } + } + } + } + return null; + } + + @Override + public String getName() { + return TAG; + } + + @Override + public void release() { + super.release(); + + if (uiManager != null) { + ByteBuffer persistenceBuffer = uiHelper.getPersistenceStorage(); + if (persistenceBuffer != null) { + persistenceBuffer.rewind(); + int unused = uiManager.destroy(persistenceBuffer, persistenceBuffer.capacity()); + @Nullable Set subscribedKeys = uiHelper.getSubscribedCodecParameterKeys(); + @Nullable + AudioRendererEventListener.EventDispatcher dispatcher = uiHelper.getEventDispatcher(); + if (subscribedKeys != null && dispatcher != null) { + if (subscribedKeys.contains(CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER)) { + dispatcher.audioCodecParametersChanged( + new CodecParameters.Builder() + .setByteBuffer(CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER, persistenceBuffer) + .build()); + } + } + } + uiManager = null; + } + } + + /** + * Returns the channel count of output audio. + */ + protected int getChannelCount() { + return outChannels; + } + + /** + * Returns the sample rate of output audio. + */ + protected int getSampleRate() { + return outSampleRate; + } + + /** + * Returns the sample mime type of output audio. + */ + protected String getSampleMimeType() { + return outSampleMimeType; + } + + /** + * Returns the PCM encoding of output audio. + */ + protected @C.PcmEncoding int getPcmEncoding() { + return outPcmEncoding; + } +} diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java index ff4cfbd8e0f..8b2520d5771 100644 --- a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghDecoder.java @@ -15,73 +15,46 @@ */ package androidx.media3.decoder.mpegh; -import static androidx.media3.decoder.mpegh.MpeghAudioRenderer.CODEC_PARAM_MPEGH_UI_CONFIG; -import static androidx.media3.decoder.mpegh.MpeghAudioRenderer.CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER; -import static com.google.common.base.Preconditions.checkNotNull; - import androidx.annotation.Nullable; +import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; -import androidx.media3.decoder.SimpleDecoder; import androidx.media3.decoder.SimpleDecoderOutputBuffer; -import androidx.media3.exoplayer.CodecParameters; -import androidx.media3.exoplayer.audio.AudioRendererEventListener; import java.nio.ByteBuffer; -import java.util.Set; /** MPEG-H decoder. */ @UnstableApi -public final class MpeghDecoder - extends SimpleDecoder { - - /** The default input buffer size. */ - private static final int DEFAULT_INPUT_BUFFER_SIZE = 2048 * 6; - - /** - * The maximum padding required for MPEG-H UI manager data added to the input buffer. This is - * based on the size of a single MPEG-H AU. - */ - private static final int UI_MANAGER_PADDING_SIZE = 2048; +public final class MpeghDecoder extends MpeghBaseDecoder { private static final int TARGET_LAYOUT_CICP = 2; - private final ByteBuffer tmpOutputBuffer; - private final MpeghUiCommandHelper uiHelper; - - private @Nullable MpeghDecoderJni decoder; + private final MpeghDecoderJni decoder; private long outPtsUs; - private int outChannels; - private int outSampleRate; - private @Nullable MpeghUiManagerJni uiManager; + + private final ByteBuffer tmpOutputBuffer; /** * Creates an MPEG-H decoder. * - * @param format The input {@link Format}. - * @param numInputBuffers The number of input buffers. + * @param format The input {@link Format}. + * @param numInputBuffers The number of input buffers. * @param numOutputBuffers The number of output buffers. - * @param uiHelper A helper class to hold variables/commands which are obtained in the {@link - * MpeghAudioRenderer} and are needed to perform the UI handling. + * @param uiHelper A helper class to hold variables/commands which are obtained in the {@link + * MpeghAudioRenderer} and are needed to perform the UI handling. * @throws MpeghDecoderException If an exception occurs when initializing the decoder. */ public MpeghDecoder( - Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper, boolean directPlayback) + Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper) throws MpeghDecoderException { - super(new DecoderInputBuffer[numInputBuffers], new SimpleDecoderOutputBuffer[numOutputBuffers]); - if (!MpeghLibrary.isAvailable()) { - throw new MpeghDecoderException("Failed to load decoder native libraries."); - } + super(format, numInputBuffers, numOutputBuffers, uiHelper); outChannels = 2; outSampleRate = 48000; - - int initialInputBufferSize = - format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; - // Add padding for MPEG-H UI manager data so we don't need to reallocate at runtime. - setInitialInputBufferSize(initialInputBufferSize + UI_MANAGER_PADDING_SIZE); + outSampleMimeType = MimeTypes.AUDIO_RAW; + outPcmEncoding = C.ENCODING_PCM_16BIT; // Allocate memory for the temporary output of the native MPEG-H decoder. tmpOutputBuffer = @@ -89,12 +62,6 @@ public MpeghDecoder( 3072 * 24 * 6 * 2); // MAX_FRAME_LENGTH * MAX_NUM_CHANNELS * MAX_NUM_FRAMES * BYTES_PER_SAMPLE - this.uiHelper = uiHelper; - - if (directPlayback) { - return; - } - byte[] configData = new byte[0]; if (!format.initializationData.isEmpty() && MimeTypes.AUDIO_MPEGH_MHA1.equals(format.sampleMimeType)) { @@ -111,145 +78,61 @@ public String getName() { return "libmpegh"; } - @Override - protected DecoderInputBuffer createInputBuffer() { - return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); - } - - @Override - protected SimpleDecoderOutputBuffer createOutputBuffer() { - return new SimpleDecoderOutputBuffer(this::releaseOutputBuffer); - } - - @Override - protected MpeghDecoderException createUnexpectedDecodeException(Throwable error) { - return new MpeghDecoderException("Unexpected decode error", error); - } - @Override @Nullable protected MpeghDecoderException decode( DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) { - if (decoder != null) { - if (reset) { - try { - decoder.flush(); - } catch (MpeghDecoderException e) { - return e; - } - } - } - // lazy initialization of UI manager - if (uiManager == null - && MimeTypes.AUDIO_MPEGH_MHM1.equals(checkNotNull(inputBuffer.format).sampleMimeType)) { - - ByteBuffer persistenceBuffer = uiHelper.getPersistenceStorage(); - - int persistenceBufferSize = 0; - if (persistenceBuffer != null) { - persistenceBufferSize = persistenceBuffer.capacity(); - } - - uiManager = new MpeghUiManagerJni(); + if (reset) { try { - uiManager.init(persistenceBuffer, persistenceBufferSize); + decoder.flush(); } catch (MpeghDecoderException e) { return e; } + } - // apply MPEG-H system settings - for (String command : uiHelper.getCommands(/* includeSystemSettings= */ true)) { - uiManager.command(command); - } + MpeghDecoderException exception = super.decode(inputBuffer, outputBuffer, reset); + if (exception != null) { + return exception; } // Get the data from the input buffer. ByteBuffer inputData = Util.castNonNull(inputBuffer.data); int inputSize = inputData.limit(); - if (uiManager != null) { - // Allow the UI manager to access the whole buffer (including padding). - inputData.limit(inputData.capacity()); - - boolean feedSuccess = uiManager.feed(inputData, inputSize); - if (feedSuccess) { - for (String command : uiHelper.getCommands(/* includeSystemSettings= */ false)) { - uiManager.command(command); - } - - // process the access unit with the UI manager - boolean forceUiUpdate = uiHelper.getForceUiUpdate(); - inputSize = uiManager.update(inputData, inputSize, forceUiUpdate); - inputData.limit(inputSize); - uiHelper.setForceUiUpdate(false); + int numBytes = 0; + tmpOutputBuffer.clear(); - boolean newOsdAvailable = uiManager.newOsdAvailable(); - if (newOsdAvailable) { - String osdXml = uiManager.getOsd(); + long inputPtsUs = inputBuffer.timeUs; - @Nullable Set subscribedKeys = uiHelper.getSubscribedCodecParameterKeys(); - @Nullable - AudioRendererEventListener.EventDispatcher dispatcher = uiHelper.getEventDispatcher(); - if (subscribedKeys != null && dispatcher != null) { - if (subscribedKeys.contains(CODEC_PARAM_MPEGH_UI_CONFIG)) { - // reset CodecParameter with KEY_MPEGH_UI_CONFIG to null as it is possible that the - // last config needs to be resent, because only 'real' changes are propagated - // further on by audioCodecParametersChanged - dispatcher.audioCodecParametersChanged( - new CodecParameters.Builder() - .setString(CODEC_PARAM_MPEGH_UI_CONFIG, null) - .build()); - // actually send the current MPEG-H UI config - dispatcher.audioCodecParametersChanged( - new CodecParameters.Builder() - .setString(CODEC_PARAM_MPEGH_UI_CONFIG, osdXml) - .build()); - } - } - } - } + // Process/decode the incoming data. + try { + decoder.process(inputData, inputSize, inputPtsUs); + } catch (MpeghDecoderException e) { + return e; } - int numBytes = 0; - tmpOutputBuffer.clear(); - if (decoder != null) { - long inputPtsUs = inputBuffer.timeUs; - - // Process/decode the incoming data. + // Get as many decoded samples as possible. + int outputSize; + int cnt = 0; + do { try { - decoder.process(inputData, inputSize, inputPtsUs); + outputSize = decoder.getSamples(tmpOutputBuffer, numBytes); } catch (MpeghDecoderException e) { return e; } + // To concatenate possible additional audio frames, increase the write position. + numBytes += outputSize; + + if (cnt == 0 && outputSize > 0) { + // Only use the first frame for info about PTS, number of channels and sample rate. + outPtsUs = decoder.getPts(); + outChannels = decoder.getNumChannels(); + outSampleRate = decoder.getSamplerate(); + } - // Get as many decoded samples as possible. - int outputSize; - int cnt = 0; - do { - try { - outputSize = decoder.getSamples(tmpOutputBuffer, numBytes); - } catch (MpeghDecoderException e) { - return e; - } - // To concatenate possible additional audio frames, increase the write position. - numBytes += outputSize; - - if (cnt == 0 && outputSize > 0) { - // Only use the first frame for info about PTS, number of channels and sample rate. - outPtsUs = decoder.getPts(); - outChannels = decoder.getNumChannels(); - outSampleRate = decoder.getSamplerate(); - } - - cnt++; - } while (outputSize > 0); - } else { - outPtsUs = inputBuffer.timeUs; - numBytes = inputData.remaining(); - tmpOutputBuffer.put(inputData); - tmpOutputBuffer.rewind(); - } + cnt++; + } while (outputSize > 0); int outputSizeTotal = numBytes; tmpOutputBuffer.limit(outputSizeTotal); @@ -262,11 +145,8 @@ protected MpeghDecoderException decode( outputBuffer.init(outPtsUs, outputSizeTotal); // copy temporary output to output buffer - if (decoder != null) { - outputBuffer.data.asShortBuffer().put(tmpOutputBuffer.asShortBuffer()); - } else { - outputBuffer.data.put(tmpOutputBuffer); - } + outputBuffer.data.asShortBuffer().put(tmpOutputBuffer.asShortBuffer()); + outputBuffer.data.rewind(); } else { // if no output data is available signalize that only decoding/processing was possible @@ -278,40 +158,6 @@ protected MpeghDecoderException decode( @Override public void release() { super.release(); - - if (uiManager != null) { - ByteBuffer persistenceBuffer = uiHelper.getPersistenceStorage(); - if (persistenceBuffer != null) { - persistenceBuffer.rewind(); - int unused = uiManager.destroy(persistenceBuffer, persistenceBuffer.capacity()); - @Nullable Set subscribedKeys = uiHelper.getSubscribedCodecParameterKeys(); - @Nullable - AudioRendererEventListener.EventDispatcher dispatcher = uiHelper.getEventDispatcher(); - if (subscribedKeys != null && dispatcher != null) { - if (subscribedKeys.contains(CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER)) { - dispatcher.audioCodecParametersChanged( - new CodecParameters.Builder() - .setByteBuffer(CODEC_PARAM_MPEGH_UI_PERSISTENCE_BUFFER, persistenceBuffer) - .build()); - } - } - } - uiManager = null; - } - - if (decoder != null) { - decoder.destroy(); - decoder = null; - } - } - - /** Returns the channel count of output audio. */ - public int getChannelCount() { - return outChannels; - } - - /** Returns the sample rate of output audio. */ - public int getSampleRate() { - return outSampleRate; + decoder.destroy(); } } diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghPassThroughDecoder.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghPassThroughDecoder.java new file mode 100644 index 00000000000..cc9d0b84a06 --- /dev/null +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghPassThroughDecoder.java @@ -0,0 +1,69 @@ +package androidx.media3.decoder.mpegh; + +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.decoder.SimpleDecoderOutputBuffer; +import java.nio.ByteBuffer; + +public class MpeghPassThroughDecoder extends MpeghBaseDecoder { + + private static final String TAG = "MpeghPassThroughDecoder"; + /** + * Creates an MPEG-H decoder. + * + * @param format The input {@link Format}. + * @param numInputBuffers The number of input buffers. + * @param numOutputBuffers The number of output buffers. + * @param uiHelper A helper class to hold variables/commands which are obtained in the {@link + * MpeghAudioRenderer} and are needed to perform the UI handling. + * @throws MpeghDecoderException If an exception occurs when initializing the decoder. + */ + public MpeghPassThroughDecoder( + Format format, int numInputBuffers, int numOutputBuffers, MpeghUiCommandHelper uiHelper) + throws MpeghDecoderException { + super(format, numInputBuffers, numOutputBuffers, uiHelper); + + outChannels = 2; + outSampleRate = 48000; + outSampleMimeType = MimeTypes.AUDIO_MPEGH_MHM1; + outPcmEncoding = C.ENCODING_INVALID; + } + + @Override + public String getName() { + return TAG; + } + + @Override + @Nullable + protected MpeghDecoderException decode( + DecoderInputBuffer inputBuffer, SimpleDecoderOutputBuffer outputBuffer, boolean reset) { + + MpeghDecoderException exception = super.decode(inputBuffer, outputBuffer, reset); + if (exception != null) { + return exception; + } + + // Get the data from the input buffer. + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + if (inputData.remaining() > 0) { + // There is output data available + + // initialize the output buffer + outputBuffer.clear(); + outputBuffer.init(inputBuffer.timeUs, inputData.remaining()); + + // copy temporary output to output buffer + outputBuffer.data.put(inputData); + outputBuffer.data.rewind(); + } else { + // if no output data is available signalize that only decoding/processing was possible + outputBuffer.shouldBeSkipped = true; + } + return null; + } +} diff --git a/libraries/decoder_mpegh/src/test/java/androidx/media3/decoder/mpegh/MpeghAudioRendererTest.java b/libraries/decoder_mpegh/src/test/java/androidx/media3/decoder/mpegh/MpeghAudioRendererTest.java new file mode 100644 index 00000000000..9d13ce95eeb --- /dev/null +++ b/libraries/decoder_mpegh/src/test/java/androidx/media3/decoder/mpegh/MpeghAudioRendererTest.java @@ -0,0 +1,213 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.decoder.mpegh; + +import static android.os.Build.VERSION.SDK_INT; +import static com.google.common.truth.Truth.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.robolectric.Shadows.shadowOf; + +import android.app.UiModeManager; +import android.content.Context; +import android.content.Intent; +import android.content.res.Configuration; +import android.media.AudioDeviceInfo; +import android.media.AudioFormat; +import android.media.AudioManager; +import android.media.AudioProfile; +import androidx.annotation.RequiresApi; +import androidx.media3.common.AudioAttributes; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.exoplayer.audio.AudioSink; +import androidx.media3.exoplayer.audio.DefaultAudioSink; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.annotation.Config; +import org.robolectric.annotation.Implementation; +import org.robolectric.annotation.Implements; +import org.robolectric.shadows.AudioDeviceInfoBuilder; +import org.robolectric.shadows.AudioProfileBuilder; +import org.robolectric.shadows.ShadowAudioManager; +import org.robolectric.shadows.ShadowAudioSystem; +import org.robolectric.shadows.ShadowAudioTrack; + +/** Unit tests for {@link MpeghAudioRenderer}. */ +@RunWith(AndroidJUnit4.class) +public final class MpeghAudioRendererTest { + + private static final int DEFAULT_MAX_CHANNEL_COUNT = 8; + + private static final Format MHM1_FORMAT = + new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_MPEGH_MHM1) + .setSampleRate(48000) + .setChannelCount(2) + .build(); + + @Test + public void constructor_setsOffloadModeEnabledOnAudioSink() { + AudioSink mockSink = mock(AudioSink.class); + + new MpeghAudioRenderer(/* eventHandler= */ null, /* eventListener= */ null, mockSink); + + verify(mockSink).setOffloadMode(AudioSink.OFFLOAD_MODE_ENABLED_GAPLESS_NOT_REQUIRED); + } + + @Test + @Config(minSdk = 31, shadows = {ShadowMpeghLibrary.class}) + public void createDecoder_withMpeghBl3DirectSupport_selectsMpeghPassThroughDecoder() + throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + setupMpeghDirectPlaybackShadow(context); + + DefaultAudioSink audioSink = new DefaultAudioSink.Builder(context).build(); + MpeghAudioRenderer renderer = new MpeghAudioRenderer(null, null, audioSink); + + renderer.supportsFormat(MHM1_FORMAT); + MpeghBaseDecoder decoder = renderer.createDecoder(MHM1_FORMAT, null); + + assertThat(decoder.getName()).isEqualTo("MpeghPassThroughDecoder"); + assertThat(decoder).isInstanceOf(MpeghPassThroughDecoder.class); + } + + @Test + @Config(minSdk = 31, shadows = {ShadowMpeghDecoderJni.class, ShadowMpeghLibrary.class}) + public void createDecoder_withoutMpeghDirectSupport_selectsMpeghDecoder() throws Exception { + // No shadow setup — direct playback is not supported by default. + Context context = ApplicationProvider.getApplicationContext(); + + DefaultAudioSink audioSink = new DefaultAudioSink.Builder(context).build(); + MpeghAudioRenderer renderer = new MpeghAudioRenderer(null, null, audioSink); + + renderer.supportsFormat(MHM1_FORMAT); + MpeghBaseDecoder decoder = renderer.createDecoder(MHM1_FORMAT, null); + + assertThat(decoder.getName()).isEqualTo("libmpegh"); + assertThat(decoder).isInstanceOf(MpeghDecoder.class); + } + + // --------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------- + + /** + * Configures the Robolectric audio system so that {@link DefaultAudioSink#getFormatSupport} + * returns {@link AudioSink#SINK_FORMAT_SUPPORTED_DIRECTLY} for MPEGH BL L3. + * + *

On API 33+, {@link androidx.media3.exoplayer.audio.AudioCapabilities} queries + * {@code AudioManager.getDirectProfilesForAttributes()} only for TV/automotive devices. We + * therefore set TV mode via {@link UiModeManager} and register a virtual output device whose + * profiles include {@code ENCODING_MPEGH_BL_L3} via {@link ShadowAudioManager}. + */ + @RequiresApi(31) + @SuppressWarnings({"StickyBroadcast", "MissingPermission"}) + private static void setupMpeghDirectPlaybackShadow(Context context) { + UiModeManager uiModeManager = + (UiModeManager) context.getSystemService(Context.UI_MODE_SERVICE); + shadowOf(uiModeManager).setCurrentModeType(Configuration.UI_MODE_TYPE_TELEVISION); + + AudioManager audioManager = (AudioManager) context.getSystemService(Context.AUDIO_SERVICE); + ShadowAudioManager shadowAudioManager = shadowOf(audioManager); + + AudioProfile mpeghProfile = + AudioProfileBuilder.newBuilder() + .setFormat(AudioFormat.ENCODING_MPEGH_BL_L3) + .setSamplingRates(new int[]{48_000}) + .setChannelMasks(new int[]{AudioFormat.CHANNEL_OUT_STEREO}) + .setChannelIndexMasks(new int[]{}) + .setEncapsulationType(AudioProfile.AUDIO_ENCAPSULATION_TYPE_NONE) + .build(); + AudioProfile pcmProfile = + AudioProfileBuilder.newBuilder() + .setFormat(AudioFormat.ENCODING_PCM_16BIT) + .setSamplingRates(new int[]{48_000}) + .setChannelMasks(new int[]{AudioFormat.CHANNEL_OUT_STEREO}) + .setChannelIndexMasks(new int[]{}) + .setEncapsulationType(AudioProfile.AUDIO_ENCAPSULATION_TYPE_NONE) + .build(); + AudioDeviceInfo device = + AudioDeviceInfoBuilder.newBuilder() + .setType(AudioDeviceInfo.TYPE_HDMI) + .setProfiles(ImmutableList.of(mpeghProfile, pcmProfile)) + .build(); + + shadowAudioManager.addOutputDevice(device, /* notifyAudioDeviceCallbacks= */ false); + shadowAudioManager.addOutputDeviceWithDirectProfiles(device); + + if (SDK_INT < 33) { + AudioFormat audioFormat = + new AudioFormat.Builder() + .setSampleRate(48000) + .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO) + .setEncoding(AudioFormat.ENCODING_MPEGH_BL_L3) + .build(); + AudioAttributes audioAttributes = + new AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_UNKNOWN) + .setUsage(C.USAGE_MEDIA) + .build(); + + ShadowAudioSystem.setOffloadSupported(audioFormat, audioAttributes.getPlatformAudioAttributes(), + true); + ShadowAudioSystem.setOffloadPlaybackSupport( + audioFormat, + audioAttributes.getPlatformAudioAttributes(), + AudioManager.PLAYBACK_OFFLOAD_SUPPORTED); + + ShadowAudioTrack.addAllowedNonPcmEncoding(AudioFormat.ENCODING_MPEGH_BL_L3); + ShadowAudioTrack.addDirectPlaybackSupport( + audioFormat, + audioAttributes.getPlatformAudioAttributes()); + Intent intent = new Intent(AudioManager.ACTION_HDMI_AUDIO_PLUG); + intent.putExtra(AudioManager.EXTRA_AUDIO_PLUG_STATE, 1); + intent.putExtra( + AudioManager.EXTRA_ENCODINGS, + new int[] {MimeTypes.getEncoding(MimeTypes.AUDIO_MPEGH_MHM1, /* codec= */ null)}); + intent.putExtra(AudioManager.EXTRA_MAX_CHANNEL_COUNT, DEFAULT_MAX_CHANNEL_COUNT); + ApplicationProvider.getApplicationContext().sendStickyBroadcast(intent); + } + } + + /** + * Robolectric shadow that stubs the necessary JNI methods of {@link MpeghDecoderJni}, allowing {@link + * MpeghDecoder} to be instantiated in unit tests without the native library present. + */ + @Implements(value = MpeghDecoderJni.class, callThroughByDefault = false) + public static final class ShadowMpeghDecoderJni { + public ShadowMpeghDecoderJni() {} + + @Implementation + protected void init(int cicpIndex, byte[] mhaConfig, int mhaConfigLength) {} + } + + /** + * Robolectric shadow that makes {@link MpeghLibrary#isAvailable()} return {@code true} without + * loading the native library. + */ + @Implements(value = MpeghLibrary.class) + public static final class ShadowMpeghLibrary { + @Implementation + public static boolean isAvailable() { + return true; + } + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java index 19aad7639b0..1716e7b1af8 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/AudioTrackAudioOutput.java @@ -40,11 +40,12 @@ import androidx.media3.common.util.Clock; import androidx.media3.common.util.ListenerSet; import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.exoplayer.analytics.PlayerId; import androidx.media3.exoplayer.audio.AudioOutputProvider.OutputConfig; -import androidx.media3.extractor.ts.MpeghUtil; +import androidx.media3.extractor.MpeghUtil; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.concurrent.Future; @@ -113,6 +114,8 @@ public interface CapabilityChangeListener { private int lastUnderrunCount; private boolean hasData; + private final ParsableByteArray mpeghScratchBytes; + /** * @deprecated Use {@link * #AudioTrackAudioOutput(AudioTrack,OutputConfig,CapabilityChangeListener,float,Clock)} @@ -175,6 +178,8 @@ public AudioTrackAudioOutput( new OnRoutingChangedListenerApi24(audioTrack, capabilityChangeListener); } offloadStreamEventCallbackV29 = isOffloadedPlayback() ? new StreamEventCallbackV29() : null; + + mpeghScratchBytes = new ParsableByteArray(); } /** Returns the {@link AudioTrack} instance used for audio output. */ @@ -246,9 +251,10 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present // If this is the first encoded sample, calculate the sample size in frames. framesPerEncodedSample = DefaultAudioSink.getFramesPerEncodedSample(config.encoding, buffer); } - int truncationSamples = 0; if (isMpegh && buffer.remaining() == buffer.limit()) { - truncationSamples = MpeghUtil.getTruncationSampleCount(buffer); + mpeghScratchBytes.reset(buffer.remaining()); + buffer.get(mpeghScratchBytes.getData(), 0, buffer.remaining()); + buffer.rewind(); } maybeReportUnderrun(); int bytesRemaining = buffer.remaining(); @@ -284,6 +290,10 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present } else if (fullyHandled) { // For non-PCM we can only be sure about the number of written frames once the entire buffer // is submitted. + int truncationSamples = 0; + if (isMpegh) { + truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); + } writtenEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; } return fullyHandled; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 3fa50768cca..266848e2caa 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -52,6 +52,7 @@ import androidx.media3.common.util.Clock; import androidx.media3.common.util.ExperimentalApi; import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.container.OpusUtil; @@ -66,7 +67,7 @@ import androidx.media3.extractor.DtsUtil; import androidx.media3.extractor.ExtractorUtil; import androidx.media3.extractor.MpegAudioUtil; -import androidx.media3.extractor.ts.MpeghUtil; +import androidx.media3.extractor.MpeghUtil; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.lang.annotation.Documented; @@ -607,6 +608,8 @@ public DefaultAudioSink build() { private long outputStreamOffsetUs; private float volume; + private final ParsableByteArray mpeghScratchBytes; + @Nullable private ByteBuffer inputBuffer; private int inputBufferAccessUnitCount; @Nullable private ByteBuffer outputBuffer; @@ -645,6 +648,7 @@ private DefaultAudioSink(Builder builder) { availableAudioProcessors = ImmutableList.of(trimmingAudioProcessor, channelMappingAudioProcessor); volume = 1f; + mpeghScratchBytes = new ParsableByteArray(); audioSessionId = C.AUDIO_SESSION_ID_UNSET; auxEffectInfo = new AuxEffectInfo(AuxEffectInfo.NO_AUX_EFFECT_ID, 0f); mediaPositionParameters = @@ -1023,9 +1027,10 @@ public boolean handleBuffer( } } - int truncationSamples = 0; - if (Util.isMpegh(configuration.outputConfig.encoding)) { - truncationSamples += MpeghUtil.getTruncationSampleCount(buffer); + if (Util.isMpegh(configuration.outputConfig.encoding) && buffer.remaining() == buffer.limit()) { + mpeghScratchBytes.reset(buffer.remaining()); + buffer.get(mpeghScratchBytes.getData(), 0, buffer.remaining()); + buffer.rewind(); } if (afterDrainParameters != null) { @@ -1071,6 +1076,10 @@ public boolean handleBuffer( if (configuration.isPcm()) { submittedPcmBytes += buffer.remaining(); } else { + int truncationSamples = 0; + if (Util.isMpegh(configuration.outputConfig.encoding)) { + truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); + } submittedEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; } @@ -1269,9 +1278,10 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept if (outputBuffer == null) { return; } - int truncationSamples = 0; if (Util.isMpegh(configuration.outputConfig.encoding) && outputBuffer.remaining() == outputBuffer.limit()) { - truncationSamples = MpeghUtil.getTruncationSampleCount(outputBuffer); + mpeghScratchBytes.reset(outputBuffer.remaining()); + outputBuffer.get(mpeghScratchBytes.getData(), 0, outputBuffer.remaining()); + outputBuffer.rewind(); } if (writeExceptionPendingExceptionHolder.shouldWaitBeforeRetry()) { return; @@ -1338,19 +1348,13 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept checkState(outputBuffer == inputBuffer); int truncationSamples = 0; if (Util.isMpegh(configuration.outputConfig.encoding)) { - ParsableByteArray byteBuffer = new ParsableByteArray(); - int bufferLimit = outputBuffer.limit(); - byteBuffer.reset(bufferLimit); - outputBuffer.get(byteBuffer.getData(), 0, bufferLimit); - outputBuffer.position(0); - truncationSamples = MpeghUtil.getTruncationSampleCount(byteBuffer); + truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); } writtenEncodedFrames += ((long) framesPerEncodedSample * inputBufferAccessUnitCount) - truncationSamples - currentBufferFramesWritten; - currentBufferFramesWritten = 0; } - outputBuffer = null; + currentBufferFramesWritten = 0; } outputBuffer = null; } else { if (!configuration.isPcm() && bytesWritten > 0) { checkState(inputBufferOriginalRemaining > 0); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java index 780d658b6c7..91ff0b63da2 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java @@ -55,6 +55,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import com.google.common.primitives.ImmutableIntArray; +import java.lang.reflect.Field; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; @@ -1763,6 +1764,93 @@ private void configureDefaultAudioSinkWithOffload() throws AudioSink.Configurati defaultAudioSink.configure(new AudioSink.AudioSinkConfig.Builder(format).build()); } + private void configureDefaultAudioSinkWithOffload_Mpegh() throws AudioSink.ConfigurationException { + Format format = + new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_MPEGH_MHM1) + .setChannelCount(/* channelCount= */ 2) + .setSampleRate(48000) + .setCodecs("mhm1") + .build(); + + defaultAudioSink.setOffloadMode(AudioSink.OFFLOAD_MODE_ENABLED_GAPLESS_NOT_REQUIRED); + defaultAudioSink.configure(new AudioSink.AudioSinkConfig.Builder(format).build()); + } + + private static final String MPEGH_TRUNC_PACKET0 = "e1480283e0"; + private static final String MPEGH_3DAFRAME_PACKET0 = "48673000d80ab1020220000f4ce0000000000c6952969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969695"; + private static final String MPEGH_TRUNCATION_FRAME = MPEGH_TRUNC_PACKET0 + MPEGH_3DAFRAME_PACKET0; + + private static final String MPEGH_SYNC_PACKET1 = "c001a5"; + private static final String MPEGH_3DACONFIG_PACKET1 = "30410b1940800101366069e2d01110088120440110000207fd8020108000103664015900c400008190028002080000bbf8040029aec00a5008000a3a04638b00e40020"; + private static final String MPEGH_ASI_PACKET1 = "70378080c01000080000248400060000000001e000032b73383a1b430b73732b62100a00180000cadcce0aa6c6cadcca84c00080fc01bc0000"; + private static final String MPEGH_BUFFERINFO_PACKET1 = "e0f00100"; + private static final String MPEGH_MARKER_PACKET1 = "e03006e0005c73e212"; + private static final String MPEGH_TRUNC_PACKET1 = "e15002a020"; + private static final String MPEGH_3DAFRAME_PACKET1 = "5132d020200fd40b5a5a401400b05ccc0000007ffd377dfb80fdebdfc29aa83fa2b44d25293baf850d9c0c08102006fd4aa35410ffc466181cb8259f2883bfa798f29c138003cf44589fb89ecea9e697b148eff82fadcfbbeb4e756625a4841ec31be87c2bf1f1fd90c7472217f2942c3fd12574b69a3014e883de42c146c9e2c889ffe158026935e513c9dd2a9bc3c4017b04a541f20ca47c7704790bdd303d55ea819aada8ac811740670457ac471163c734adcb6815e457d65c9100b5e1f42c1c0216a860bc9948255bbdbc0d0d085af3beda99c234d2de38e8bfe00b98307ad2c78791fd9875b392e72bea1e259a6686392bd7e14b3b0b0621007751dcbedbcdc8724577124553438f4c9b38b608e3b81669e80887e2331b1867f8f0868e28cff81ffe261cee52cb86283001f44010b0efdc"; + private static final String MPEGH_CONFIG_FRAME = + MPEGH_SYNC_PACKET1 + MPEGH_3DACONFIG_PACKET1 + MPEGH_ASI_PACKET1 + MPEGH_BUFFERINFO_PACKET1 + + MPEGH_MARKER_PACKET1 + MPEGH_TRUNC_PACKET1 + MPEGH_3DAFRAME_PACKET1; + + @Config(minSdk = 31) + @Test + public void getAudioTrackBufferDurationUs_withMpegh() throws Exception { + + getShadowUiModeManager().setCurrentModeType(Configuration.UI_MODE_TYPE_TELEVISION); + addHdmiDevice(); + + AudioFormat audioFormat = + new AudioFormat.Builder() + .setSampleRate(48000) + .setChannelMask(AudioFormat.CHANNEL_OUT_STEREO) + .setEncoding(AudioFormat.ENCODING_MPEGH_BL_L3) + .build(); + AudioAttributes audioAttributes = + new AudioAttributes.Builder() + .setContentType(C.AUDIO_CONTENT_TYPE_UNKNOWN) + .setUsage(C.USAGE_MEDIA) + .setAllowedCapturePolicy(C.ALLOW_CAPTURE_BY_ALL) + .build(); + ShadowAudioSystem.setOffloadSupported( + audioFormat, audioAttributes.getPlatformAudioAttributes(), true); + ShadowAudioSystem.setOffloadPlaybackSupport( + audioFormat, + audioAttributes.getPlatformAudioAttributes(), + AudioManager.PLAYBACK_OFFLOAD_SUPPORTED); + ShadowAudioSystem.setDirectPlaybackSupport( + audioFormat, + audioAttributes.getPlatformAudioAttributes(), + AudioManager.DIRECT_PLAYBACK_OFFLOAD_SUPPORTED); + + configureDefaultAudioSinkWithOffload_Mpegh(); + + byte[] data1 = Util.getBytesFromHexString(MPEGH_CONFIG_FRAME); + ByteBuffer buffer = ByteBuffer.wrap(data1); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + assertThat( + defaultAudioSink.handleBuffer( + buffer, + /* presentationTimeUs= */ 0, + /* encodedAccessUnitCount= */ 1)) + .isTrue(); + Field submittedField = defaultAudioSink.getClass().getDeclaredField("submittedEncodedFrames"); + submittedField.setAccessible(true); + assertThat(submittedField.getLong(defaultAudioSink)).isEqualTo(992); + + byte[] data2 = Util.getBytesFromHexString(MPEGH_TRUNCATION_FRAME); + buffer = ByteBuffer.wrap(data2); + buffer.order(ByteOrder.LITTLE_ENDIAN); + + assertThat( + defaultAudioSink.handleBuffer( + buffer, + /* presentationTimeUs= */ 0, + /* encodedAccessUnitCount= */ 1)) + .isTrue(); + assertThat(submittedField.getLong(defaultAudioSink)).isEqualTo(1024); + } + // Adding the permission to the test AndroidManifest.xml doesn't work to appease lint. @SuppressWarnings({"StickyBroadcast", "MissingPermission"}) private void addHdmiDevice() { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java index 2746f3e5d2e..76187811433 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ExtractorUtil.java @@ -21,7 +21,6 @@ import androidx.media3.common.ParserException; import androidx.media3.common.util.UnstableApi; import androidx.media3.container.OpusUtil; -import androidx.media3.extractor.ts.MpeghUtil; import java.io.EOFException; import java.io.IOException; import org.checkerframework.dataflow.qual.Pure; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java similarity index 98% rename from libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java rename to libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java index 3b6042a2815..676293f7e53 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package androidx.media3.extractor.ts; +package androidx.media3.extractor; import static com.google.common.base.Preconditions.checkArgument; import static java.lang.annotation.ElementType.TYPE_USE; @@ -23,6 +23,7 @@ import androidx.media3.common.C; import androidx.media3.common.ParserException; import androidx.media3.common.util.ParsableBitArray; +import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import com.google.common.math.IntMath; import com.google.common.math.LongMath; @@ -37,7 +38,7 @@ public final class MpeghUtil { /** - * Maximum rate for an MPEG-H audio stream, in bytes per second. + * Maximum rate for an MPEG-H audio stream, in bytes per second. * 56 channels × 288 kbps/channel = 16128 kbps -> 2016000 bytes/s * assumes 1024-sample frame length at 48000 Hz for MPEG-H BL L4 maximum */ @@ -753,14 +754,9 @@ private Mpegh3daConfig( * @param buffer The data to parse, containing complete MPEG-H access units * @return The number of truncated samples. */ - public static int getTruncationSampleCount(ByteBuffer buffer) { + public static int getTruncationSampleCount(ParsableByteArray buffer) { int truncationSamples = 0; - int bufferPos = buffer.position(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - buffer.position(bufferPos); - ParsableBitArray bitArray = new ParsableBitArray(bytes); - + ParsableBitArray bitArray = new ParsableBitArray(buffer.getData(), buffer.limit()); MhasPacketHeader header = new MhasPacketHeader(); while (bitArray.bitsLeft() > 0) { try { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghReader.java b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghReader.java index acf5f616432..a067c2da241 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghReader.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/ts/MpeghReader.java @@ -32,6 +32,7 @@ import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.MpeghUtil; import androidx.media3.extractor.TrackOutput; import com.google.common.collect.ImmutableList; import java.lang.annotation.Documented; diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java new file mode 100644 index 00000000000..2334fe4b0f3 --- /dev/null +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java @@ -0,0 +1,250 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.extractor; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.media3.common.util.ParsableBitArray; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.nio.ByteBuffer; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link MpeghUtil}. */ +@RunWith(AndroidJUnit4.class) +public final class MpeghUtilTest { + + private static final String MPEGH_TRUNC_PACKET0 = "e1480283e0"; + private static final String MPEGH_3DAFRAME_PACKET0 = "48673000d80ab1020220000f4ce0000000000c6952969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969696969695"; + private static final String MPEGH_TRUNCATION_FRAME = MPEGH_TRUNC_PACKET0 + MPEGH_3DAFRAME_PACKET0; + + private static final String MPEGH_SYNC_PACKET1 = "c001a5"; + private static final String MPEGH_3DACONFIG_PACKET1 = "30410b1940800101366069e2d01110088120440110000207fd8020108000103664015900c400008190028002080000bbf8040029aec00a5008000a3a04638b00e40020"; + private static final String MPEGH_ASI_PACKET1 = "70378080c01000080000248400060000000001e000032b73383a1b430b73732b62100a00180000cadcce0aa6c6cadcca84c00080fc01bc0000"; + private static final String MPEGH_BUFFERINFO_PACKET1 = "e0f00100"; + private static final String MPEGH_MARKER_PACKET1 = "e03006e0005c73e212"; + private static final String MPEGH_TRUNC_PACKET1 = "e15002a020"; + private static final String MPEGH_3DAFRAME_PACKET1 = "5132d020200fd40b5a5a401400b05ccc0000007ffd377dfb80fdebdfc29aa83fa2b44d25293baf850d9c0c08102006fd4aa35410ffc466181cb8259f2883bfa798f29c138003cf44589fb89ecea9e697b148eff82fadcfbbeb4e756625a4841ec31be87c2bf1f1fd90c7472217f2942c3fd12574b69a3014e883de42c146c9e2c889ffe158026935e513c9dd2a9bc3c4017b04a541f20ca47c7704790bdd303d55ea819aada8ac811740670457ac471163c734adcb6815e457d65c9100b5e1f42c1c0216a860bc9948255bbdbc0d0d085af3beda99c234d2de38e8bfe00b98307ad2c78791fd9875b392e72bea1e259a6686392bd7e14b3b0b0621007751dcbedbcdc8724577124553438f4c9b38b608e3b81669e80887e2331b1867f8f0868e28cff81ffe261cee52cb86283001f44010b0efdc"; + private static final String MPEGH_CONFIG_FRAME = + MPEGH_SYNC_PACKET1 + MPEGH_3DACONFIG_PACKET1 + MPEGH_ASI_PACKET1 + MPEGH_BUFFERINFO_PACKET1 + + MPEGH_MARKER_PACKET1 + MPEGH_TRUNC_PACKET1 + MPEGH_3DAFRAME_PACKET1; + + private static final String MPEGH_COMBINED_FRAMES = MPEGH_TRUNCATION_FRAME + MPEGH_CONFIG_FRAME; + + + @Test + public void parseStandardFrameLength_configFrame() { + byte[] data = Util.getBytesFromHexString(MPEGH_CONFIG_FRAME); + ByteBuffer buffer = ByteBuffer.wrap(data); + int frameLength = MpeghUtil.getStandardFrameLength(buffer); + assertThat(frameLength).isEqualTo(1024); + } + + @Test + public void parseStandardFrameLength_combinedFrames() { + byte[] data = Util.getBytesFromHexString(MPEGH_COMBINED_FRAMES); + ByteBuffer buffer = ByteBuffer.wrap(data); + int frameLength = MpeghUtil.getStandardFrameLength(buffer); + assertThat(frameLength).isEqualTo(1024); + } + + @Test + public void parseStandardFrameLength_3daConfigPacket() { + byte[] data = Util.getBytesFromHexString(MPEGH_3DACONFIG_PACKET1); + ByteBuffer buffer = ByteBuffer.wrap(data); + int frameLength = MpeghUtil.getStandardFrameLength(buffer); + assertThat(frameLength).isEqualTo(1024); + } + + @Test + public void parseTruncationSampleCount_configFrame() { + byte[] data = Util.getBytesFromHexString(MPEGH_CONFIG_FRAME); + ParsableByteArray byteArray = new ParsableByteArray(data); + int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + assertThat(truncationSamples).isEqualTo(32); + } + + @Test + public void parseTruncationSampleCount_truncationFrame() { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNCATION_FRAME); + ParsableByteArray byteArray = new ParsableByteArray(data); + int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + assertThat(truncationSamples).isEqualTo(992); + } + + @Test + public void parseTruncationSampleCount_combinedFrames() { + byte[] data = Util.getBytesFromHexString(MPEGH_COMBINED_FRAMES); + ParsableByteArray byteArray = new ParsableByteArray(data); + int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + assertThat(truncationSamples).isEqualTo(1024); + } + + @Test + public void parseTruncationSampleCount_truncationPacket0() { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET0); + ParsableByteArray byteArray = new ParsableByteArray(data); + int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + assertThat(truncationSamples).isEqualTo(992); + } + + @Test + public void parseTruncationSampleCount_truncationPacket1() { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET1); + ParsableByteArray byteArray = new ParsableByteArray(data); + int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + assertThat(truncationSamples).isEqualTo(32); + } + + @Test + public void parseMhasPacketHeader_syncPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_SYNC_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_SYNC); + assertThat(header.packetLength).isEqualTo(1); + assertThat(header.packetLabel).isEqualTo(0); + } + + @Test + public void parseMhasPacketHeader_3daConfigPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_3DACONFIG_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_MPEGH3DACFG); + assertThat(header.packetLength).isEqualTo(65); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMhasPacketHeader_audioSceneInfoPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_ASI_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_AUDIOSCENEINFO); + assertThat(header.packetLength).isEqualTo(55); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMhasPacketHeader_bufferInfoPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_BUFFERINFO_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_BUFFERINFO); + assertThat(header.packetLength).isEqualTo(1); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMhasPacketHeader_markerPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_MARKER_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_MARKER); + assertThat(header.packetLength).isEqualTo(6); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMhasPacketHeader_truncationPacket0() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET0); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_AUDIOTRUNCATION); + assertThat(header.packetLength).isEqualTo(2); + assertThat(header.packetLabel).isEqualTo(1); + } + + @Test + public void parseMhasPacketHeader_truncationPacket1() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_AUDIOTRUNCATION); + assertThat(header.packetLength).isEqualTo(2); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMhasPacketHeader_3daFramePacket0() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_3DAFRAME_PACKET0); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_MPEGH3DAFRAME); + assertThat(header.packetLength).isEqualTo(103); + assertThat(header.packetLabel).isEqualTo(1); + } + + @Test + public void parseMhasPacketHeader_3daFramePacket1() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_3DAFRAME_PACKET1); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.MhasPacketHeader header = new MpeghUtil.MhasPacketHeader(); + boolean success = MpeghUtil.parseMhasPacketHeader(bitArray, header); + assertThat(success).isEqualTo(true); + assertThat(header.packetType).isEqualTo(MpeghUtil.MhasPacketHeader.PACTYP_MPEGH3DAFRAME); + assertThat(header.packetLength).isEqualTo(306); + assertThat(header.packetLabel).isEqualTo(2); + } + + @Test + public void parseMpegh3daConfig_3daConfigPacket() throws Exception { + byte[] data = Util.getBytesFromHexString(MPEGH_3DACONFIG_PACKET1.substring(4)); + ParsableBitArray bitArray = new ParsableBitArray(data); + MpeghUtil.Mpegh3daConfig config = MpeghUtil.parseMpegh3daConfig(bitArray); + assertThat(config.standardFrameLength).isEqualTo(1024); + assertThat(config.samplingFrequency).isEqualTo(48000); + assertThat(config.profileLevelIndication).isEqualTo(11); + byte[] compat = new byte[1]; + compat[0] = 16; + assertThat(config.compatibleProfileLevelSet).isEqualTo(compat); + } + + @Test + public void parseAudioTruncationInfo_truncationPacket0() { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET0.substring(6)); + ParsableBitArray bitArray = new ParsableBitArray(data); + int truncSamples = MpeghUtil.parseAudioTruncationInfo(bitArray); + assertThat(truncSamples).isEqualTo(992); + } + + @Test + public void parseAudioTruncationInfo_truncationPacket1() { + byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET1.substring(6)); + ParsableBitArray bitArray = new ParsableBitArray(data); + int truncSamples = MpeghUtil.parseAudioTruncationInfo(bitArray); + assertThat(truncSamples).isEqualTo(32); + } +} From 7aa3d3d45b7eddbecaf667c123856d995b628fc7 Mon Sep 17 00:00:00 2001 From: Rohit Singh Date: Tue, 4 Aug 2026 17:23:32 +0100 Subject: [PATCH 3/3] Refactor MPEG-H abstraction to meet Media3 standards --- .../decoder/mpegh/MpeghAudioRenderer.java | 2 +- .../audio/AudioTrackAudioOutput.java | 22 +++-------- .../exoplayer/audio/DefaultAudioSink.java | 39 ++++++++----------- .../exoplayer/audio/DefaultAudioSinkTest.java | 4 +- .../androidx/media3/extractor/MpeghUtil.java | 21 +++++----- .../media3/extractor/MpeghUtilTest.java | 10 ++--- 6 files changed, 39 insertions(+), 59 deletions(-) diff --git a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java index 810bbf4e056..e3e797ae16d 100644 --- a/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java +++ b/libraries/decoder_mpegh/src/main/java/androidx/media3/decoder/mpegh/MpeghAudioRenderer.java @@ -61,7 +61,7 @@ public final class MpeghAudioRenderer extends DecoderAudioRenderer listeners; @@ -114,8 +112,6 @@ public interface CapabilityChangeListener { private int lastUnderrunCount; private boolean hasData; - private final ParsableByteArray mpeghScratchBytes; - /** * @deprecated Use {@link * #AudioTrackAudioOutput(AudioTrack,OutputConfig,CapabilityChangeListener,float,Clock)} @@ -160,8 +156,7 @@ public AudioTrackAudioOutput( int channelCount = Integer.bitCount(config.channelMask); pcmFrameSize = Util.getPcmFrameSize(config.encoding, channelCount); } else { - isMpegh = Util.isMpegh(config.encoding); - pcmFrameSize = C.LENGTH_UNSET; + pcmFrameSize = C.LENGTH_UNSET; } audioTrackPositionTracker = @@ -178,8 +173,6 @@ public AudioTrackAudioOutput( new OnRoutingChangedListenerApi24(audioTrack, capabilityChangeListener); } offloadStreamEventCallbackV29 = isOffloadedPlayback() ? new StreamEventCallbackV29() : null; - - mpeghScratchBytes = new ParsableByteArray(); } /** Returns the {@link AudioTrack} instance used for audio output. */ @@ -251,11 +244,6 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present // If this is the first encoded sample, calculate the sample size in frames. framesPerEncodedSample = DefaultAudioSink.getFramesPerEncodedSample(config.encoding, buffer); } - if (isMpegh && buffer.remaining() == buffer.limit()) { - mpeghScratchBytes.reset(buffer.remaining()); - buffer.get(mpeghScratchBytes.getData(), 0, buffer.remaining()); - buffer.rewind(); - } maybeReportUnderrun(); int bytesRemaining = buffer.remaining(); int bytesWrittenOrError; @@ -290,11 +278,11 @@ public boolean write(ByteBuffer buffer, int encodedAccessUnitCount, long present } else if (fullyHandled) { // For non-PCM we can only be sure about the number of written frames once the entire buffer // is submitted. - int truncationSamples = 0; - if (isMpegh) { - truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); + int paddingSamples = 0; + if (Util.isMpegh(config.encoding)) { + paddingSamples = MpeghUtil.getTruncationSampleCount(buffer); } - writtenEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; + writtenEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - paddingSamples; } return fullyHandled; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 266848e2caa..dd2e4599de4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -52,7 +52,6 @@ import androidx.media3.common.util.Clock; import androidx.media3.common.util.ExperimentalApi; import androidx.media3.common.util.Log; -import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.container.OpusUtil; @@ -608,7 +607,6 @@ public DefaultAudioSink build() { private long outputStreamOffsetUs; private float volume; - private final ParsableByteArray mpeghScratchBytes; @Nullable private ByteBuffer inputBuffer; private int inputBufferAccessUnitCount; @@ -648,7 +646,6 @@ private DefaultAudioSink(Builder builder) { availableAudioProcessors = ImmutableList.of(trimmingAudioProcessor, channelMappingAudioProcessor); volume = 1f; - mpeghScratchBytes = new ParsableByteArray(); audioSessionId = C.AUDIO_SESSION_ID_UNSET; auxEffectInfo = new AuxEffectInfo(AuxEffectInfo.NO_AUX_EFFECT_ID, 0f); mediaPositionParameters = @@ -1027,11 +1024,6 @@ public boolean handleBuffer( } } - if (Util.isMpegh(configuration.outputConfig.encoding) && buffer.remaining() == buffer.limit()) { - mpeghScratchBytes.reset(buffer.remaining()); - buffer.get(mpeghScratchBytes.getData(), 0, buffer.remaining()); - buffer.rewind(); - } if (afterDrainParameters != null) { if (!drainToEndOfStream()) { @@ -1076,11 +1068,8 @@ public boolean handleBuffer( if (configuration.isPcm()) { submittedPcmBytes += buffer.remaining(); } else { - int truncationSamples = 0; - if (Util.isMpegh(configuration.outputConfig.encoding)) { - truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); - } - submittedEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - truncationSamples; + int paddingSamples = getPerBufferTruncationSamples(configuration.outputConfig.encoding, buffer); + submittedEncodedFrames += (long) framesPerEncodedSample * encodedAccessUnitCount - paddingSamples; } inputBuffer = buffer; @@ -1278,11 +1267,6 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept if (outputBuffer == null) { return; } - if (Util.isMpegh(configuration.outputConfig.encoding) && outputBuffer.remaining() == outputBuffer.limit()) { - mpeghScratchBytes.reset(outputBuffer.remaining()); - outputBuffer.get(mpeghScratchBytes.getData(), 0, outputBuffer.remaining()); - outputBuffer.rewind(); - } if (writeExceptionPendingExceptionHolder.shouldWaitBeforeRetry()) { return; } @@ -1346,13 +1330,10 @@ private void drainOutputBuffer(long avSyncPresentationTimeUs) throws WriteExcept // When playing non-PCM, the inputBuffer is never processed, thus the last inputBuffer // must be the current input buffer. checkState(outputBuffer == inputBuffer); - int truncationSamples = 0; - if (Util.isMpegh(configuration.outputConfig.encoding)) { - truncationSamples = MpeghUtil.getTruncationSampleCount(mpeghScratchBytes); - } + int paddingSamples = getPerBufferTruncationSamples(configuration.outputConfig.encoding, outputBuffer); writtenEncodedFrames += ((long) framesPerEncodedSample * inputBufferAccessUnitCount) - - truncationSamples + - paddingSamples - currentBufferFramesWritten; currentBufferFramesWritten = 0; } outputBuffer = null; } else { @@ -1888,6 +1869,18 @@ private FormatConfig getFormatConfig(Format format, int preferredBufferSize) { .build(); } + private static int getPerBufferTruncationSamples(@C.Encoding int encoding, ByteBuffer buffer) { + switch (encoding) { + case C.ENCODING_MPEGH_BL_L3: + case C.ENCODING_MPEGH_BL_L4: + case C.ENCODING_MPEGH_LC_L3: + case C.ENCODING_MPEGH_LC_L4: + return MpeghUtil.getTruncationSampleCount(buffer); + default: + return 0; + } + } + /* package */ static int getFramesPerEncodedSample(@C.Encoding int encoding, ByteBuffer buffer) { switch (encoding) { case C.ENCODING_MP3: diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java index 91ff0b63da2..32bfb76c8cd 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DefaultAudioSinkTest.java @@ -1764,7 +1764,7 @@ private void configureDefaultAudioSinkWithOffload() throws AudioSink.Configurati defaultAudioSink.configure(new AudioSink.AudioSinkConfig.Builder(format).build()); } - private void configureDefaultAudioSinkWithOffload_Mpegh() throws AudioSink.ConfigurationException { + private void configureDefaultAudioSinkWithOffloadMpegh() throws AudioSink.ConfigurationException { Format format = new Format.Builder() .setSampleMimeType(MimeTypes.AUDIO_MPEGH_MHM1) @@ -1822,7 +1822,7 @@ public void getAudioTrackBufferDurationUs_withMpegh() throws Exception { audioAttributes.getPlatformAudioAttributes(), AudioManager.DIRECT_PLAYBACK_OFFLOAD_SUPPORTED); - configureDefaultAudioSinkWithOffload_Mpegh(); + configureDefaultAudioSinkWithOffloadMpegh(); byte[] data1 = Util.getBytesFromHexString(MPEGH_CONFIG_FRAME); ByteBuffer buffer = ByteBuffer.wrap(data1); diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java b/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java index 676293f7e53..f1e18beffce 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/MpeghUtil.java @@ -23,7 +23,6 @@ import androidx.media3.common.C; import androidx.media3.common.ParserException; import androidx.media3.common.util.ParsableBitArray; -import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import com.google.common.math.IntMath; import com.google.common.math.LongMath; @@ -447,7 +446,7 @@ private static void skipMpegh3daDecoderConfig( switch (usacElementType) { case 0 /*ID_USAC_SCE*/: - parseMpegh3daCoreConfig(data); // coreConfig + boolean unusedCoreConfig = parseMpegh3daCoreConfig(data); // coreConfig if (sbrRatioIndex > 0) { skipSbrConfig(data); // sbrConfig } @@ -493,11 +492,11 @@ private static void skipMpegh3daDecoderConfig( } break; case 3 /*ID_USAC_EXT*/: - readEscapedIntValue(data, 4, 8, 16); // usacExtElementType + int unusedUsac = readEscapedIntValue(data, 4, 8, 16); // usacExtElementType int usacExtElementConfigLength = readEscapedIntValue(data, 4, 8, 16); if (data.readBit()) { // usacExtElementDefaultLengthPresent - readEscapedIntValue(data, 8, 16, 0) /* +1 */; // usacExtElementDefaultLength + int unusedUsacDef = readEscapedIntValue(data, 8, 16, 0) /* +1 */; // usacExtElementDefaultLength } data.skipBit(); // usacExtElementPayloadFrag @@ -754,9 +753,11 @@ private Mpegh3daConfig( * @param buffer The data to parse, containing complete MPEG-H access units * @return The number of truncated samples. */ - public static int getTruncationSampleCount(ParsableByteArray buffer) { + public static int getTruncationSampleCount(ByteBuffer buffer) { int truncationSamples = 0; - ParsableBitArray bitArray = new ParsableBitArray(buffer.getData(), buffer.limit()); + ParsableBitArray bitArray = + new ParsableBitArray(buffer.array(), buffer.limit()); + bitArray.setPosition(buffer.position() * 8); MhasPacketHeader header = new MhasPacketHeader(); while (bitArray.bitsLeft() > 0) { try { @@ -782,11 +783,9 @@ public static int getTruncationSampleCount(ParsableByteArray buffer) { * @return The standard audio frame length. */ public static int getStandardFrameLength(ByteBuffer buffer) { - int bufferPos = buffer.position(); - byte[] bytes = new byte[buffer.remaining()]; - buffer.get(bytes); - buffer.position(bufferPos); - ParsableBitArray bitArray = new ParsableBitArray(bytes); + ParsableBitArray bitArray = + new ParsableBitArray(buffer.array(), buffer.limit()); + bitArray.setPosition(buffer.position() * 8); MhasPacketHeader header = new MhasPacketHeader(); while (bitArray.bitsLeft() > 0) { diff --git a/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java b/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java index 2334fe4b0f3..de55211dede 100644 --- a/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java +++ b/libraries/extractor/src/test/java/androidx/media3/extractor/MpeghUtilTest.java @@ -75,7 +75,7 @@ public void parseStandardFrameLength_3daConfigPacket() { public void parseTruncationSampleCount_configFrame() { byte[] data = Util.getBytesFromHexString(MPEGH_CONFIG_FRAME); ParsableByteArray byteArray = new ParsableByteArray(data); - int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + int truncationSamples = MpeghUtil.getTruncationSampleCount(ByteBuffer.wrap(byteArray.getData(), 0, byteArray.limit())); assertThat(truncationSamples).isEqualTo(32); } @@ -83,7 +83,7 @@ public void parseTruncationSampleCount_configFrame() { public void parseTruncationSampleCount_truncationFrame() { byte[] data = Util.getBytesFromHexString(MPEGH_TRUNCATION_FRAME); ParsableByteArray byteArray = new ParsableByteArray(data); - int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + int truncationSamples = MpeghUtil.getTruncationSampleCount(ByteBuffer.wrap(byteArray.getData(), 0, byteArray.limit())); assertThat(truncationSamples).isEqualTo(992); } @@ -91,7 +91,7 @@ public void parseTruncationSampleCount_truncationFrame() { public void parseTruncationSampleCount_combinedFrames() { byte[] data = Util.getBytesFromHexString(MPEGH_COMBINED_FRAMES); ParsableByteArray byteArray = new ParsableByteArray(data); - int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + int truncationSamples = MpeghUtil.getTruncationSampleCount(ByteBuffer.wrap(byteArray.getData(), 0, byteArray.limit())); assertThat(truncationSamples).isEqualTo(1024); } @@ -99,7 +99,7 @@ public void parseTruncationSampleCount_combinedFrames() { public void parseTruncationSampleCount_truncationPacket0() { byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET0); ParsableByteArray byteArray = new ParsableByteArray(data); - int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + int truncationSamples = MpeghUtil.getTruncationSampleCount(ByteBuffer.wrap(byteArray.getData(), 0, byteArray.limit())); assertThat(truncationSamples).isEqualTo(992); } @@ -107,7 +107,7 @@ public void parseTruncationSampleCount_truncationPacket0() { public void parseTruncationSampleCount_truncationPacket1() { byte[] data = Util.getBytesFromHexString(MPEGH_TRUNC_PACKET1); ParsableByteArray byteArray = new ParsableByteArray(data); - int truncationSamples = MpeghUtil.getTruncationSampleCount(byteArray); + int truncationSamples = MpeghUtil.getTruncationSampleCount(ByteBuffer.wrap(byteArray.getData(), 0, byteArray.limit())); assertThat(truncationSamples).isEqualTo(32); }