diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java index 5922757ac3e..33033051881 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Format.java +++ b/libraries/common/src/main/java/androidx/media3/common/Format.java @@ -172,6 +172,7 @@ public static final class Builder { @Nullable private DrmInitData drmInitData; private long subsampleOffsetUs; private boolean hasPrerollSamples; + private boolean hasReliablePresentationTimestamps; // Video specific. @@ -218,6 +219,7 @@ public Builder() { maxInputSize = NO_VALUE; maxNumReorderSamples = NO_VALUE; subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE; + hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps // Video specific. width = NO_VALUE; height = NO_VALUE; @@ -270,6 +272,7 @@ private Builder(Format format) { this.drmInitData = format.drmInitData; this.subsampleOffsetUs = format.subsampleOffsetUs; this.hasPrerollSamples = format.hasPrerollSamples; + this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps; // Video specific. this.width = format.width; this.height = format.height; @@ -590,6 +593,20 @@ public Builder setHasPrerollSamples(boolean hasPrerollSamples) { return this; } + /** + * Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}. + * + * @param hasReliablePresentationTimestamps The {@link + * Format#hasReliablePresentationTimestamps}. + * @return The builder. + */ + @CanIgnoreReturnValue + public Builder setHasReliablePresentationTimestamps( + boolean hasReliablePresentationTimestamps) { + this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps; + return this; + } + // Video specific. /** @@ -1051,6 +1068,15 @@ public Format build() { */ @UnstableApi public final boolean hasPrerollSamples; + /** + * Indicates whether per-sample presentation timestamps for this track are known to be accurate. + * + *

When {@code false}, the source couldn't determine each sample's true presentation time + * (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the + * timestamps as ground truth for scheduling decisions. Defaults to {@code true}. + */ + @UnstableApi public final boolean hasReliablePresentationTimestamps; + // Video specific. /** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */ @@ -1218,6 +1244,7 @@ private Format(Builder builder) { drmInitData = builder.drmInitData; subsampleOffsetUs = builder.subsampleOffsetUs; hasPrerollSamples = builder.hasPrerollSamples; + hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps; // Video specific. width = builder.width; height = builder.height; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index cec7fdbc1e6..abbf079981e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -47,6 +47,7 @@ import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; @@ -417,6 +418,16 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; + // Used only when Format#hasReliablePresentationTimestamps is false: replays each sample's own + // queued presentationTimeUs in output-arrival order instead of trusting MediaCodec's echoed + // (possibly non-monotonic) timestamp. Preserves true per-sample duration, unlike deriving from + // a single assumed frame rate. + private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); + + @VisibleForTesting + /* package */ int getArrivalOrderPtsQueueSizeForTesting() { + return arrivalOrderPtsQueue.size(); + } /** * @param context A context. @@ -1112,6 +1123,7 @@ protected void resetCodecStateForFlush() { codecReconfigured ? RECONFIGURATION_STATE_WRITE_PENDING : RECONFIGURATION_STATE_NONE; hasSkippedFlushAndWaitingForQueueInputBuffer = false; skippedFlushOffsetUs = 0; + arrivalOrderPtsQueue.clear(); } /** @@ -1649,6 +1661,20 @@ private boolean feedInputBuffer() throws ExoPlaybackException { hasSkippedFlushAndWaitingForQueueInputBuffer = false; } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !getConfiguration().tunneling) { + // codecInputFormat, not inputFormat: this must describe the format actually governing the + // buffer being queued right now, which during a full codec reinit can briefly lag behind + // inputFormat (see drainOutputBuffer's matching check). Skipped entirely in tunneling mode, + // where drainOutputBuffer -- and so the corresponding pop -- never runs at all; queueing + // here without ever draining would just grow this unboundedly. + // + // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer + // converts dequeued timestamps back to. + arrivalOrderPtsQueue.addLast(presentationTimeUs); + } onQueueInputBuffer(buffer); int flags = getCodecBufferFlags(buffer); presentationTimeUs += skippedFlushOffsetUs; @@ -2255,6 +2281,17 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) return false; } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !arrivalOrderPtsQueue.isEmpty()) { + // codecInputFormat, not inputFormat -- see the matching check in feedInputBuffer. No + // tunneling check needed here: in tunneling mode this method isn't called at all (see + // updateOutputFormatForTime's javadoc), so the push side already prevents anything from + // reaching this queue to begin with. + outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); + } + this.outputIndex = outputIndex; outputBuffer = codec.getOutputBuffer(outputIndex); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index dabfc2f9299..69c5deb70f2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2165,6 +2165,16 @@ protected void onProcessedStreamChange() { */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // earlyUs is derived from a reconstructed timestamp here, not ground truth, so the normal + // threshold is too trigger-happy: pipeline depth from the decoder's own reorder buffering + // can plausibly look this late even with no real backlog. But it's still a genuine + // pacing signal (see arrivalOrderPtsQueue's declaration), so don't disable dropping + // outright either -- that would leave no recovery path if the decoder is genuinely falling + // behind. Require VERY_LATE-level lateness instead of the normal threshold as a backstop. + return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; + } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; } @@ -2180,6 +2190,13 @@ protected boolean shouldDropOutputBuffer( */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // Unlike shouldDropOutputBuffer, stays disabled here: this skips to a keyframe based on + // *how many* buffers to discard, which needs real per-sample identity we don't have in this + // case -- getting that wrong risks skipping past buffers that were never actually late. + return false; + } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 212b08073bc..18c32f9cc52 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -939,6 +939,41 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters assertThat(stringListCaptor.getValue()).containsExactly("key1", "key2"); } + @Test + public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue() + throws Exception { + // In tunneling mode drainOutputBuffer -- and so the queue's pop side -- never runs (see its + // javadoc), so feedInputBuffer must skip pushing altogether or the queue grows unboundedly for + // the life of the session. Track type must be video: the guard is a no-op for other types. + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + new RendererConfiguration(/* tunneling= */ true), + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + private TestRenderer setUpAndEnableRenderer(Format format) throws Exception { MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class); MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter; @@ -1106,6 +1141,107 @@ protected DecoderReuseEvaluation canReuseCodec( } } + /** Same as {@link TestRenderer}, but with {@link C#TRACK_TYPE_VIDEO} for video-only guards. */ + private static class VideoTestRenderer extends MediaCodecRenderer { + + VideoTestRenderer() { + this(MediaCodecAdapter.Factory.getDefault(ApplicationProvider.getApplicationContext())); + } + + VideoTestRenderer(MediaCodecAdapter.Factory mediaCodecAdapterFactory) { + super( + ApplicationProvider.getApplicationContext(), + C.TRACK_TYPE_VIDEO, + mediaCodecAdapterFactory, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + experimentalEnableProcessedStreamChangedAtStart(); + } + + @Override + public String getName() { + return "videoTest"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + MediaFormat mediaFormat = new MediaFormat(); + applyCodecParametersToMediaFormat(mediaFormat); + return MediaCodecAdapter.Configuration.createForVideoDecoding( + codecInfo, mediaFormat, format, /* surface= */ null, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + } + return true; + } + return false; + } + + @Override + protected void onCodecParametersChanged(CodecParameters codecParameters) { + // No-op for this test renderer. + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, + Format oldFormat, + Format newFormat, + boolean isAdaptiveFormatChange) { + return codecInfo.canReuseCodec(oldFormat, newFormat); + } + } + /** * A {@link MediaCodecAdapter} that throws a pre-specified exception from every decoding-related * interaction. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 54936bbf900..e906d5e8db6 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -40,6 +40,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -305,6 +306,103 @@ public void render_withLateBuffer_dropsBuffer() throws Exception { verify(eventListener).onDroppedFrames(eq(1), anyLong()); } + @Test + public void render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop() throws Exception { + // Point 3: unreliable timestamps must not disable drop detection outright (see + // shouldDropOutputBuffer), but do need much more slack than normal since earlyUs isn't ground + // truth here. 100ms late clears the normal 30ms threshold but not the widened 500ms one. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Moderately late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 150_001; // ~100ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener, never()).onDroppedFrames(anyInt(), anyLong()); + } + + @Test + public void render_withUnreliablePtsAndVeryLateBuffer_stillDrops() throws Exception { + // Same as render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop, but ~600ms late -- + // past the widened threshold, proving the backstop actually engages. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Very late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 650_001; // ~600ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onDroppedFrames(eq(1), anyLong()); + } + @Test public void render_withVeryLateBuffer_dropsBuffersUpstream() throws Exception { FakeSampleStream fakeSampleStream = @@ -1247,6 +1345,179 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); } + @Config(maxSdk = 29) + @Test + public void + render_withUnreliablePtsDuringIncompatibleFormatChange_stillFifoOrdersDrainingOldCodecOutput() + throws Exception { + // codecInputFormat (not inputFormat) must gate arrivalOrderPtsQueue, since inputFormat can + // advance to the new format before the old codec has finished draining its buffered output + // under the old format. Reuses render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec's + // 24fps->30fps trigger for a full codec reinit (REUSE_RESULT_NO) on SDK<=29, but the first + // format also has hasReliablePresentationTimestamps=false and out-of-order sample timestamps, + // and the codec adapter reorders dequeued output by ascending timestamp (simulating an + // untrustworthy decoder echo) so a wrong field choice is actually observable. + SystemClock.setCurrentTimeMillis(876_000_000); + Format unreliablePts24Fps = + VIDEO_H264_24FPS.buildUpon().setHasReliablePresentationTimestamps(false).build(); + Format reliablePts30Fps = + VIDEO_H264_30FPS.buildUpon().setHasReliablePresentationTimestamps(true).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts24Fps, + ImmutableList.of( + // Deliberately out of order: with no ctts, these are decode-order values, not + // true presentation times, which is the whole point of hasReliablePresentationTimestamps. + oneByteSample(/* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 0))); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts24Fps, reliablePts30Fps}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Render first sample to initialize codec. + renderer.render(/* positionUs= */ 0, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + shadowOf(testMainLooper).idle(); + + // Feed format change from 24->30 fps, which is not generally compatible and should reset + // codec, while the old codec may still be draining its format-A output. + fakeSampleStream.append( + ImmutableList.of( + format(reliablePts30Fps), + oneByteSample(/* timeUs= */ 80_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 40_000); + renderer.setCurrentStreamFinal(); + int positionUs = 40_000; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoDecoderReleased(any()); + verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); + // The reordering adapter would otherwise hand these back sorted ascending (0, 40_000). Seeing + // them in feed order proves codecInputFormat -- not the already-advanced inputFormat -- gated + // the FIFO replay while the old codec's format-A output was still draining. + assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); + } + + @Test + public void render_withUnreliablePtsAndVariableFrameSpacing_preservesPerSampleDurations() + throws Exception { + // Point 6: a VFR stream with real reordering and no ctts can't have its true per-frame + // presentation instants reconstructed exactly (there's no ctts to say what they are). What + // arrivalOrderPtsQueue can and must do is replay each sample's own queued timestamp exactly, + // not flatten irregular gaps to an assumed constant frame rate -- the bug this queue replaced. + Format unreliablePtsVfr = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePtsVfr, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 40_000), + oneByteSample(/* timeUs= */ 80_000), + oneByteSample(/* timeUs= */ 280_000), // Large VFR jump, e.g. a scene cut. + oneByteSample(/* timeUs= */ 300_000), // Small gap right after. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory(codecAdapterFactory) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVfr}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + int positionUs = 0; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 280_000L, 300_000L) + .inOrder(); + } + @Config(minSdk = 30) @Test public void render_withIncompatibleFrameRateChangeFromSdk30_keepsCodec() throws Exception { diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index 8ace3db723b..0c58a414ad6 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -448,6 +448,20 @@ public static TrackSampleTable parseStbl( GaplessInfoHolder gaplessInfoHolder, boolean omitTrackSampleTable) throws ParserException { + // Computed and applied to track up front, before any of the return paths below (including + // the sampleCount==0 and omitTrackSampleTable ones), so all of them carry the correct flag -- + // this doesn't depend on anything computed later in this method. See the field's javadoc for + // what NO_VALUE-vs-0 means here. + boolean hasReliablePresentationTimestamps = + !(track.type == C.TRACK_TYPE_VIDEO + && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null + && track.format.maxNumReorderSamples != 0); + if (!hasReliablePresentationTimestamps) { + track = + track.copyWithFormat( + track.format.buildUpon().setHasReliablePresentationTimestamps(false).build()); + } + SampleSizeBox sampleSizeBox; @Nullable LeafBox stszAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stsz); if (stszAtom != null) { @@ -961,6 +975,8 @@ public static TrackSampleTable parseStbl( } long editedDurationUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale); + // hasReliablePresentationTimestamps was already applied to track's format at the top of this + // method; only hasPrerollSamples (which isn't known until now) remains to apply here. if (hasPrerollSamples) { Format format = track.format.buildUpon().setHasPrerollSamples(true).build(); track = track.copyWithFormat(format);