From 325f2c5469d2ccb6a8bda381d874c8f30efb30e4 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Tue, 28 Jul 2026 19:47:45 -0500 Subject: [PATCH 01/10] Fix non-monotonic output timestamps from c2.mtk.hevc.decoder On Google TV Streamer 4K (MT8696, c2.mtk.hevc.decoder), specific HEVC Main10 content causes the codec to return output buffers with non-monotonic presentationTimeUs, while the buffer release order itself remains correct. MediaCodecRenderer processes output buffers in raw release order and uses the codec-reported timestamp directly for the render/drop decision, causing MediaCodecVideoRenderer to treat these buffers as arriving too late and drop them. Measured on affected content: ~30% of decoded video frames dropped, sustained throughout playback. This re-derives each output buffer's presentationTimeUs from its release order and the stream's known frame duration, instead of trusting the codec-reported value. Buffer release order is untouched. Verified on-device on the affected hardware/content: dropped frames go to 0, playback confirmed visually smooth by direct observation. --- .../mediacodec/MediaCodecRenderer.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index cec7fdbc1e6..c634404ea5b 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -417,6 +417,13 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; + // Some MediaCodec implementations (observed on c2.mtk.hevc.decoder) return output buffers + // whose presentationTimeUs is not monotonically increasing, while the codec's own buffer + // *release order* remains correct. Re-deriving each buffer's timestamp from its release order + // and the stream's known frame duration corrects this without reordering or holding buffers. + private long arrivalOrderPtsFrameDurationUs = C.TIME_UNSET; + private long arrivalOrderPtsBaseUs = C.TIME_UNSET; + private long arrivalOrderPtsFrameIndex; /** * @param context A context. @@ -1112,6 +1119,8 @@ protected void resetCodecStateForFlush() { codecReconfigured ? RECONFIGURATION_STATE_WRITE_PENDING : RECONFIGURATION_STATE_NONE; hasSkippedFlushAndWaitingForQueueInputBuffer = false; skippedFlushOffsetUs = 0; + arrivalOrderPtsBaseUs = C.TIME_UNSET; + arrivalOrderPtsFrameIndex = 0; } /** @@ -2255,6 +2264,25 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) return false; } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && inputFormat != null + && inputFormat.frameRate != Format.NO_VALUE + && inputFormat.frameRate > 0) { + if (arrivalOrderPtsFrameDurationUs == C.TIME_UNSET) { + arrivalOrderPtsFrameDurationUs = Math.round(1_000_000.0 / inputFormat.frameRate); + } + if (arrivalOrderPtsBaseUs == C.TIME_UNSET) { + // Anchor to the first real output buffer's own reported timestamp; only the spacing + // of subsequent buffers is re-derived, not the absolute position in the stream. + arrivalOrderPtsBaseUs = outputBufferInfo.presentationTimeUs; + arrivalOrderPtsFrameIndex = 0; + } else { + arrivalOrderPtsFrameIndex++; + outputBufferInfo.presentationTimeUs = + arrivalOrderPtsBaseUs + arrivalOrderPtsFrameIndex * arrivalOrderPtsFrameDurationUs; + } + } + this.outputIndex = outputIndex; outputBuffer = codec.getOutputBuffer(outputIndex); From 295aba9912736e2fd5e02d43889a8101e0a263c7 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 29 Jul 2026 07:51:24 -0500 Subject: [PATCH 02/10] Fix accumulating rounding error in output timestamp re-derivation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The per-frame duration was rounded to the nearest microsecond once, then multiplied by a growing frame index. Any fractional-microsecond remainder in the true frame duration (e.g. 41708.333...us for 24000/1001fps content) was silently dropped every frame and never recovered, so the error accumulated linearly with runtime: ~0.7ms over a 90-second clip (undetectable, and the only length tested before this fix), but ~58ms by the end of a 2-hour movie — enough to produce audible/visible A/V desync that gets worse the longer playback continues. Fix: keep the per-frame duration unrounded and compute each buffer's offset independently as round(frameIndex * unroundedDurationUs), rounding only the final result. Error no longer accumulates; it stays bounded to at most +/-0.5us for the life of the stream. Found via real-world testing in a third-party player (Plezy) on full- length movies, where the 90-second on-device test clips used to validate the original fix could never have revealed it. --- .../exoplayer/mediacodec/MediaCodecRenderer.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index c634404ea5b..98a521b3f76 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -421,7 +421,11 @@ private static String buildCustomDiagnosticInfo(int errorCode) { // whose presentationTimeUs is not monotonically increasing, while the codec's own buffer // *release order* remains correct. Re-deriving each buffer's timestamp from its release order // and the stream's known frame duration corrects this without reordering or holding buffers. - private long arrivalOrderPtsFrameDurationUs = C.TIME_UNSET; + // Kept unrounded and only rounded per-frame at the point of use (see below) — rounding this + // once and multiplying by a growing frame index would accumulate error linearly over the + // life of the stream (e.g. ~0.33us/frame for 24000/1001fps content is only 0.7ms over a 90s + // clip, but ~58ms over a 2-hour movie). + private double arrivalOrderPtsFrameDurationUs = -1; private long arrivalOrderPtsBaseUs = C.TIME_UNSET; private long arrivalOrderPtsFrameIndex; @@ -2268,8 +2272,8 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) && inputFormat != null && inputFormat.frameRate != Format.NO_VALUE && inputFormat.frameRate > 0) { - if (arrivalOrderPtsFrameDurationUs == C.TIME_UNSET) { - arrivalOrderPtsFrameDurationUs = Math.round(1_000_000.0 / inputFormat.frameRate); + if (arrivalOrderPtsFrameDurationUs < 0) { + arrivalOrderPtsFrameDurationUs = 1_000_000.0 / inputFormat.frameRate; } if (arrivalOrderPtsBaseUs == C.TIME_UNSET) { // Anchor to the first real output buffer's own reported timestamp; only the spacing @@ -2278,8 +2282,11 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) arrivalOrderPtsFrameIndex = 0; } else { arrivalOrderPtsFrameIndex++; + // Round only the final offset, not the per-frame duration, so rounding error can't + // accumulate across the length of the stream. outputBufferInfo.presentationTimeUs = - arrivalOrderPtsBaseUs + arrivalOrderPtsFrameIndex * arrivalOrderPtsFrameDurationUs; + arrivalOrderPtsBaseUs + + Math.round(arrivalOrderPtsFrameIndex * arrivalOrderPtsFrameDurationUs); } } From c3d379e7edd91734e4557b5fa4911f3de6e57b99 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Tue, 4 Aug 2026 21:25:46 -0500 Subject: [PATCH 03/10] Make the reordering fix precise: gate it on a real "missing PTS" signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous fix (325f2c5, 295aba9) re-derives every video track's output presentationTimeUs from arrival order unconditionally, for any track where the frame rate is known. That's safe for the content that motivated it, but it's a blunt instrument: it discards real per-sample timing information even on correctly-muxed content (harmless there only because arrival order already matches presentation order for a well-behaved decoder), and it would silently corrupt genuine variable-frame-rate content, which was never part of what triggered this in the first place. Root cause (see issue #3347) is not a decoder defect: it's that the source file's MP4 track has no ctts box, so every sample's presentationTimeUs is just its decode time. B-frame-reordered content muxed this way was never given real composition timing, and there's no way to reconstruct it after the fact — MediaCodec just echoes back whatever (undifferentiated) timestamp it was queued with. This adds Format#hasReliablePresentationTimestamps (default true) and sets it to false in Mp4Extractor/BoxParser exactly when a video track has no ctts box — the same signal a completely independent player (VLC, via its MP4 demuxer's MP4_TrackGetPTSDelta) already uses to decide it can't trust a sample's PTS for a video track. Confirmed by instrumenting and running VLC itself on-device against the same repro files: it receives equally/more disordered raw decoder output than ExoPlayer (52.2% vs ~42% non-monotonic steps) but never drops a frame for lateness, because by the time anything checks, its own PTS reconstruction has already replaced the disordered decoder echo with a clean synthetic timeline. Two changes gated on the new flag: - MediaCodecRenderer's arrival-order relabeling now only fires when the track told us up front it has no reliable per-sample timing, instead of unconditionally for every video track. - MediaCodecVideoRenderer's shouldDropOutputBuffer/ shouldDropBuffersToKeyframe no longer drop for lateness on such tracks — the relabeled timestamp is a best-effort reconstruction, not ground truth, so a "how late is this" reading isn't meaningful enough to discard a frame over. Render everything in arrival order instead, matching what VLC's own (independently-implemented, not copied) approach does in practice. Behavior for every other format (anything with a ctts box, or not MP4 at all) is unchanged: the new flag defaults to true and both changed code paths are no-ops unless a track explicitly says otherwise. --- .../java/androidx/media3/common/Format.java | 34 +++++++++++++++++++ .../mediacodec/MediaCodecRenderer.java | 15 +++++--- .../video/MediaCodecVideoRenderer.java | 17 ++++++++++ .../media3/extractor/mp4/BoxParser.java | 17 ++++++++-- 4 files changed, 77 insertions(+), 6 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java index 5922757ac3e..8590bb89510 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Format.java +++ b/libraries/common/src/main/java/androidx/media3/common/Format.java @@ -172,6 +172,7 @@ public static final class Builder { @Nullable private DrmInitData drmInitData; private long subsampleOffsetUs; private boolean hasPrerollSamples; + private boolean hasReliablePresentationTimestamps; // Video specific. @@ -218,6 +219,10 @@ public Builder() { maxInputSize = NO_VALUE; maxNumReorderSamples = NO_VALUE; subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE; + // Default to true: nearly every extractor/format provides trustworthy per-sample + // presentation timestamps. Only set to false where the source is known not to (see + // Format#hasReliablePresentationTimestamps). + hasReliablePresentationTimestamps = true; // Video specific. width = NO_VALUE; height = NO_VALUE; @@ -270,6 +275,7 @@ private Builder(Format format) { this.drmInitData = format.drmInitData; this.subsampleOffsetUs = format.subsampleOffsetUs; this.hasPrerollSamples = format.hasPrerollSamples; + this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps; // Video specific. this.width = format.width; this.height = format.height; @@ -590,6 +596,20 @@ public Builder setHasPrerollSamples(boolean hasPrerollSamples) { return this; } + /** + * Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}. + * + * @param hasReliablePresentationTimestamps The {@link + * Format#hasReliablePresentationTimestamps}. + * @return The builder. + */ + @CanIgnoreReturnValue + public Builder setHasReliablePresentationTimestamps( + boolean hasReliablePresentationTimestamps) { + this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps; + return this; + } + // Video specific. /** @@ -1051,6 +1071,19 @@ public Format build() { */ @UnstableApi public final boolean hasPrerollSamples; + /** + * Indicates whether per-sample presentation timestamps for this track are known to be + * accurate. + * + *

When {@code false}, the source (typically a container demuxer) could not determine each + * sample's true composition/presentation time — for example an MP4 track with B-frame + * reordering but no {@code ctts} box, where every sample's timestamp collapses to its decode + * time. Renderers should not treat such timestamps as ground truth for scheduling decisions + * (e.g. dropping "late" output buffers); doing so can misread decoder-level reordering as + * lateness. Defaults to {@code true}. + */ + @UnstableApi public final boolean hasReliablePresentationTimestamps; + // Video specific. /** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */ @@ -1218,6 +1251,7 @@ private Format(Builder builder) { drmInitData = builder.drmInitData; subsampleOffsetUs = builder.subsampleOffsetUs; hasPrerollSamples = builder.hasPrerollSamples; + hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps; // Video specific. width = builder.width; height = builder.height; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 98a521b3f76..e137474dbd4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -417,10 +417,16 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; - // Some MediaCodec implementations (observed on c2.mtk.hevc.decoder) return output buffers - // whose presentationTimeUs is not monotonically increasing, while the codec's own buffer - // *release order* remains correct. Re-deriving each buffer's timestamp from its release order - // and the stream's known frame duration corrects this without reordering or holding buffers. + // Used only when Format#hasReliablePresentationTimestamps is false (see that field's javadoc): + // the source told us it has no real per-sample composition timing, e.g. an MP4 track with + // B-frame reordering but no ctts box, where every sample's presentationTimeUs is just its + // decode time. MediaCodec faithfully echoes back whatever (unreliable) timestamp it was + // queued with, so the codec's raw output can look non-monotonic even though its buffer + // *release order* is correct. Re-deriving each buffer's timestamp from its release order and + // the stream's known frame duration corrects this without reordering or holding buffers — this + // is a best-effort reconstruction for a source that has already told us it doesn't know the + // real answer, not a workaround for a specific decoder defect (see + // https://github.com/androidx/media/issues/3347). // Kept unrounded and only rounded per-frame at the point of use (see below) — rounding this // once and multiplying by a growing frame index would accumulate error linearly over the // life of the stream (e.g. ~0.33us/frame for 24000/1001fps content is only 0.7ms over a 90s @@ -2270,6 +2276,7 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) if (getTrackType() == C.TRACK_TYPE_VIDEO && inputFormat != null + && !inputFormat.hasReliablePresentationTimestamps && inputFormat.frameRate != Format.NO_VALUE && inputFormat.frameRate > 0) { if (arrivalOrderPtsFrameDurationUs < 0) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index dabfc2f9299..0693a86126d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2165,6 +2165,16 @@ protected void onProcessedStreamChange() { */ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // The source told us it has no real per-sample composition timing (see + // Format#hasReliablePresentationTimestamps), so presentationTimeUs here is at best a + // best-effort reconstruction, not ground truth. Treating it as ground truth for a "how + // late is this buffer" decision risks dropping buffers whose only real fault is a + // decoder-level or reconstruction-level ordering wobble, not genuine lateness — render + // everything and let it look approximately right instead. + return false; + } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; } @@ -2180,6 +2190,13 @@ protected boolean shouldDropOutputBuffer( */ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { + @Nullable Format codecInputFormat = getCodecInputFormat(); + if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { + // See shouldDropOutputBuffer: an apparent large "very late" reading can itself be a + // reconstruction artifact (e.g. before arrivalOrderPtsBaseUs has anchored) rather than + // genuinely falling behind, so don't act on it here either. + return false; + } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; } diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index 8ace3db723b..7809fa960c9 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -961,8 +961,21 @@ public static TrackSampleTable parseStbl( } long editedDurationUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale); - if (hasPrerollSamples) { - Format format = track.format.buildUpon().setHasPrerollSamples(true).build(); + // A video track with no ctts box has no real per-sample composition-time information: every + // sample's timestamp is just its decode time. For content with B-frame reordering that's a + // silently-wrong presentation timestamp, not merely a missing optimization (see + // https://github.com/androidx/media/issues/3347) — flag it so renderers don't treat these + // timestamps as ground truth for lateness-based decisions. + boolean hasReliablePresentationTimestamps = + !(track.type == C.TRACK_TYPE_VIDEO && ctts == null); + if (hasPrerollSamples || !hasReliablePresentationTimestamps) { + Format format = + track + .format + .buildUpon() + .setHasPrerollSamples(hasPrerollSamples || track.format.hasPrerollSamples) + .setHasReliablePresentationTimestamps(hasReliablePresentationTimestamps) + .build(); track = track.copyWithFormat(format); } return new TrackSampleTable( From 1ffc7317cf26bb14d5317a2de00ec8a3b2ebced2 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Tue, 4 Aug 2026 22:17:27 -0500 Subject: [PATCH 04/10] Fix: relabeling was silently flattening genuine VFR content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The relabeling added in the previous commit re-derived every gated buffer's timestamp from a single assumed constant frame duration (1_000_000/Format#frameRate). Tested that assumption directly against a constructed file with no B-frames (so no ctts is legitimately correct, not a symptom of anything) but genuinely irregular per-sample durations (two spliced segments, 24fps and 8fps): the extractor reports a single blended Format#frameRate (16fps, the average), and the old logic used that one value for every frame in the file, flattening both segments to a uniform ~62.5ms spacing instead of the real ~41.7ms / 125ms — confirmed on-device via added diagnostic logging before this fix, removed after. Replaced the single-frame-duration synthesis with a FIFO of each video sample's own presentationTimeUs, recorded in queue order as input buffers are fed to the codec, and popped in output-arrival order to relabel each dequeued buffer. Queue order is always monotonic by construction (a direct echo of the container's own decode-time-to-sample table) and preserves each sample's true declared duration exactly, whether constant or not — this is what a completely independent player (VLC, via its MP4 demuxer's timestamp FIFO) already does for the same situation, arrived at independently during instrumentation of VLC on real device (see issue #3347 discussion). Re-verified the full on-device test matrix after this change: broken_95s.mp4, good_95s.mp4, waterboy_90s_clip.mp4 (all 0 dropped frames, reach ENDED normally), tos_real_footage_matched.mp4 (real ctts present, hasReliablePresentationTimestamps stays true, behavior unchanged), and the new VFR file (real ~41.7ms/125ms spacing now preserved exactly, confirmed via per-buffer diagnostic logging). --- .../mediacodec/MediaCodecRenderer.java | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index e137474dbd4..6972303dec1 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -422,18 +422,18 @@ private static String buildCustomDiagnosticInfo(int errorCode) { // B-frame reordering but no ctts box, where every sample's presentationTimeUs is just its // decode time. MediaCodec faithfully echoes back whatever (unreliable) timestamp it was // queued with, so the codec's raw output can look non-monotonic even though its buffer - // *release order* is correct. Re-deriving each buffer's timestamp from its release order and - // the stream's known frame duration corrects this without reordering or holding buffers — this - // is a best-effort reconstruction for a source that has already told us it doesn't know the - // real answer, not a workaround for a specific decoder defect (see - // https://github.com/androidx/media/issues/3347). - // Kept unrounded and only rounded per-frame at the point of use (see below) — rounding this - // once and multiplying by a growing frame index would accumulate error linearly over the - // life of the stream (e.g. ~0.33us/frame for 24000/1001fps content is only 0.7ms over a 90s - // clip, but ~58ms over a 2-hour movie). - private double arrivalOrderPtsFrameDurationUs = -1; - private long arrivalOrderPtsBaseUs = C.TIME_UNSET; - private long arrivalOrderPtsFrameIndex; + // *release order* is correct. + // + // Re-associating each dequeued output buffer with the presentationTimeUs of the next + // input buffer *in queue order* (not the codec's own echoed value) corrects this without + // reordering or holding buffers. This mirrors what a completely independent player (VLC, via + // its MP4 demuxer + a FIFO of queued-but-not-yet-consumed timestamps) already does for exactly + // this situation. Queue order is always monotonic by construction (it's a direct echo of the + // container's decode-time-to-sample table), and — importantly — this preserves each sample's + // *real* duration exactly as declared by the container: unlike re-deriving from a single + // assumed constant frame rate, this does not corrupt genuinely variable-frame-rate content + // (where Format#frameRate is at best an average, not a true per-sample value). + private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); /** * @param context A context. @@ -1129,8 +1129,7 @@ protected void resetCodecStateForFlush() { codecReconfigured ? RECONFIGURATION_STATE_WRITE_PENDING : RECONFIGURATION_STATE_NONE; hasSkippedFlushAndWaitingForQueueInputBuffer = false; skippedFlushOffsetUs = 0; - arrivalOrderPtsBaseUs = C.TIME_UNSET; - arrivalOrderPtsFrameIndex = 0; + arrivalOrderPtsQueue.clear(); } /** @@ -1668,6 +1667,14 @@ private boolean feedInputBuffer() throws ExoPlaybackException { hasSkippedFlushAndWaitingForQueueInputBuffer = false; } + if (getTrackType() == C.TRACK_TYPE_VIDEO + && inputFormat != null + && !inputFormat.hasReliablePresentationTimestamps) { + // Recorded in the same (pre skippedFlushOffsetUs) scale that drainOutputBuffer converts + // dequeued output timestamps back to, so it can be substituted directly for the codec's + // own echoed value there. See arrivalOrderPtsQueue's declaration for why. + arrivalOrderPtsQueue.addLast(presentationTimeUs); + } onQueueInputBuffer(buffer); int flags = getCodecBufferFlags(buffer); presentationTimeUs += skippedFlushOffsetUs; @@ -2277,24 +2284,13 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) if (getTrackType() == C.TRACK_TYPE_VIDEO && inputFormat != null && !inputFormat.hasReliablePresentationTimestamps - && inputFormat.frameRate != Format.NO_VALUE - && inputFormat.frameRate > 0) { - if (arrivalOrderPtsFrameDurationUs < 0) { - arrivalOrderPtsFrameDurationUs = 1_000_000.0 / inputFormat.frameRate; - } - if (arrivalOrderPtsBaseUs == C.TIME_UNSET) { - // Anchor to the first real output buffer's own reported timestamp; only the spacing - // of subsequent buffers is re-derived, not the absolute position in the stream. - arrivalOrderPtsBaseUs = outputBufferInfo.presentationTimeUs; - arrivalOrderPtsFrameIndex = 0; - } else { - arrivalOrderPtsFrameIndex++; - // Round only the final offset, not the per-frame duration, so rounding error can't - // accumulate across the length of the stream. - outputBufferInfo.presentationTimeUs = - arrivalOrderPtsBaseUs - + Math.round(arrivalOrderPtsFrameIndex * arrivalOrderPtsFrameDurationUs); - } + && !arrivalOrderPtsQueue.isEmpty()) { + // Replace the codec's own (unreliable, possibly-reordered) echoed timestamp with the + // presentationTimeUs of the next sample in queue order. Queue order is a direct echo of + // the container's decode-time-to-sample table, so it's always monotonic and always + // reflects each sample's true declared duration — including on genuinely + // variable-frame-rate content, where a single Format#frameRate value would not. + outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); } this.outputIndex = outputIndex; From cc9d0e6b0130123a20677e6594e9641de15fd569 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 10:12:32 -0500 Subject: [PATCH 05/10] Tighten comments to match the codebase's existing style The comments added across the last two commits ran 3-5x longer than comparable existing ones in this file (e.g. hasPrerollSamples's field javadoc, or skippedFlushOffsetUs, which gets two sentences on its getter and no comment at all at the field declaration) and repeated the same reasoning at every call site instead of stating it once. Also fixes a stale reference: shouldDropBuffersToKeyframe's comment still named arrivalOrderPtsBaseUs, a field the VFR fix in the previous commit removed. --- .../java/androidx/media3/common/Format.java | 17 +++------- .../mediacodec/MediaCodecRenderer.java | 31 +++++-------------- .../video/MediaCodecVideoRenderer.java | 11 ++----- .../media3/extractor/mp4/BoxParser.java | 7 ++--- 4 files changed, 16 insertions(+), 50 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/Format.java b/libraries/common/src/main/java/androidx/media3/common/Format.java index 8590bb89510..33033051881 100644 --- a/libraries/common/src/main/java/androidx/media3/common/Format.java +++ b/libraries/common/src/main/java/androidx/media3/common/Format.java @@ -219,10 +219,7 @@ public Builder() { maxInputSize = NO_VALUE; maxNumReorderSamples = NO_VALUE; subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE; - // Default to true: nearly every extractor/format provides trustworthy per-sample - // presentation timestamps. Only set to false where the source is known not to (see - // Format#hasReliablePresentationTimestamps). - hasReliablePresentationTimestamps = true; + hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps // Video specific. width = NO_VALUE; height = NO_VALUE; @@ -1072,15 +1069,11 @@ public Format build() { @UnstableApi public final boolean hasPrerollSamples; /** - * Indicates whether per-sample presentation timestamps for this track are known to be - * accurate. + * Indicates whether per-sample presentation timestamps for this track are known to be accurate. * - *

When {@code false}, the source (typically a container demuxer) could not determine each - * sample's true composition/presentation time — for example an MP4 track with B-frame - * reordering but no {@code ctts} box, where every sample's timestamp collapses to its decode - * time. Renderers should not treat such timestamps as ground truth for scheduling decisions - * (e.g. dropping "late" output buffers); doing so can misread decoder-level reordering as - * lateness. Defaults to {@code true}. + *

When {@code false}, the source couldn't determine each sample's true presentation time + * (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the + * timestamps as ground truth for scheduling decisions. Defaults to {@code true}. */ @UnstableApi public final boolean hasReliablePresentationTimestamps; diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 6972303dec1..dd61324bfeb 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -417,22 +417,10 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private CodecParameters activeCodecParameters; private CodecParameters lastDispatchedCodecParameters; private ImmutableSet subscribedCodecParameterKeys; - // Used only when Format#hasReliablePresentationTimestamps is false (see that field's javadoc): - // the source told us it has no real per-sample composition timing, e.g. an MP4 track with - // B-frame reordering but no ctts box, where every sample's presentationTimeUs is just its - // decode time. MediaCodec faithfully echoes back whatever (unreliable) timestamp it was - // queued with, so the codec's raw output can look non-monotonic even though its buffer - // *release order* is correct. - // - // Re-associating each dequeued output buffer with the presentationTimeUs of the next - // input buffer *in queue order* (not the codec's own echoed value) corrects this without - // reordering or holding buffers. This mirrors what a completely independent player (VLC, via - // its MP4 demuxer + a FIFO of queued-but-not-yet-consumed timestamps) already does for exactly - // this situation. Queue order is always monotonic by construction (it's a direct echo of the - // container's decode-time-to-sample table), and — importantly — this preserves each sample's - // *real* duration exactly as declared by the container: unlike re-deriving from a single - // assumed constant frame rate, this does not corrupt genuinely variable-frame-rate content - // (where Format#frameRate is at best an average, not a true per-sample value). + // Used only when Format#hasReliablePresentationTimestamps is false: replays each sample's own + // queued presentationTimeUs in output-arrival order instead of trusting MediaCodec's echoed + // (possibly non-monotonic) timestamp. Preserves true per-sample duration, unlike deriving from + // a single assumed frame rate. private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); /** @@ -1670,9 +1658,8 @@ private boolean feedInputBuffer() throws ExoPlaybackException { if (getTrackType() == C.TRACK_TYPE_VIDEO && inputFormat != null && !inputFormat.hasReliablePresentationTimestamps) { - // Recorded in the same (pre skippedFlushOffsetUs) scale that drainOutputBuffer converts - // dequeued output timestamps back to, so it can be substituted directly for the codec's - // own echoed value there. See arrivalOrderPtsQueue's declaration for why. + // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer + // converts dequeued timestamps back to. arrivalOrderPtsQueue.addLast(presentationTimeUs); } onQueueInputBuffer(buffer); @@ -2285,11 +2272,7 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) && inputFormat != null && !inputFormat.hasReliablePresentationTimestamps && !arrivalOrderPtsQueue.isEmpty()) { - // Replace the codec's own (unreliable, possibly-reordered) echoed timestamp with the - // presentationTimeUs of the next sample in queue order. Queue order is a direct echo of - // the container's decode-time-to-sample table, so it's always monotonic and always - // reflects each sample's true declared duration — including on genuinely - // variable-frame-rate content, where a single Format#frameRate value would not. + // See arrivalOrderPtsQueue's declaration. outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 0693a86126d..a0b99d960f2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2167,12 +2167,7 @@ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { @Nullable Format codecInputFormat = getCodecInputFormat(); if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // The source told us it has no real per-sample composition timing (see - // Format#hasReliablePresentationTimestamps), so presentationTimeUs here is at best a - // best-effort reconstruction, not ground truth. Treating it as ground truth for a "how - // late is this buffer" decision risks dropping buffers whose only real fault is a - // decoder-level or reconstruction-level ordering wobble, not genuine lateness — render - // everything and let it look approximately right instead. + // earlyUs here is derived from a reconstructed timestamp, not ground truth — don't drop on it. return false; } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; @@ -2192,9 +2187,7 @@ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { @Nullable Format codecInputFormat = getCodecInputFormat(); if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // See shouldDropOutputBuffer: an apparent large "very late" reading can itself be a - // reconstruction artifact (e.g. before arrivalOrderPtsBaseUs has anchored) rather than - // genuinely falling behind, so don't act on it here either. + // See shouldDropOutputBuffer. return false; } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index 7809fa960c9..82df69497bd 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -961,11 +961,8 @@ public static TrackSampleTable parseStbl( } long editedDurationUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale); - // A video track with no ctts box has no real per-sample composition-time information: every - // sample's timestamp is just its decode time. For content with B-frame reordering that's a - // silently-wrong presentation timestamp, not merely a missing optimization (see - // https://github.com/androidx/media/issues/3347) — flag it so renderers don't treat these - // timestamps as ground truth for lateness-based decisions. + // No ctts means no real per-sample composition time was ever recorded (see + // https://github.com/androidx/media/issues/3347) — every sample's timestamp is just decode time. boolean hasReliablePresentationTimestamps = !(track.type == C.TRACK_TYPE_VIDEO && ctts == null); if (hasPrerollSamples || !hasReliablePresentationTimestamps) { From ce9dde04ebb8292c27fd51c73a2fb728eb55519f Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 12:41:39 -0500 Subject: [PATCH 06/10] Narrow the missing-ctts trigger to tracks that actually reorder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review on #3347: missing ctts alone was too broad. A correctly muxed video with no B-frame reordering legitimately has no ctts, and under the old check it would still get flagged hasReliablePresentationTimestamps=false — needlessly activating the FIFO relabeling *and*, more importantly, permanently disabling shouldDropOutputBuffer/shouldDropBuffersToKeyframe for content that never had a timestamp problem. On a genuinely overloaded device, that would let video drift behind audio with no way to recover. Now also requires track.format.maxNumReorderSamples != 0 -- this comes from the bitstream's own SPS (sps_max_num_reorder_pics for HEVC, via HevcConfig.java), not the container, and is already parsed before parseStbl runs. 0 means the encoder positively declared it doesn't reorder; NO_VALUE (-1, config box missing/unparsed) is treated the same as "might reorder" rather than "doesn't" -- conservative by design, per discussion. Verified on-device, all via the same diagnostic logging used throughout this investigation (added temporarily, confirmed, removed): - broken_95s.mp4 (maxNumReorderSamples=2, real reordering): still triggers the workaround, still 0 dropped frames. - good_95s.mp4 (maxNumReorderSamples=0, encoder confirms no reordering): now correctly does NOT trigger -- normal shouldDropOutputBuffer behavior is back in effect for this file, still 0 dropped frames since it never needed the workaround. - tos_real_footage_matched.mp4 (real ctts present): unaffected, as before. - New: mp4v_no_ctts_no_maxreorder.mp4 -- MPEG-4 Part 2 (mp4v) with 159 real B-frames and no ctts, built by encoding normally (ffmpeg mpeg4 encoder, -bf 2, which produces a correctly-muxed file with real ctts) and then surgically stripping just the ctts box (strip_ctts.py, included alongside) rather than going through the raw-ES-remux pipeline used elsewhere in this investigation, since that pipeline doesn't handle mpeg4 cleanly. media3 has no maxNumReorderSamples-setting code path for mp4v at all (only avcC/hvcC/vvcC set it), so this is a real, not contrived, maxNumReorderSamples=NO_VALUE case with genuine reordering. Confirmed hasReliablePresentationTimestamps=false (the conservative choice correctly engaging), played on-device via c2.mtk.mpeg4.decoder, 0 dropped frames, ENDED cleanly. --- .../java/androidx/media3/extractor/mp4/BoxParser.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index 82df69497bd..149f6525ee8 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -962,9 +962,14 @@ public static TrackSampleTable parseStbl( long editedDurationUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale); // No ctts means no real per-sample composition time was ever recorded (see - // https://github.com/androidx/media/issues/3347) — every sample's timestamp is just decode time. + // https://github.com/androidx/media/issues/3347) — every sample's timestamp is just decode + // time, which only matters if the bitstream actually reorders frames. maxNumReorderSamples + // (from the SPS, not the container) is 0 only when we positively know it doesn't; treat + // NO_VALUE (couldn't be determined) the same as "might reorder", not as "doesn't". boolean hasReliablePresentationTimestamps = - !(track.type == C.TRACK_TYPE_VIDEO && ctts == null); + !(track.type == C.TRACK_TYPE_VIDEO + && ctts == null + && track.format.maxNumReorderSamples != 0); if (hasPrerollSamples || !hasReliablePresentationTimestamps) { Format format = track From 4a4132e5b2167feaef8cff00a214f64d782ba163 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 13:21:03 -0500 Subject: [PATCH 07/10] Compute hasReliablePresentationTimestamps at the top of parseStbl Per review point 1 on #3347: the flag was only being applied near the final return of parseStbl(), so the sampleCount==0 early return (and by extension the omitTrackSampleTable case, which still runs the rest of the method but was untested against this specific path) never carried it. Many video tracks without ctts would silently never activate the workaround. Moved the computation to the very start of the method, before any return path -- everything it depends on (track.type, whether a ctts box exists, track.format.maxNumReorderSamples) is already available from the parameters, none of it requires the per-sample parsing that happens later. Applies to track's format immediately, so every one of the 6 return sites in this method now carries the correct value. hasPrerollSamples is unaffected -- it genuinely isn't known until the edit-list processing near the bottom of the method runs, so it stays where it was; only removed the accidental "OR with existing value" logic I'd added when the two were sharing one if-block, restoring the original single-flag behavior there. Re-verified on-device: broken_95s.mp4 (still triggers, 0 drops), good_95s.mp4 (still correctly skips, 0 drops), mp4v_no_ctts_no_maxreorder.mp4 (still triggers via the conservative NO_VALUE path, 0 drops). --- .../media3/extractor/mp4/BoxParser.java | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java index 149f6525ee8..0c58a414ad6 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/BoxParser.java @@ -448,6 +448,20 @@ public static TrackSampleTable parseStbl( GaplessInfoHolder gaplessInfoHolder, boolean omitTrackSampleTable) throws ParserException { + // Computed and applied to track up front, before any of the return paths below (including + // the sampleCount==0 and omitTrackSampleTable ones), so all of them carry the correct flag -- + // this doesn't depend on anything computed later in this method. See the field's javadoc for + // what NO_VALUE-vs-0 means here. + boolean hasReliablePresentationTimestamps = + !(track.type == C.TRACK_TYPE_VIDEO + && stblBox.getLeafBoxOfType(Mp4Box.TYPE_ctts) == null + && track.format.maxNumReorderSamples != 0); + if (!hasReliablePresentationTimestamps) { + track = + track.copyWithFormat( + track.format.buildUpon().setHasReliablePresentationTimestamps(false).build()); + } + SampleSizeBox sampleSizeBox; @Nullable LeafBox stszAtom = stblBox.getLeafBoxOfType(Mp4Box.TYPE_stsz); if (stszAtom != null) { @@ -961,23 +975,10 @@ public static TrackSampleTable parseStbl( } long editedDurationUs = Util.scaleLargeTimestamp(pts, C.MICROS_PER_SECOND, track.movieTimescale); - // No ctts means no real per-sample composition time was ever recorded (see - // https://github.com/androidx/media/issues/3347) — every sample's timestamp is just decode - // time, which only matters if the bitstream actually reorders frames. maxNumReorderSamples - // (from the SPS, not the container) is 0 only when we positively know it doesn't; treat - // NO_VALUE (couldn't be determined) the same as "might reorder", not as "doesn't". - boolean hasReliablePresentationTimestamps = - !(track.type == C.TRACK_TYPE_VIDEO - && ctts == null - && track.format.maxNumReorderSamples != 0); - if (hasPrerollSamples || !hasReliablePresentationTimestamps) { - Format format = - track - .format - .buildUpon() - .setHasPrerollSamples(hasPrerollSamples || track.format.hasPrerollSamples) - .setHasReliablePresentationTimestamps(hasReliablePresentationTimestamps) - .build(); + // hasReliablePresentationTimestamps was already applied to track's format at the top of this + // method; only hasPrerollSamples (which isn't known until now) remains to apply here. + if (hasPrerollSamples) { + Format format = track.format.buildUpon().setHasPrerollSamples(true).build(); track = track.copyWithFormat(format); } return new TrackSampleTable( From b4332803d18068a942cee613a8df499675330607 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 15:07:46 -0500 Subject: [PATCH 08/10] Fix format-instance mismatch and tunneling leak in the PTS FIFO Per review points 4 and 5 on #3347. Point 4: the push (feedInputBuffer) and pop (drainOutputBuffer) sides of arrivalOrderPtsQueue were gated on inputFormat, while shouldDropOutputBuffer/shouldDropBuffersToKeyframe (MediaCodecVideoRenderer) are gated on codecInputFormat. These are usually the same value -- onInputFormatChanged sets both together for the two seamless-reuse cases (REUSE_RESULT_YES_WITH_FLUSH/RECONFIGURATION) -- but diverge during a full codec reinit (REUSE_RESULT_NO): inputFormat advances immediately, codecInputFormat doesn't catch up until the new codec is actually configured, and in between the old codec is still draining old-format output under the old format's policy. Switched both queue sites to codecInputFormat, which is a field on this same class (no new plumbing) and is the more correct authority anyway: it specifically tracks what the active codec instance is configured with, which is what governs the buffers actually flowing through queue/dequeue. Point 5: MediaCodecVideoRenderer's tunneling mode has its own output path (see updateOutputFormatForTime's javadoc: buffers aren't dequeued from the decoder at all in that mode), so drainOutputBuffer -- and therefore the pop -- never runs. The push wasn't tunneling-aware, so arrivalOrderPtsQueue would grow unboundedly for the duration of any tunneled session with unreliable timestamps. Guarded the push with !getConfiguration().tunneling (BaseRenderer, already accessible, no video-specific field needed). Existing resetCodecStateForFlush() clearing already handles cleanup if tunneling toggles mid-session. No matching guard needed on the pop side -- it's provably unreachable in tunneling mode already, so anything there would be dead code. Re-verified on-device (default demo app config, not tunneled): broken_95s.mp4, good_95s.mp4, mp4v_no_ctts_no_maxreorder.mp4 -- all unchanged, 0 dropped frames each. The specific codecInputFormat-lag scenario (full reinit mid-stream) isn't exercised by any of these; a dedicated seamless-format-change test is still on the list from the original review (along with tunneling itself, which I could not directly exercise against the demo app's default config). --- .../mediacodec/MediaCodecRenderer.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index dd61324bfeb..8f582176f6c 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -1656,8 +1656,15 @@ private boolean feedInputBuffer() throws ExoPlaybackException { } if (getTrackType() == C.TRACK_TYPE_VIDEO - && inputFormat != null - && !inputFormat.hasReliablePresentationTimestamps) { + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps + && !getConfiguration().tunneling) { + // codecInputFormat, not inputFormat: this must describe the format actually governing the + // buffer being queued right now, which during a full codec reinit can briefly lag behind + // inputFormat (see drainOutputBuffer's matching check). Skipped entirely in tunneling mode, + // where drainOutputBuffer -- and so the corresponding pop -- never runs at all; queueing + // here without ever draining would just grow this unboundedly. + // // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer // converts dequeued timestamps back to. arrivalOrderPtsQueue.addLast(presentationTimeUs); @@ -2269,10 +2276,13 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) } if (getTrackType() == C.TRACK_TYPE_VIDEO - && inputFormat != null - && !inputFormat.hasReliablePresentationTimestamps + && codecInputFormat != null + && !codecInputFormat.hasReliablePresentationTimestamps && !arrivalOrderPtsQueue.isEmpty()) { - // See arrivalOrderPtsQueue's declaration. + // codecInputFormat, not inputFormat -- see the matching check in feedInputBuffer. No + // tunneling check needed here: in tunneling mode this method isn't called at all (see + // updateOutputFormatForTime's javadoc), so the push side already prevents anything from + // reaching this queue to begin with. outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst(); } From 6d8bc043ee0388a380a310504eea0fcb7573d48e Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 16:21:42 -0500 Subject: [PATCH 09/10] Add regression tests for points 4 and 5 (codecInputFormat/inputFormat mismatch, tunneling leak) Adds a package-private @VisibleForTesting arrivalOrderPtsQueue size accessor to MediaCodecRenderer -- there's no black-box way to observe the tunneling leak fix otherwise, since nothing ever reads the queue back out in that mode. Point 4 (MediaCodecVideoRendererTest): reuses the existing render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec 24fps->30fps REUSE_RESULT_NO trigger, but the first format now has hasReliablePresentationTimestamps=false and out-of-order sample timestamps, and the codec adapter is ForwardingSynchronousMediaCodecAdapterWithReordering (sorts dequeued output ascending, simulating an untrustworthy decoder echo). Captures onProcessedOutputBuffer's sequence and asserts the first format's two samples come back in feed order, not sorted -- proving codecInputFormat (which still lags at "old format" during the drain) governs the FIFO replay rather than the already-advanced inputFormat. Point 5 (MediaCodecRendererTest): adds a minimal video-track-type TestRenderer variant (the existing one hardcodes TRACK_TYPE_AUDIO, but the tunneling guard only applies to video), enables it with RendererConfiguration(tunneling=true) and an unreliable-PTS format, feeds several samples, and asserts the queue size is still 0 -- the queue would otherwise grow for the life of the session since drainOutputBuffer never runs in tunneling mode. Full MediaCodecVideoRendererTest (176 cases) and MediaCodecRendererTest (21 cases) suites re-run clean alongside the new tests. --- .../mediacodec/MediaCodecRenderer.java | 6 + .../mediacodec/MediaCodecRendererTest.java | 136 ++++++++++++++++++ .../video/MediaCodecVideoRendererTest.java | 98 +++++++++++++ 3 files changed, 240 insertions(+) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 8f582176f6c..abbf079981e 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -47,6 +47,7 @@ import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; @@ -423,6 +424,11 @@ private static String buildCustomDiagnosticInfo(int errorCode) { // a single assumed frame rate. private final ArrayDeque arrivalOrderPtsQueue = new ArrayDeque<>(); + @VisibleForTesting + /* package */ int getArrivalOrderPtsQueueSizeForTesting() { + return arrivalOrderPtsQueue.size(); + } + /** * @param context A context. * @param trackType The {@link C.TrackType track type} that the renderer handles. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index 212b08073bc..18c32f9cc52 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -939,6 +939,41 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters assertThat(stringListCaptor.getValue()).containsExactly("key1", "key2"); } + @Test + public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue() + throws Exception { + // In tunneling mode drainOutputBuffer -- and so the queue's pop side -- never runs (see its + // javadoc), so feedInputBuffer must skip pushing altogether or the queue grows unboundedly for + // the life of the session. Track type must be video: the guard is a no-op for other types. + Format unreliablePtsVideo = + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_H264) + .setHasReliablePresentationTimestamps(false) + .build(); + FakeSampleStream fakeSampleStream = + createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200); + VideoTestRenderer renderer = new VideoTestRenderer(); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.enable( + new RendererConfiguration(/* tunneling= */ true), + new Format[] {unreliablePtsVideo}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0); + } + private TestRenderer setUpAndEnableRenderer(Format format) throws Exception { MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class); MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter; @@ -1106,6 +1141,107 @@ protected DecoderReuseEvaluation canReuseCodec( } } + /** Same as {@link TestRenderer}, but with {@link C#TRACK_TYPE_VIDEO} for video-only guards. */ + private static class VideoTestRenderer extends MediaCodecRenderer { + + VideoTestRenderer() { + this(MediaCodecAdapter.Factory.getDefault(ApplicationProvider.getApplicationContext())); + } + + VideoTestRenderer(MediaCodecAdapter.Factory mediaCodecAdapterFactory) { + super( + ApplicationProvider.getApplicationContext(), + C.TRACK_TYPE_VIDEO, + mediaCodecAdapterFactory, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + experimentalEnableProcessedStreamChangedAtStart(); + } + + @Override + public String getName() { + return "videoTest"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + MediaFormat mediaFormat = new MediaFormat(); + applyCodecParametersToMediaFormat(mediaFormat); + return MediaCodecAdapter.Configuration.createForVideoDecoding( + codecInfo, mediaFormat, format, /* surface= */ null, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + if (codec != null) { + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + } + return true; + } + return false; + } + + @Override + protected void onCodecParametersChanged(CodecParameters codecParameters) { + // No-op for this test renderer. + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, + Format oldFormat, + Format newFormat, + boolean isAdaptiveFormatChange) { + return codecInfo.canReuseCodec(oldFormat, newFormat); + } + } + /** * A {@link MediaCodecAdapter} that throws a pre-specified exception from every decoding-related * interaction. diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 54936bbf900..73364630251 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -1247,6 +1247,104 @@ public void render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec() thro verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); } + @Config(maxSdk = 29) + @Test + public void + render_withUnreliablePtsDuringIncompatibleFormatChange_stillFifoOrdersDrainingOldCodecOutput() + throws Exception { + // codecInputFormat (not inputFormat) must gate arrivalOrderPtsQueue, since inputFormat can + // advance to the new format before the old codec has finished draining its buffered output + // under the old format. Reuses render_withIncompatibleFrameRateChangeUpToSdk29_discardsCodec's + // 24fps->30fps trigger for a full codec reinit (REUSE_RESULT_NO) on SDK<=29, but the first + // format also has hasReliablePresentationTimestamps=false and out-of-order sample timestamps, + // and the codec adapter reorders dequeued output by ascending timestamp (simulating an + // untrustworthy decoder echo) so a wrong field choice is actually observable. + SystemClock.setCurrentTimeMillis(876_000_000); + Format unreliablePts24Fps = + VIDEO_H264_24FPS.buildUpon().setHasReliablePresentationTimestamps(false).build(); + Format reliablePts30Fps = + VIDEO_H264_30FPS.buildUpon().setHasReliablePresentationTimestamps(true).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts24Fps, + ImmutableList.of( + // Deliberately out of order: with no ctts, these are decode-order values, not + // true presentation times, which is the whole point of hasReliablePresentationTimestamps. + oneByteSample(/* timeUs= */ 40_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 0))); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory( + new ForwardingSynchronousMediaCodecAdapterWithReordering.Factory()) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts24Fps, reliablePts30Fps}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + // Render first sample to initialize codec. + renderer.render(/* positionUs= */ 0, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + shadowOf(testMainLooper).idle(); + + // Feed format change from 24->30 fps, which is not generally compatible and should reset + // codec, while the old codec may still be draining its format-A output. + fakeSampleStream.append( + ImmutableList.of( + format(reliablePts30Fps), + oneByteSample(/* timeUs= */ 80_000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 40_000); + renderer.setCurrentStreamFinal(); + int positionUs = 40_000; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onVideoDecoderReleased(any()); + verify(eventListener, times(2)).onVideoDecoderInitialized(any(), anyLong(), anyLong()); + // The reordering adapter would otherwise hand these back sorted ascending (0, 40_000). Seeing + // them in feed order proves codecInputFormat -- not the already-advanced inputFormat -- gated + // the FIFO replay while the old codec's format-A output was still draining. + assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); + } + @Config(minSdk = 30) @Test public void render_withIncompatibleFrameRateChangeFromSdk30_keepsCodec() throws Exception { From 410db95b9f0d4f5c465801e28658ace0d3191297 Mon Sep 17 00:00:00 2001 From: Andrew Malota <2bitoperations@gmail.com> Date: Wed, 5 Aug 2026 21:59:46 -0500 Subject: [PATCH 10/10] Address points 3 and 6: widen the drop threshold instead of disabling it, add VFR regression test Point 3: shouldDropOutputBuffer() and shouldDropBuffersToKeyframe() returned false unconditionally when hasReliablePresentationTimestamps is false, disabling all recovery from genuine decoder slowness -- but earlyUs under that flag is still a real pacing signal (it's derived from the same arrival-order-corrected timestamp arrivalOrderPtsQueue substitutes), just less precise than ground truth. shouldDropOutputBuffer now requires VERY_LATE-level lateness (500ms, vs the normal 30ms) instead of disabling outright -- enough slack for decoder reorder-buffer depth to not look like backlog, but a real backstop against runaway lateness. shouldDropBuffersToKeyframe stays disabled: it decides how many buffers to discard, which needs real per-sample identity this case doesn't have. Two new tests cover the widened-not-removed threshold: a buffer 100ms late (past the old 30ms threshold, short of the new 500ms one) isn't dropped; one 600ms late still is. Point 6: added a regression test feeding irregularly-spaced (VFR) sample timestamps with hasReliablePresentationTimestamps=false, asserting arrivalOrderPtsQueue replays them exactly rather than flattening to an assumed constant frame rate (the bug the arrival-order queue replaced). FongMi's point stands that true per-frame presentation instants can't be reconstructed without ctts; this validates the best achievable fallback (faithful arrival-order replay) instead. Full MediaCodecVideoRendererTest (179 cases) and MediaCodecRendererTest (21 cases) suites re-run clean. --- .../video/MediaCodecVideoRenderer.java | 13 +- .../video/MediaCodecVideoRendererTest.java | 173 ++++++++++++++++++ 2 files changed, 183 insertions(+), 3 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index a0b99d960f2..69c5deb70f2 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -2167,8 +2167,13 @@ protected boolean shouldDropOutputBuffer( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { @Nullable Format codecInputFormat = getCodecInputFormat(); if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // earlyUs here is derived from a reconstructed timestamp, not ground truth — don't drop on it. - return false; + // earlyUs is derived from a reconstructed timestamp here, not ground truth, so the normal + // threshold is too trigger-happy: pipeline depth from the decoder's own reorder buffering + // can plausibly look this late even with no real backlog. But it's still a genuine + // pacing signal (see arrivalOrderPtsQueue's declaration), so don't disable dropping + // outright either -- that would leave no recovery path if the decoder is genuinely falling + // behind. Require VERY_LATE-level lateness instead of the normal threshold as a backstop. + return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; } return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer; } @@ -2187,7 +2192,9 @@ protected boolean shouldDropBuffersToKeyframe( long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) { @Nullable Format codecInputFormat = getCodecInputFormat(); if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) { - // See shouldDropOutputBuffer. + // Unlike shouldDropOutputBuffer, stays disabled here: this skips to a keyframe based on + // *how many* buffers to discard, which needs real per-sample identity we don't have in this + // case -- getting that wrong risks skipping past buffers that were never actually late. return false; } return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 73364630251..e906d5e8db6 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -40,6 +40,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; @@ -305,6 +306,103 @@ public void render_withLateBuffer_dropsBuffer() throws Exception { verify(eventListener).onDroppedFrames(eq(1), anyLong()); } + @Test + public void render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop() throws Exception { + // Point 3: unreliable timestamps must not disable drop detection outright (see + // shouldDropOutputBuffer), but do need much more slack than normal since earlyUs isn't ground + // truth here. 100ms late clears the normal 30ms threshold but not the widened 500ms one. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Moderately late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 150_001; // ~100ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener, never()).onDroppedFrames(anyInt(), anyLong()); + } + + @Test + public void render_withUnreliablePtsAndVeryLateBuffer_stillDrops() throws Exception { + // Same as render_withUnreliablePtsAndModeratelyLateBuffer_doesNotDrop, but ~600ms late -- + // past the widened threshold, proving the backstop actually engages. + Format unreliablePts = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePts, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), // First buffer. + oneByteSample(/* timeUs= */ 50_000), // Very late buffer. + oneByteSample(/* timeUs= */ 100_000), // Last buffer. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePts}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + /* mediaPeriodId= */ new MediaSource.MediaPeriodId(new Object())); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000); + for (int i = 0; i < 5; i++) { + mediaCodecVideoRenderer.render(40_000, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + } + mediaCodecVideoRenderer.setCurrentStreamFinal(); + int posUs = 650_001; // ~600ms late relative to the 50_000 sample. + while (!mediaCodecVideoRenderer.isEnded()) { + mediaCodecVideoRenderer.render(posUs, SystemClock.elapsedRealtime() * 1000); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + posUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + verify(eventListener).onDroppedFrames(eq(1), anyLong()); + } + @Test public void render_withVeryLateBuffer_dropsBuffersUpstream() throws Exception { FakeSampleStream fakeSampleStream = @@ -1345,6 +1443,81 @@ protected void onProcessedOutputBuffer(long presentationTimeUs) { assertThat(processedTimestamps.subList(0, 2)).containsExactly(40_000L, 0L).inOrder(); } + @Test + public void render_withUnreliablePtsAndVariableFrameSpacing_preservesPerSampleDurations() + throws Exception { + // Point 6: a VFR stream with real reordering and no ctts can't have its true per-frame + // presentation instants reconstructed exactly (there's no ctts to say what they are). What + // arrivalOrderPtsQueue can and must do is replay each sample's own queued timestamp exactly, + // not flatten irregular gaps to an assumed constant frame rate -- the bug this queue replaced. + Format unreliablePtsVfr = + VIDEO_H264.buildUpon().setHasReliablePresentationTimestamps(false).build(); + List processedTimestamps = new ArrayList<>(); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ unreliablePtsVfr, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 40_000), + oneByteSample(/* timeUs= */ 80_000), + oneByteSample(/* timeUs= */ 280_000), // Large VFR jump, e.g. a scene cut. + oneByteSample(/* timeUs= */ 300_000), // Small gap right after. + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + MediaCodecVideoRenderer renderer = + new MediaCodecVideoRenderer( + new MediaCodecVideoRenderer.Builder(ApplicationProvider.getApplicationContext()) + .setCodecAdapterFactory(codecAdapterFactory) + .setMediaCodecSelector(mediaCodecSelector) + .setAllowedJoiningTimeMs(0) + .setEnableDecoderFallback(false) + .setEventHandler(new Handler(testMainLooper)) + .setEventListener(eventListener) + .setMaxDroppedFramesToNotify(1)) { + @Override + protected @Capabilities int supportsFormat( + MediaCodecSelector mediaCodecSelector, Format format) { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected void onProcessedOutputBuffer(long presentationTimeUs) { + super.onProcessedOutputBuffer(presentationTimeUs); + processedTimestamps.add(presentationTimeUs); + } + }; + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + renderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {unreliablePtsVfr}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + renderer.start(); + renderer.setCurrentStreamFinal(); + int positionUs = 0; + while (!renderer.isEnded()) { + ShadowSystemClock.advanceBy(Duration.ofMillis(40)); + renderer.render(positionUs, msToUs(SystemClock.elapsedRealtime())); + codecAdapterFactory.idleQueueingAndCallbackThreads(); + positionUs += 40_000; + } + shadowOf(testMainLooper).idle(); + + assertThat(processedTimestamps) + .containsExactly(0L, 40_000L, 80_000L, 280_000L, 300_000L) + .inOrder(); + } + @Config(minSdk = 30) @Test public void render_withIncompatibleFrameRateChangeFromSdk30_keepsCodec() throws Exception {