Recover from non-monotonic presentationTimeUs caused by naively-muxed B-frame content

libraries/common/src/main/java/androidx/media3/common/Format.java

@@ -172,6 +172,7 @@ public static final class Builder {
     @Nullable private DrmInitData drmInitData;
     private long subsampleOffsetUs;
     private boolean hasPrerollSamples;
+    private boolean hasReliablePresentationTimestamps;
 
     // Video specific.
 
@@ -218,6 +219,7 @@ public Builder() {
       maxInputSize = NO_VALUE;
       maxNumReorderSamples = NO_VALUE;
       subsampleOffsetUs = OFFSET_SAMPLE_RELATIVE;
+      hasReliablePresentationTimestamps = true; // most formats have accurate per-sample timestamps
       // Video specific.
       width = NO_VALUE;
       height = NO_VALUE;
@@ -270,6 +272,7 @@ private Builder(Format format) {
       this.drmInitData = format.drmInitData;
       this.subsampleOffsetUs = format.subsampleOffsetUs;
       this.hasPrerollSamples = format.hasPrerollSamples;
+      this.hasReliablePresentationTimestamps = format.hasReliablePresentationTimestamps;
       // Video specific.
       this.width = format.width;
       this.height = format.height;
@@ -590,6 +593,20 @@ public Builder setHasPrerollSamples(boolean hasPrerollSamples) {
       return this;
     }
 
+    /**
+     * Sets {@link Format#hasReliablePresentationTimestamps}. The default value is {@code true}.
+     *
+     * @param hasReliablePresentationTimestamps The {@link
+     *     Format#hasReliablePresentationTimestamps}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setHasReliablePresentationTimestamps(
+        boolean hasReliablePresentationTimestamps) {
+      this.hasReliablePresentationTimestamps = hasReliablePresentationTimestamps;
+      return this;
+    }
+
     // Video specific.
 
     /**
@@ -1051,6 +1068,15 @@ public Format build() {
    */
   @UnstableApi public final boolean hasPrerollSamples;
 
+  /**
+   * Indicates whether per-sample presentation timestamps for this track are known to be accurate.
+   *
+   * <p>When {@code false}, the source couldn't determine each sample's true presentation time
+   * (for example an MP4 track with no {@code ctts} box). Consumers shouldn't treat the
+   * timestamps as ground truth for scheduling decisions. Defaults to {@code true}.
+   */
+  @UnstableApi public final boolean hasReliablePresentationTimestamps;
+
   // Video specific.
 
   /** The width of the video in pixels, or {@link #NO_VALUE} if unknown or not applicable. */
@@ -1218,6 +1244,7 @@ private Format(Builder builder) {
     drmInitData = builder.drmInitData;
     subsampleOffsetUs = builder.subsampleOffsetUs;
     hasPrerollSamples = builder.hasPrerollSamples;
+    hasReliablePresentationTimestamps = builder.hasReliablePresentationTimestamps;
     // Video specific.
     width = builder.width;
     height = builder.height;

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java

@@ -47,6 +47,7 @@
 import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import androidx.annotation.RequiresApi;
+import androidx.annotation.VisibleForTesting;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
 import androidx.media3.common.MimeTypes;
@@ -417,6 +418,16 @@ private static String buildCustomDiagnosticInfo(int errorCode) {
   private CodecParameters activeCodecParameters;
   private CodecParameters lastDispatchedCodecParameters;
   private ImmutableSet<String> subscribedCodecParameterKeys;
+  // Used only when Format#hasReliablePresentationTimestamps is false: replays each sample's own
+  // queued presentationTimeUs in output-arrival order instead of trusting MediaCodec's echoed
+  // (possibly non-monotonic) timestamp. Preserves true per-sample duration, unlike deriving from
+  // a single assumed frame rate.
+  private final ArrayDeque<Long> arrivalOrderPtsQueue = new ArrayDeque<>();
+
+  @VisibleForTesting
+  /* package */ int getArrivalOrderPtsQueueSizeForTesting() {
+    return arrivalOrderPtsQueue.size();
+  }
 
   /**
    * @param context A context.
@@ -1112,6 +1123,7 @@ protected void resetCodecStateForFlush() {
         codecReconfigured ? RECONFIGURATION_STATE_WRITE_PENDING : RECONFIGURATION_STATE_NONE;
     hasSkippedFlushAndWaitingForQueueInputBuffer = false;
     skippedFlushOffsetUs = 0;
+    arrivalOrderPtsQueue.clear();
   }
 
   /**
@@ -1649,6 +1661,20 @@ private boolean feedInputBuffer() throws ExoPlaybackException {
       hasSkippedFlushAndWaitingForQueueInputBuffer = false;
     }
 
+    if (getTrackType() == C.TRACK_TYPE_VIDEO
+        && codecInputFormat != null
+        && !codecInputFormat.hasReliablePresentationTimestamps
+        && !getConfiguration().tunneling) {
+      // codecInputFormat, not inputFormat: this must describe the format actually governing the
+      // buffer being queued right now, which during a full codec reinit can briefly lag behind
+      // inputFormat (see drainOutputBuffer's matching check). Skipped entirely in tunneling mode,
+      // where drainOutputBuffer -- and so the corresponding pop -- never runs at all; queueing
+      // here without ever draining would just grow this unboundedly.
+      //
+      // Must run before the += skippedFlushOffsetUs below, to match the scale drainOutputBuffer
+      // converts dequeued timestamps back to.
+      arrivalOrderPtsQueue.addLast(presentationTimeUs);
+    }
     onQueueInputBuffer(buffer);
     int flags = getCodecBufferFlags(buffer);
     presentationTimeUs += skippedFlushOffsetUs;
@@ -2255,6 +2281,17 @@ private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
         return false;
       }
 
+      if (getTrackType() == C.TRACK_TYPE_VIDEO
+          && codecInputFormat != null
+          && !codecInputFormat.hasReliablePresentationTimestamps
+          && !arrivalOrderPtsQueue.isEmpty()) {
+        // codecInputFormat, not inputFormat -- see the matching check in feedInputBuffer. No
+        // tunneling check needed here: in tunneling mode this method isn't called at all (see
+        // updateOutputFormatForTime's javadoc), so the push side already prevents anything from
+        // reaching this queue to begin with.
+        outputBufferInfo.presentationTimeUs = arrivalOrderPtsQueue.removeFirst();
+      }
+
       this.outputIndex = outputIndex;
       outputBuffer = codec.getOutputBuffer(outputIndex);
 

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java

@@ -2165,6 +2165,16 @@ protected void onProcessedStreamChange() {
    */
   protected boolean shouldDropOutputBuffer(
       long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
+    @Nullable Format codecInputFormat = getCodecInputFormat();
+    if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) {
+      // earlyUs is derived from a reconstructed timestamp here, not ground truth, so the normal
+      // threshold is too trigger-happy: pipeline depth from the decoder's own reorder buffering
+      // can plausibly look this late even with no real backlog. But it's still a genuine
+      // pacing signal (see arrivalOrderPtsQueue's declaration), so don't disable dropping
+      // outright either -- that would leave no recovery path if the decoder is genuinely falling
+      // behind. Require VERY_LATE-level lateness instead of the normal threshold as a backstop.
+      return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer;
+    }
     return earlyUs < MIN_EARLY_US_LATE_THRESHOLD && !isLastBuffer;
   }
 
@@ -2180,6 +2190,13 @@ protected boolean shouldDropOutputBuffer(
    */
   protected boolean shouldDropBuffersToKeyframe(
       long earlyUs, long elapsedRealtimeUs, boolean isLastBuffer) {
+    @Nullable Format codecInputFormat = getCodecInputFormat();
+    if (codecInputFormat != null && !codecInputFormat.hasReliablePresentationTimestamps) {
+      // Unlike shouldDropOutputBuffer, stays disabled here: this skips to a keyframe based on
+      // *how many* buffers to discard, which needs real per-sample identity we don't have in this
+      // case -- getting that wrong risks skipping past buffers that were never actually late.
+      return false;
+    }
     return earlyUs < MIN_EARLY_US_VERY_LATE_THRESHOLD && !isLastBuffer;
   }
 

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java

@@ -939,6 +939,41 @@ public void codecReinitialized_withSubscribedKeys_resubscribesToVendorParameters
     assertThat(stringListCaptor.getValue()).containsExactly("key1", "key2");
   }
 
+  @Test
+  public void feedInputBuffer_withUnreliablePtsAndTunneling_doesNotGrowArrivalOrderPtsQueue()
+      throws Exception {
+    // In tunneling mode drainOutputBuffer -- and so the queue's pop side -- never runs (see its
+    // javadoc), so feedInputBuffer must skip pushing altogether or the queue grows unboundedly for
+    // the life of the session. Track type must be video: the guard is a no-op for other types.
+    Format unreliablePtsVideo =
+        new Format.Builder()
+            .setSampleMimeType(MimeTypes.VIDEO_H264)
+            .setHasReliablePresentationTimestamps(false)
+            .build();
+    FakeSampleStream fakeSampleStream =
+        createFakeSampleStream(unreliablePtsVideo, /* sampleTimesUs...= */ 0, 100, 200);
+    VideoTestRenderer renderer = new VideoTestRenderer();
+    renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
+    renderer.enable(
+        new RendererConfiguration(/* tunneling= */ true),
+        new Format[] {unreliablePtsVideo},
+        fakeSampleStream,
+        /* positionUs= */ 0,
+        /* joining= */ false,
+        /* mayRenderStartOfStream= */ true,
+        /* startPositionUs= */ 0,
+        /* offsetUs= */ 0,
+        new MediaSource.MediaPeriodId(new Object()));
+    renderer.start();
+    long positionUs = 0;
+    while (!renderer.hasReadStreamToEnd()) {
+      renderer.render(positionUs, SystemClock.elapsedRealtime());
+      positionUs += 100;
+    }
+
+    assertThat(renderer.getArrivalOrderPtsQueueSizeForTesting()).isEqualTo(0);
+  }
+
   private TestRenderer setUpAndEnableRenderer(Format format) throws Exception {
     MediaCodecAdapter mockCodecAdapter = mock(MediaCodecAdapter.class);
     MediaCodecAdapter.Factory mockCodecAdapterFactory = configuration -> mockCodecAdapter;
@@ -1106,6 +1141,107 @@ protected DecoderReuseEvaluation canReuseCodec(
     }
   }
 
+  /** Same as {@link TestRenderer}, but with {@link C#TRACK_TYPE_VIDEO} for video-only guards. */
+  private static class VideoTestRenderer extends MediaCodecRenderer {
+
+    VideoTestRenderer() {
+      this(MediaCodecAdapter.Factory.getDefault(ApplicationProvider.getApplicationContext()));
+    }
+
+    VideoTestRenderer(MediaCodecAdapter.Factory mediaCodecAdapterFactory) {
+      super(
+          ApplicationProvider.getApplicationContext(),
+          C.TRACK_TYPE_VIDEO,
+          mediaCodecAdapterFactory,
+          /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) ->
+              Collections.singletonList(
+                  MediaCodecInfo.newInstance(
+                      /* name= */ "name",
+                      /* mimeType= */ mimeType,
+                      /* codecMimeType= */ mimeType,
+                      /* capabilities= */ null,
+                      /* hardwareAccelerated= */ false,
+                      /* softwareOnly= */ true,
+                      /* vendor= */ false,
+                      /* forceDisableAdaptive= */ false,
+                      /* forceSecure= */ false)),
+          /* enableDecoderFallback= */ false,
+          /* assumedMinimumCodecOperatingRate= */ 44100);
+      experimentalEnableProcessedStreamChangedAtStart();
+    }
+
+    @Override
+    public String getName() {
+      return "videoTest";
+    }
+
+    @Override
+    protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) {
+      return RendererCapabilities.create(C.FORMAT_HANDLED);
+    }
+
+    @Override
+    protected List<MediaCodecInfo> getDecoderInfos(
+        MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder)
+        throws MediaCodecUtil.DecoderQueryException {
+      return mediaCodecSelector.getDecoderInfos(
+          format.sampleMimeType,
+          /* requiresSecureDecoder= */ false,
+          /* requiresTunnelingDecoder= */ false);
+    }
+
+    @Override
+    protected MediaCodecAdapter.Configuration getMediaCodecConfiguration(
+        MediaCodecInfo codecInfo,
+        Format format,
+        @Nullable MediaCrypto crypto,
+        float codecOperatingRate) {
+      MediaFormat mediaFormat = new MediaFormat();
+      applyCodecParametersToMediaFormat(mediaFormat);
+      return MediaCodecAdapter.Configuration.createForVideoDecoding(
+          codecInfo, mediaFormat, format, /* surface= */ null, crypto);
+    }
+
+    @Override
+    protected boolean processOutputBuffer(
+        long positionUs,
+        long elapsedRealtimeUs,
+        @Nullable MediaCodecAdapter codec,
+        @Nullable ByteBuffer buffer,
+        int bufferIndex,
+        int bufferFlags,
+        int sampleCount,
+        long bufferPresentationTimeUs,
+        boolean isDecodeOnlyBuffer,
+        boolean isLastBuffer,
+        Format format)
+        throws ExoPlaybackException {
+      if (bufferPresentationTimeUs <= positionUs) {
+        // Only release buffers when the position advances far enough for realistic behavior where
+        // input of buffers to the codec is faster than output.
+        if (codec != null) {
+          codec.releaseOutputBuffer(bufferIndex, /* render= */ true);
+        }
+        return true;
+      }
+      return false;
+    }
+
+    @Override
+    protected void onCodecParametersChanged(CodecParameters codecParameters) {
+      // No-op for this test renderer.
+    }
+
+    @Override
+    protected DecoderReuseEvaluation canReuseCodec(
+        MediaCodecInfo codecInfo,
+        Format oldFormat,
+        Format newFormat,
+        boolean isAdaptiveFormatChange) {
+      return codecInfo.canReuseCodec(oldFormat, newFormat);
+    }
+  }
+
   /**
    * A {@link MediaCodecAdapter} that throws a pre-specified exception from every decoding-related
    * interaction.