Allow adjusting sideloaded subtitle time offsets during playback

RELEASENOTES.md

@@ -62,6 +62,11 @@
     *   Fix reporting of late video frames with identical release timestamps so
         that they are reported as dropped instead of skipped.
 *   Text:
+    *   Add `MediaItem.SubtitleConfiguration.timeOffsetUs` to shift the
+        timestamps of sideloaded subtitles relative to the media. The offset
+        can be changed during playback with
+        `Player.replaceMediaItem(int, MediaItem)` without interrupting playback
+        ([#1976](https://github.com/androidx/media/issues/1976)).
 *   Metadata:
 *   Image:
 *   DataSource:

libraries/common/src/main/java/androidx/media3/common/MediaItem.java

@@ -1567,6 +1567,7 @@ public static final class Builder {
       private @C.RoleFlags int roleFlags;
       @Nullable private String label;
       @Nullable private String id;
+      private long timeOffsetUs;
 
       /**
        * Constructs an instance.
@@ -1585,6 +1586,7 @@ private Builder(SubtitleConfiguration subtitleConfiguration) {
         this.roleFlags = subtitleConfiguration.roleFlags;
         this.label = subtitleConfiguration.label;
         this.id = subtitleConfiguration.id;
+        this.timeOffsetUs = subtitleConfiguration.timeOffsetUs;
       }
 
       /** Sets the {@link Uri} to the subtitle file. */
@@ -1636,6 +1638,34 @@ public Builder setId(@Nullable String id) {
         return this;
       }
 
+      /**
+       * Sets the offset that is added to the timestamps of the cues in this subtitle track, in
+       * microseconds.
+       *
+       * <p>A positive value shifts the cues to be displayed later relative to the media, a negative
+       * value shifts them to be displayed earlier.
+       *
+       * <p>The offset can be changed during playback by passing an updated {@link MediaItem} to
+       * {@code Player.replaceMediaItem(int, MediaItem)}. If only the time offsets of the {@link
+       * SubtitleConfiguration} instances are changed, playback continues uninterrupted and the new
+       * offsets apply to cues that have not been read by the renderer yet. To also apply the new
+       * offset to the cues currently on screen, disable and re-enable the text track, for example
+       * with {@code TrackSelectionParameters.Builder.setTrackTypeDisabled(C.TRACK_TYPE_TEXT,
+       * boolean)}.
+       *
+       * <p>The offset only takes effect if the subtitles are parsed during extraction (the default
+       * behaviour of {@code DefaultMediaSourceFactory}), and is ignored by the deprecated legacy
+       * subtitle decoding path.
+       *
+       * <p>The default value is 0.
+       */
+      @CanIgnoreReturnValue
+      @UnstableApi
+      public Builder setTimeOffsetUs(long timeOffsetUs) {
+        this.timeOffsetUs = timeOffsetUs;
+        return this;
+      }
+
       /** Creates a {@link SubtitleConfiguration} from the values of this builder. */
       public SubtitleConfiguration build() {
         return new SubtitleConfiguration(this);
@@ -1671,6 +1701,12 @@ private Subtitle buildSubtitle() {
      */
     @Nullable public final String id;
 
+    /**
+     * The offset that is added to the timestamps of the cues in this subtitle track, in
+     * microseconds. See {@link Builder#setTimeOffsetUs(long)} for details.
+     */
+    @UnstableApi public final long timeOffsetUs;
+
     private SubtitleConfiguration(
         Uri uri,
         String mimeType,
@@ -1686,6 +1722,7 @@ private SubtitleConfiguration(
       this.roleFlags = roleFlags;
       this.label = label;
       this.id = id;
+      this.timeOffsetUs = 0;
     }
 
     private SubtitleConfiguration(Builder builder) {
@@ -1696,6 +1733,7 @@ private SubtitleConfiguration(Builder builder) {
       this.roleFlags = builder.roleFlags;
       this.label = builder.label;
       this.id = builder.id;
+      this.timeOffsetUs = builder.timeOffsetUs;
     }
 
     /** Returns a {@link Builder} initialized with the values of this instance. */
@@ -1720,7 +1758,8 @@ public boolean equals(@Nullable Object obj) {
           && selectionFlags == other.selectionFlags
           && roleFlags == other.roleFlags
           && Objects.equals(label, other.label)
-          && Objects.equals(id, other.id);
+          && Objects.equals(id, other.id)
+          && timeOffsetUs == other.timeOffsetUs;
     }
 
     @Override
@@ -1732,6 +1771,7 @@ public int hashCode() {
       result = 31 * result + roleFlags;
       result = 31 * result + (label == null ? 0 : label.hashCode());
       result = 31 * result + (id == null ? 0 : id.hashCode());
+      result = (int) (31L * result + timeOffsetUs);
       return result;
     }
 
@@ -1742,6 +1782,7 @@ public int hashCode() {
     private static final String FIELD_ROLE_FLAGS = Util.intToStringMaxRadix(4);
     private static final String FIELD_LABEL = Util.intToStringMaxRadix(5);
     private static final String FIELD_ID = Util.intToStringMaxRadix(6);
+    private static final String FIELD_TIME_OFFSET_US = Util.intToStringMaxRadix(7);
 
     /** Restores a {@code SubtitleConfiguration} from a {@link Bundle}. */
     @UnstableApi
@@ -1753,6 +1794,7 @@ public static SubtitleConfiguration fromBundle(Bundle bundle) {
       @C.RoleFlags int roleFlags = bundle.getInt(FIELD_ROLE_FLAGS, 0);
       @Nullable String label = bundle.getString(FIELD_LABEL);
       @Nullable String id = bundle.getString(FIELD_ID);
+      long timeOffsetUs = bundle.getLong(FIELD_TIME_OFFSET_US, 0);
 
       SubtitleConfiguration.Builder builder = new SubtitleConfiguration.Builder(uri);
       return builder
@@ -1762,6 +1804,7 @@ public static SubtitleConfiguration fromBundle(Bundle bundle) {
           .setRoleFlags(roleFlags)
           .setLabel(label)
           .setId(id)
+          .setTimeOffsetUs(timeOffsetUs)
           .build();
     }
 
@@ -1787,6 +1830,9 @@ public Bundle toBundle() {
       if (id != null) {
         bundle.putString(FIELD_ID, id);
       }
+      if (timeOffsetUs != 0) {
+        bundle.putLong(FIELD_TIME_OFFSET_US, timeOffsetUs);
+      }
       return bundle;
     }
   }

libraries/common/src/test/java/androidx/media3/common/MediaItemTest.java

@@ -328,6 +328,7 @@ public void builderSetSubtitleConfigurations() {
                 .setRoleFlags(C.ROLE_FLAG_ALTERNATE)
                 .setLabel("label")
                 .setId("id")
+                .setTimeOffsetUs(500_000)
                 .build());
 
     MediaItem mediaItem =
@@ -396,6 +397,7 @@ public void createSubtitleConfigurationInstance_roundTripViaBundle_yieldsEqualIn
             .setRoleFlags(C.ROLE_FLAG_ALTERNATE)
             .setLabel("label")
             .setId("id")
+            .setTimeOffsetUs(-300_000)
             .build();
 
     MediaItem.SubtitleConfiguration subtitleConfigurationFromBundle =
@@ -800,6 +802,7 @@ public void createLocalConfigurationInstance_roundTripViaBundle_yieldsEqualInsta
                         .setRoleFlags(C.ROLE_FLAG_ALTERNATE)
                         .setLabel("label")
                         .setId("id")
+                        .setTimeOffsetUs(250_000)
                         .build()))
             .setDrmConfiguration(
                 new MediaItem.DrmConfiguration.Builder(C.WIDEVINE_UUID)
@@ -953,6 +956,7 @@ public void buildUpon_individualSetters_equalsToOriginal() {
                         .setRoleFlags(C.ROLE_FLAG_ALTERNATE)
                         .setLabel("label")
                         .setId("id")
+                        .setTimeOffsetUs(250_000)
                         .build()))
             .setTag(new Object())
             .build();
@@ -1010,6 +1014,7 @@ public void buildUpon_wholeObjectSetters_equalsToOriginal() {
                         .setRoleFlags(C.ROLE_FLAG_ALTERNATE)
                         .setLabel("label")
                         .setId("id")
+                        .setTimeOffsetUs(250_000)
                         .build()))
             .setRequestMetadata(
                 new RequestMetadata.Builder()

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java

@@ -598,10 +598,9 @@ public MediaSource createMediaSource(MediaItem mediaItem) {
     List<MediaItem.SubtitleConfiguration> subtitleConfigurations =
         castNonNull(mediaItem.localConfiguration).subtitleConfigurations;
     if (!subtitleConfigurations.isEmpty()) {
-      MediaSource[] mediaSources = new MediaSource[subtitleConfigurations.size() + 1];
-      mediaSources[0] = mediaSource;
-      for (int i = 0; i < subtitleConfigurations.size(); i++) {
-        if (parseSubtitlesDuringExtraction) {
+      if (parseSubtitlesDuringExtraction) {
+        MediaSource[] subtitleMediaSources = new MediaSource[subtitleConfigurations.size()];
+        for (int i = 0; i < subtitleConfigurations.size(); i++) {
           Format format =
               new Format.Builder()
                   .setSampleMimeType(subtitleConfigurations.get(i).mimeType)
@@ -636,10 +635,17 @@ public MediaSource createMediaSource(MediaItem mediaItem) {
           if (loadErrorHandlingPolicy != null) {
             progressiveMediaSourceFactory.setLoadErrorHandlingPolicy(loadErrorHandlingPolicy);
           }
-          mediaSources[i + 1] =
+          subtitleMediaSources[i] =
               progressiveMediaSourceFactory.createMediaSource(
                   MediaItem.fromUri(subtitleConfigurations.get(i).uri.toString()));
-        } else {
+        }
+        mediaSource =
+            new SideloadedSubtitlesMediaSource(
+                mediaSource, subtitleConfigurations, subtitleMediaSources);
+      } else {
+        MediaSource[] mediaSources = new MediaSource[subtitleConfigurations.size() + 1];
+        mediaSources[0] = mediaSource;
+        for (int i = 0; i < subtitleConfigurations.size(); i++) {
           SingleSampleMediaSource.Factory singleSampleMediaSourceFactory =
               new SingleSampleMediaSource.Factory(dataSourceFactory);
           if (loadErrorHandlingPolicy != null) {
@@ -649,9 +655,8 @@ public MediaSource createMediaSource(MediaItem mediaItem) {
               singleSampleMediaSourceFactory.createMediaSource(
                   subtitleConfigurations.get(i), /* durationUs= */ C.TIME_UNSET);
         }
+        mediaSource = new MergingMediaSource(mediaSources);
       }
-
-      mediaSource = new MergingMediaSource(mediaSources);
     }
     return maybeWrapWithAdsMediaSource(
         mediaItem, maybeClipMediaSource(mediaItem, mediaSource, enableClippingInMediaPeriod));