diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 70886c2bb48..f6bc7f3fd5b 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -62,6 +62,11 @@ * Fix reporting of late video frames with identical release timestamps so that they are reported as dropped instead of skipped. * Text: + * Add `MediaItem.SubtitleConfiguration.timeOffsetUs` to shift the + timestamps of sideloaded subtitles relative to the media. The offset + can be changed during playback with + `Player.replaceMediaItem(int, MediaItem)` without interrupting playback + ([#1976](https://github.com/androidx/media/issues/1976)). * Metadata: * Image: * DataSource: diff --git a/libraries/common/src/main/java/androidx/media3/common/MediaItem.java b/libraries/common/src/main/java/androidx/media3/common/MediaItem.java index 6be88f395a3..3d30ef454f7 100644 --- a/libraries/common/src/main/java/androidx/media3/common/MediaItem.java +++ b/libraries/common/src/main/java/androidx/media3/common/MediaItem.java @@ -1567,6 +1567,7 @@ public static final class Builder { private @C.RoleFlags int roleFlags; @Nullable private String label; @Nullable private String id; + private long timeOffsetUs; /** * Constructs an instance. @@ -1585,6 +1586,7 @@ private Builder(SubtitleConfiguration subtitleConfiguration) { this.roleFlags = subtitleConfiguration.roleFlags; this.label = subtitleConfiguration.label; this.id = subtitleConfiguration.id; + this.timeOffsetUs = subtitleConfiguration.timeOffsetUs; } /** Sets the {@link Uri} to the subtitle file. */ @@ -1636,6 +1638,34 @@ public Builder setId(@Nullable String id) { return this; } + /** + * Sets the offset that is added to the timestamps of the cues in this subtitle track, in + * microseconds. + * + *

A positive value shifts the cues to be displayed later relative to the media, a negative + * value shifts them to be displayed earlier. + * + *

The offset can be changed during playback by passing an updated {@link MediaItem} to + * {@code Player.replaceMediaItem(int, MediaItem)}. If only the time offsets of the {@link + * SubtitleConfiguration} instances are changed, playback continues uninterrupted and the new + * offsets apply to cues that have not been read by the renderer yet. To also apply the new + * offset to the cues currently on screen, disable and re-enable the text track, for example + * with {@code TrackSelectionParameters.Builder.setTrackTypeDisabled(C.TRACK_TYPE_TEXT, + * boolean)}. + * + *

The offset only takes effect if the subtitles are parsed during extraction (the default + * behaviour of {@code DefaultMediaSourceFactory}), and is ignored by the deprecated legacy + * subtitle decoding path. + * + *

The default value is 0. + */ + @CanIgnoreReturnValue + @UnstableApi + public Builder setTimeOffsetUs(long timeOffsetUs) { + this.timeOffsetUs = timeOffsetUs; + return this; + } + /** Creates a {@link SubtitleConfiguration} from the values of this builder. */ public SubtitleConfiguration build() { return new SubtitleConfiguration(this); @@ -1671,6 +1701,12 @@ private Subtitle buildSubtitle() { */ @Nullable public final String id; + /** + * The offset that is added to the timestamps of the cues in this subtitle track, in + * microseconds. See {@link Builder#setTimeOffsetUs(long)} for details. + */ + @UnstableApi public final long timeOffsetUs; + private SubtitleConfiguration( Uri uri, String mimeType, @@ -1686,6 +1722,7 @@ private SubtitleConfiguration( this.roleFlags = roleFlags; this.label = label; this.id = id; + this.timeOffsetUs = 0; } private SubtitleConfiguration(Builder builder) { @@ -1696,6 +1733,7 @@ private SubtitleConfiguration(Builder builder) { this.roleFlags = builder.roleFlags; this.label = builder.label; this.id = builder.id; + this.timeOffsetUs = builder.timeOffsetUs; } /** Returns a {@link Builder} initialized with the values of this instance. */ @@ -1720,7 +1758,8 @@ public boolean equals(@Nullable Object obj) { && selectionFlags == other.selectionFlags && roleFlags == other.roleFlags && Objects.equals(label, other.label) - && Objects.equals(id, other.id); + && Objects.equals(id, other.id) + && timeOffsetUs == other.timeOffsetUs; } @Override @@ -1732,6 +1771,7 @@ public int hashCode() { result = 31 * result + roleFlags; result = 31 * result + (label == null ? 0 : label.hashCode()); result = 31 * result + (id == null ? 0 : id.hashCode()); + result = (int) (31L * result + timeOffsetUs); return result; } @@ -1742,6 +1782,7 @@ public int hashCode() { private static final String FIELD_ROLE_FLAGS = Util.intToStringMaxRadix(4); private static final String FIELD_LABEL = Util.intToStringMaxRadix(5); private static final String FIELD_ID = Util.intToStringMaxRadix(6); + private static final String FIELD_TIME_OFFSET_US = Util.intToStringMaxRadix(7); /** Restores a {@code SubtitleConfiguration} from a {@link Bundle}. */ @UnstableApi @@ -1753,6 +1794,7 @@ public static SubtitleConfiguration fromBundle(Bundle bundle) { @C.RoleFlags int roleFlags = bundle.getInt(FIELD_ROLE_FLAGS, 0); @Nullable String label = bundle.getString(FIELD_LABEL); @Nullable String id = bundle.getString(FIELD_ID); + long timeOffsetUs = bundle.getLong(FIELD_TIME_OFFSET_US, 0); SubtitleConfiguration.Builder builder = new SubtitleConfiguration.Builder(uri); return builder @@ -1762,6 +1804,7 @@ public static SubtitleConfiguration fromBundle(Bundle bundle) { .setRoleFlags(roleFlags) .setLabel(label) .setId(id) + .setTimeOffsetUs(timeOffsetUs) .build(); } @@ -1787,6 +1830,9 @@ public Bundle toBundle() { if (id != null) { bundle.putString(FIELD_ID, id); } + if (timeOffsetUs != 0) { + bundle.putLong(FIELD_TIME_OFFSET_US, timeOffsetUs); + } return bundle; } } diff --git a/libraries/common/src/test/java/androidx/media3/common/MediaItemTest.java b/libraries/common/src/test/java/androidx/media3/common/MediaItemTest.java index 60084362b52..27d8d946b82 100644 --- a/libraries/common/src/test/java/androidx/media3/common/MediaItemTest.java +++ b/libraries/common/src/test/java/androidx/media3/common/MediaItemTest.java @@ -328,6 +328,7 @@ public void builderSetSubtitleConfigurations() { .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") .setId("id") + .setTimeOffsetUs(500_000) .build()); MediaItem mediaItem = @@ -396,6 +397,7 @@ public void createSubtitleConfigurationInstance_roundTripViaBundle_yieldsEqualIn .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") .setId("id") + .setTimeOffsetUs(-300_000) .build(); MediaItem.SubtitleConfiguration subtitleConfigurationFromBundle = @@ -800,6 +802,7 @@ public void createLocalConfigurationInstance_roundTripViaBundle_yieldsEqualInsta .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") .setId("id") + .setTimeOffsetUs(250_000) .build())) .setDrmConfiguration( new MediaItem.DrmConfiguration.Builder(C.WIDEVINE_UUID) @@ -953,6 +956,7 @@ public void buildUpon_individualSetters_equalsToOriginal() { .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") .setId("id") + .setTimeOffsetUs(250_000) .build())) .setTag(new Object()) .build(); @@ -1010,6 +1014,7 @@ public void buildUpon_wholeObjectSetters_equalsToOriginal() { .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") .setId("id") + .setTimeOffsetUs(250_000) .build())) .setRequestMetadata( new RequestMetadata.Builder() diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java index e7974d00e54..cc0c7c9b045 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactory.java @@ -598,10 +598,9 @@ public MediaSource createMediaSource(MediaItem mediaItem) { List subtitleConfigurations = castNonNull(mediaItem.localConfiguration).subtitleConfigurations; if (!subtitleConfigurations.isEmpty()) { - MediaSource[] mediaSources = new MediaSource[subtitleConfigurations.size() + 1]; - mediaSources[0] = mediaSource; - for (int i = 0; i < subtitleConfigurations.size(); i++) { - if (parseSubtitlesDuringExtraction) { + if (parseSubtitlesDuringExtraction) { + MediaSource[] subtitleMediaSources = new MediaSource[subtitleConfigurations.size()]; + for (int i = 0; i < subtitleConfigurations.size(); i++) { Format format = new Format.Builder() .setSampleMimeType(subtitleConfigurations.get(i).mimeType) @@ -636,10 +635,17 @@ public MediaSource createMediaSource(MediaItem mediaItem) { if (loadErrorHandlingPolicy != null) { progressiveMediaSourceFactory.setLoadErrorHandlingPolicy(loadErrorHandlingPolicy); } - mediaSources[i + 1] = + subtitleMediaSources[i] = progressiveMediaSourceFactory.createMediaSource( MediaItem.fromUri(subtitleConfigurations.get(i).uri.toString())); - } else { + } + mediaSource = + new SideloadedSubtitlesMediaSource( + mediaSource, subtitleConfigurations, subtitleMediaSources); + } else { + MediaSource[] mediaSources = new MediaSource[subtitleConfigurations.size() + 1]; + mediaSources[0] = mediaSource; + for (int i = 0; i < subtitleConfigurations.size(); i++) { SingleSampleMediaSource.Factory singleSampleMediaSourceFactory = new SingleSampleMediaSource.Factory(dataSourceFactory); if (loadErrorHandlingPolicy != null) { @@ -649,9 +655,8 @@ public MediaSource createMediaSource(MediaItem mediaItem) { singleSampleMediaSourceFactory.createMediaSource( subtitleConfigurations.get(i), /* durationUs= */ C.TIME_UNSET); } + mediaSource = new MergingMediaSource(mediaSources); } - - mediaSource = new MergingMediaSource(mediaSources); } return maybeWrapWithAdsMediaSource( mediaItem, maybeClipMediaSource(mediaItem, mediaSource, enableClippingInMediaPeriod)); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SideloadedSubtitlesMediaSource.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SideloadedSubtitlesMediaSource.java new file mode 100644 index 00000000000..706956b0281 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/SideloadedSubtitlesMediaSource.java @@ -0,0 +1,197 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.source; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +import androidx.annotation.Nullable; +import androidx.media3.common.MediaItem; +import androidx.media3.exoplayer.upstream.Allocator; +import java.util.ArrayList; +import java.util.List; + +/** + * A {@link MediaSource} merging a content source with its sideloaded subtitle sources, applying + * the {@linkplain MediaItem.SubtitleConfiguration.Builder#setTimeOffsetUs(long) time offsets} of + * the subtitle configurations and handling {@link MediaItem} updates that change them. + * + *

Time offset changes are forwarded to the affected subtitle sources without interrupting + * playback. Updates that change the subtitle configurations in any other way are rejected from + * {@link #canUpdateMediaItem}, so that the player falls back to re-preparing the item. + */ +/* package */ final class SideloadedSubtitlesMediaSource extends WrappingMediaSource { + + private final TimeOffsetMediaSource[] subtitleSources; + + private List subtitleConfigurations; + + /** + * Creates the media source. + * + * @param contentMediaSource The content {@link MediaSource}. + * @param subtitleConfigurations The {@link MediaItem.SubtitleConfiguration} instances of the + * sideloaded subtitles. + * @param subtitleMediaSources The sideloaded subtitle {@link MediaSource} instances, in the same + * order as {@code subtitleConfigurations}. + */ + public SideloadedSubtitlesMediaSource( + MediaSource contentMediaSource, + List subtitleConfigurations, + MediaSource[] subtitleMediaSources) { + this( + subtitleConfigurations, + createTimeOffsetSources(subtitleConfigurations, subtitleMediaSources), + contentMediaSource); + } + + private SideloadedSubtitlesMediaSource( + List subtitleConfigurations, + TimeOffsetMediaSource[] subtitleSources, + MediaSource contentMediaSource) { + super(createMergingMediaSource(contentMediaSource, subtitleSources)); + this.subtitleConfigurations = subtitleConfigurations; + this.subtitleSources = subtitleSources; + } + + @Override + public boolean canUpdateMediaItem(MediaItem mediaItem) { + return super.canUpdateMediaItem(mediaItem) && canUpdateSubtitleConfigurations(mediaItem); + } + + @Override + public void updateMediaItem(MediaItem mediaItem) { + super.updateMediaItem(mediaItem); + List newSubtitleConfigurations = + checkNotNull(mediaItem.localConfiguration).subtitleConfigurations; + for (int i = 0; i < subtitleSources.length; i++) { + subtitleSources[i].setTimeOffsetUs(newSubtitleConfigurations.get(i).timeOffsetUs); + } + subtitleConfigurations = newSubtitleConfigurations; + } + + private boolean canUpdateSubtitleConfigurations(MediaItem mediaItem) { + @Nullable MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration; + if (localConfiguration == null + || localConfiguration.subtitleConfigurations.size() != subtitleConfigurations.size()) { + return false; + } + for (int i = 0; i < subtitleConfigurations.size(); i++) { + if (!equalsIgnoringTimeOffset( + localConfiguration.subtitleConfigurations.get(i), subtitleConfigurations.get(i))) { + return false; + } + } + return true; + } + + private static boolean equalsIgnoringTimeOffset( + MediaItem.SubtitleConfiguration subtitleConfiguration, + MediaItem.SubtitleConfiguration other) { + return subtitleConfiguration + .buildUpon() + .setTimeOffsetUs(other.timeOffsetUs) + .build() + .equals(other); + } + + private static TimeOffsetMediaSource[] createTimeOffsetSources( + List subtitleConfigurations, + MediaSource[] subtitleMediaSources) { + checkArgument(subtitleConfigurations.size() == subtitleMediaSources.length); + TimeOffsetMediaSource[] timeOffsetSources = + new TimeOffsetMediaSource[subtitleMediaSources.length]; + for (int i = 0; i < subtitleMediaSources.length; i++) { + timeOffsetSources[i] = + new TimeOffsetMediaSource( + subtitleMediaSources[i], subtitleConfigurations.get(i).timeOffsetUs); + } + return timeOffsetSources; + } + + private static MergingMediaSource createMergingMediaSource( + MediaSource contentMediaSource, TimeOffsetMediaSource[] subtitleSources) { + MediaSource[] mediaSources = new MediaSource[subtitleSources.length + 1]; + mediaSources[0] = contentMediaSource; + System.arraycopy(subtitleSources, 0, mediaSources, 1, subtitleSources.length); + return new MergingMediaSource(mediaSources); + } + + /** + * A {@link MediaSource} that applies a time offset to the timestamps of a wrapped {@link + * MediaSource}, and allows updating the offset during playback. + * + *

A positive offset shifts the samples of the wrapped source to later positions on the + * playback timeline, a negative offset shifts them to earlier positions. The {@link + * androidx.media3.common.Timeline} of the wrapped source is not adjusted, so this source relies + * on being merged with another source that defines the timeline. + */ + private static final class TimeOffsetMediaSource extends WrappingMediaSource { + + private final ArrayList activeMediaPeriods; + + private long timeOffsetUs; + + /** + * Creates the time offset source. + * + * @param mediaSource The wrapped {@link MediaSource}. + * @param timeOffsetUs The offset to apply to all timestamps coming from the wrapped source, in + * microseconds. + */ + public TimeOffsetMediaSource(MediaSource mediaSource, long timeOffsetUs) { + super(mediaSource); + this.timeOffsetUs = timeOffsetUs; + this.activeMediaPeriods = new ArrayList<>(); + } + + /** + * Updates the offset that is applied to all timestamps coming from the wrapped source. + * + *

Must be called on the playback thread. + * + *

The new offset is applied to all future interactions with this source and its active + * {@linkplain MediaPeriod media periods}. Data already read from the sample streams of active + * periods is unaffected, see {@link TimeOffsetMediaPeriod#updateTimeOffsetUs(long)}. + * + * @param timeOffsetUs The offset to apply to all timestamps coming from the wrapped source, in + * microseconds. + */ + public void setTimeOffsetUs(long timeOffsetUs) { + this.timeOffsetUs = timeOffsetUs; + for (int i = 0; i < activeMediaPeriods.size(); i++) { + activeMediaPeriods.get(i).updateTimeOffsetUs(timeOffsetUs); + } + } + + @Override + public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long startPositionUs) { + TimeOffsetMediaPeriod mediaPeriod = + new TimeOffsetMediaPeriod( + mediaSource.createPeriod(id, allocator, startPositionUs - timeOffsetUs), + timeOffsetUs); + activeMediaPeriods.add(mediaPeriod); + return mediaPeriod; + } + + @Override + public void releasePeriod(MediaPeriod mediaPeriod) { + TimeOffsetMediaPeriod timeOffsetMediaPeriod = (TimeOffsetMediaPeriod) mediaPeriod; + activeMediaPeriods.remove(timeOffsetMediaPeriod); + mediaSource.releasePeriod(timeOffsetMediaPeriod.getWrappedMediaPeriod()); + } + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriod.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriod.java index 9c72c0cd7ba..d38f99c4e9f 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriod.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriod.java @@ -31,12 +31,12 @@ import java.util.List; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; -/** A {@link MediaPeriod} that applies a fixed time offset to all timestamps */ +/** A {@link MediaPeriod} that applies a time offset to all timestamps */ /* package */ final class TimeOffsetMediaPeriod implements MediaPeriod, MediaPeriod.Callback { private final MediaPeriod mediaPeriod; - private final long timeOffsetUs; + private long timeOffsetUs; private @MonotonicNonNull Callback callback; /** @@ -56,6 +56,21 @@ public MediaPeriod getWrappedMediaPeriod() { return mediaPeriod; } + /** + * Updates the offset that is applied to all timestamps coming from the wrapped period. + * + *

The new offset applies to all future interactions with this period and its {@linkplain + * SampleStream sample streams}. Data already read with the previous offset is unaffected. To + * apply the new offset to the data at the current playback position, the tracks of this period + * need to be re-selected (or seeked) so that the wrapped period re-reads this data. + * + * @param timeOffsetUs The offset to apply to all timestamps coming from the wrapped period, in + * microseconds. + */ + public void updateTimeOffsetUs(long timeOffsetUs) { + this.timeOffsetUs = timeOffsetUs; + } + @Override public void prepare(Callback callback, long positionUs) { this.callback = callback; @@ -102,7 +117,7 @@ public long selectTracks( streams[i] = null; } else if (streams[i] == null || ((TimeOffsetSampleStream) streams[i]).getChildStream() != childStream) { - streams[i] = new TimeOffsetSampleStream(childStream, timeOffsetUs); + streams[i] = new TimeOffsetSampleStream(childStream); } } return startPositionUs + timeOffsetUs; @@ -192,14 +207,12 @@ public long setEndPositionUs(long endPositionUs) { : actualEndPositionUs + timeOffsetUs; } - private static final class TimeOffsetSampleStream implements SampleStream { + private final class TimeOffsetSampleStream implements SampleStream { private final SampleStream sampleStream; - private final long timeOffsetUs; - public TimeOffsetSampleStream(SampleStream sampleStream, long timeOffsetUs) { + public TimeOffsetSampleStream(SampleStream sampleStream) { this.sampleStream = sampleStream; - this.timeOffsetUs = timeOffsetUs; } public SampleStream getChildStream() { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/SubtitlePlaybackTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/SubtitlePlaybackTest.java index 30b2a7038dd..3ccfeb911b7 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/SubtitlePlaybackTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/e2etest/SubtitlePlaybackTest.java @@ -16,6 +16,7 @@ package androidx.media3.exoplayer.e2etest; import static androidx.media3.test.utils.robolectric.TestPlayerRunHelper.advance; +import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.truth.Truth.assertThat; import android.content.Context; @@ -28,6 +29,7 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.ParserException; import androidx.media3.common.Player; +import androidx.media3.common.text.CueGroup; import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.analytics.AnalyticsListener; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; @@ -62,6 +64,198 @@ public class SubtitlePlaybackTest { public ShadowMediaCodecConfig mediaCodecConfig = ShadowMediaCodecConfig.withAllDefaultSupportedCodecs(); + // https://github.com/androidx/media/issues/1976 + @Test + public void sideloadedSubtitle_withPositiveTimeOffset_cuesShiftedLater() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ true); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext, clock); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setClock(clock) + .build(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1)); + player.setVideoSurface(surface); + List cueChangeTimesUs = new ArrayList<>(); + List cueTexts = new ArrayList<>(); + player.addListener(createNonEmptyCueGroupCollectingListener(cueChangeTimesUs, cueTexts)); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri("asset:///media/mp4/preroll-5s.mp4") + .setSubtitleConfigurations( + ImmutableList.of( + new MediaItem.SubtitleConfiguration.Builder( + Uri.parse("asset:///media/webvtt/typical")) + .setMimeType(MimeTypes.TEXT_VTT) + .setLanguage("en") + .setSelectionFlags(C.SELECTION_FLAG_DEFAULT) + .setTimeOffsetUs(300_000) + .build())) + .build(); + + player.setMediaItem(mediaItem); + player.prepare(); + advance(player).untilState(Player.STATE_READY); + advance(player).untilFullyBuffered(); + player.play(); + advance(player).untilState(Player.STATE_ENDED); + player.release(); + surface.release(); + + assertThat(cueChangeTimesUs).containsExactly(300_000L, 2_645_000L).inOrder(); + assertThat(cueTexts) + .containsExactly("This is the first subtitle.", "This is the second subtitle.") + .inOrder(); + } + + // https://github.com/androidx/media/issues/1976 + @Test + public void sideloadedSubtitle_withNegativeTimeOffset_cuesShiftedEarlier() throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ true); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext, clock); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setClock(clock) + .build(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1)); + player.setVideoSurface(surface); + List cueChangeTimesUs = new ArrayList<>(); + List cueTexts = new ArrayList<>(); + player.addListener(createNonEmptyCueGroupCollectingListener(cueChangeTimesUs, cueTexts)); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri("asset:///media/mp4/preroll-5s.mp4") + .setSubtitleConfigurations( + ImmutableList.of( + new MediaItem.SubtitleConfiguration.Builder( + Uri.parse("asset:///media/webvtt/typical")) + .setMimeType(MimeTypes.TEXT_VTT) + .setLanguage("en") + .setSelectionFlags(C.SELECTION_FLAG_DEFAULT) + .setTimeOffsetUs(-2_000_000) + .build())) + .build(); + + player.setMediaItem(mediaItem); + player.prepare(); + advance(player).untilState(Player.STATE_READY); + advance(player).untilFullyBuffered(); + player.play(); + advance(player).untilState(Player.STATE_ENDED); + player.release(); + surface.release(); + + // The first cue ends before the start of the media, so only the second one is shown. + assertThat(cueChangeTimesUs).containsExactly(345_000L); + assertThat(cueTexts).containsExactly("This is the second subtitle."); + } + + // https://github.com/androidx/media/issues/1976 + @Test + public void sideloadedSubtitle_timeOffsetUpdatedDuringPlayback_playbackContinuesWithShiftedCues() + throws Exception { + Context applicationContext = ApplicationProvider.getApplicationContext(); + FakeClock clock = new FakeClock(/* isAutoAdvancing= */ true); + CapturingRenderersFactory capturingRenderersFactory = + new CapturingRenderersFactory(applicationContext, clock); + ExoPlayer player = + new ExoPlayer.Builder(applicationContext, capturingRenderersFactory) + .setClock(clock) + .build(); + Surface surface = new Surface(new SurfaceTexture(/* texName= */ 1)); + player.setVideoSurface(surface); + List cueChangeTimesUs = new ArrayList<>(); + List cueTexts = new ArrayList<>(); + player.addListener(createNonEmptyCueGroupCollectingListener(cueChangeTimesUs, cueTexts)); + List playbackStates = new ArrayList<>(); + player.addListener( + new Player.Listener() { + @Override + public void onPlaybackStateChanged(@Player.State int playbackState) { + playbackStates.add(playbackState); + } + }); + MediaItem.SubtitleConfiguration subtitleConfiguration = + new MediaItem.SubtitleConfiguration.Builder(Uri.parse("asset:///media/webvtt/typical")) + .setMimeType(MimeTypes.TEXT_VTT) + .setLanguage("en") + .setSelectionFlags(C.SELECTION_FLAG_DEFAULT) + .build(); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri("asset:///media/mp4/preroll-5s.mp4") + .setSubtitleConfigurations(ImmutableList.of(subtitleConfiguration)) + .build(); + + player.setMediaItem(mediaItem); + player.prepare(); + advance(player).untilState(Player.STATE_READY); + advance(player).untilFullyBuffered(); + advance(player).untilPosition(/* mediaItemIndex= */ 0, /* positionMs= */ 2000); + // Shift the subtitles two seconds later and re-enable the text track to apply the new offset + // to the cues around the current position. + player.replaceMediaItem( + /* index= */ 0, + mediaItem + .buildUpon() + .setSubtitleConfigurations( + ImmutableList.of( + subtitleConfiguration.buildUpon().setTimeOffsetUs(2_000_000).build())) + .build()); + player.setTrackSelectionParameters( + player + .getTrackSelectionParameters() + .buildUpon() + .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, /* disabled= */ true) + .build()); + player.setTrackSelectionParameters( + player + .getTrackSelectionParameters() + .buildUpon() + .setTrackTypeDisabled(C.TRACK_TYPE_TEXT, /* disabled= */ false) + .build()); + player.play(); + advance(player).untilState(Player.STATE_ENDED); + long updatedTimeOffsetUs = + checkNotNull(player.getCurrentMediaItem().localConfiguration) + .subtitleConfigurations + .get(0) + .timeOffsetUs; + player.release(); + surface.release(); + + assertThat(updatedTimeOffsetUs).isEqualTo(2_000_000); + // The first cue is shown with the initial zero offset, then again when the text track is + // re-enabled at two seconds with the new offset, followed by the shifted second cue. + assertThat(cueChangeTimesUs).containsExactly(0L, 2_000_000L, 4_345_000L).inOrder(); + assertThat(cueTexts) + .containsExactly( + "This is the first subtitle.", + "This is the first subtitle.", + "This is the second subtitle.") + .inOrder(); + // The media item update must not interrupt playback with a re-preparation. + assertThat(playbackStates) + .containsExactly(Player.STATE_BUFFERING, Player.STATE_READY, Player.STATE_ENDED) + .inOrder(); + } + + private static Player.Listener createNonEmptyCueGroupCollectingListener( + List cueChangeTimesUs, List cueTexts) { + return new Player.Listener() { + @Override + public void onCues(CueGroup cueGroup) { + if (!cueGroup.cues.isEmpty()) { + cueChangeTimesUs.add(cueGroup.presentationTimeUs); + cueTexts.add(String.valueOf(cueGroup.cues.get(0).text)); + } + } + }; + } + // https://github.com/androidx/media/issues/1721 @Test public void multipleSideloadedSubtitles_noneSelected_noneLoaded() throws Exception { diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactoryTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactoryTest.java index 1ba66100260..e35700c8a78 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactoryTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/DefaultMediaSourceFactoryTest.java @@ -29,6 +29,7 @@ import androidx.media3.exoplayer.source.ads.AdsMediaSource; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; import java.util.Arrays; import java.util.List; import org.junit.Test; @@ -75,7 +76,7 @@ public void createMediaSource_withPath_progressiveSource() { } @Test - public void createMediaSource_withSubtitle_isMergingMediaSource() { + public void createMediaSource_withSubtitle_isSideloadedSubtitlesMediaSource() { DefaultMediaSourceFactory defaultMediaSourceFactory = new DefaultMediaSourceFactory((Context) ApplicationProvider.getApplicationContext()); List subtitleConfigurations = @@ -97,7 +98,63 @@ public void createMediaSource_withSubtitle_isMergingMediaSource() { MediaSource mediaSource = defaultMediaSourceFactory.createMediaSource(mediaItem); - assertThat(mediaSource).isInstanceOf(MergingMediaSource.class); + assertThat(mediaSource).isInstanceOf(SideloadedSubtitlesMediaSource.class); + } + + @Test + public void canUpdateMediaItem_withChangedSubtitleTimeOffset_returnsTrue() { + DefaultMediaSourceFactory defaultMediaSourceFactory = + new DefaultMediaSourceFactory((Context) ApplicationProvider.getApplicationContext()); + MediaItem.SubtitleConfiguration subtitleConfiguration = + new MediaItem.SubtitleConfiguration.Builder(Uri.parse(URI_TEXT)) + .setMimeType(MimeTypes.APPLICATION_TTML) + .setLanguage("en") + .build(); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(URI_MEDIA) + .setSubtitleConfigurations(ImmutableList.of(subtitleConfiguration)) + .build(); + MediaSource mediaSource = defaultMediaSourceFactory.createMediaSource(mediaItem); + + MediaItem updatedMediaItem = + mediaItem + .buildUpon() + .setSubtitleConfigurations( + ImmutableList.of( + subtitleConfiguration.buildUpon().setTimeOffsetUs(1_000_000).build())) + .build(); + + assertThat(mediaSource.canUpdateMediaItem(updatedMediaItem)).isTrue(); + } + + @Test + public void canUpdateMediaItem_withStructurallyChangedSubtitleConfigurations_returnsFalse() { + DefaultMediaSourceFactory defaultMediaSourceFactory = + new DefaultMediaSourceFactory((Context) ApplicationProvider.getApplicationContext()); + MediaItem.SubtitleConfiguration subtitleConfiguration = + new MediaItem.SubtitleConfiguration.Builder(Uri.parse(URI_TEXT)) + .setMimeType(MimeTypes.APPLICATION_TTML) + .setLanguage("en") + .build(); + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(URI_MEDIA) + .setSubtitleConfigurations(ImmutableList.of(subtitleConfiguration)) + .build(); + MediaSource mediaSource = defaultMediaSourceFactory.createMediaSource(mediaItem); + + MediaItem mediaItemWithChangedLanguage = + mediaItem + .buildUpon() + .setSubtitleConfigurations( + ImmutableList.of(subtitleConfiguration.buildUpon().setLanguage("de").build())) + .build(); + MediaItem mediaItemWithoutSubtitles = + mediaItem.buildUpon().setSubtitleConfigurations(ImmutableList.of()).build(); + + assertThat(mediaSource.canUpdateMediaItem(mediaItemWithChangedLanguage)).isFalse(); + assertThat(mediaSource.canUpdateMediaItem(mediaItemWithoutSubtitles)).isFalse(); } @Test diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriodTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriodTest.java index d4dc94ebebb..d3a6e5b69b7 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriodTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/source/TimeOffsetMediaPeriodTest.java @@ -83,6 +83,37 @@ public void selectTracks_createsSampleStreamCorrectingOffset() throws Exception assertThat(readEndOfStreamBuffer).isTrue(); } + @Test + public void updateTimeOffsetUs_appliesNewOffsetToExistingSampleStreamsAndSeeks() throws Exception { + FakeMediaPeriod fakeMediaPeriod = + createFakeMediaPeriod( + ImmutableList.of( + oneByteSample(/* timeUs= */ 8000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 12000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + MediaPeriod spyPeriod = spy(fakeMediaPeriod); + TimeOffsetMediaPeriod timeOffsetMediaPeriod = + new TimeOffsetMediaPeriod(spyPeriod, /* timeOffsetUs= */ -3000); + prepareMediaPeriodSync(timeOffsetMediaPeriod, /* positionUs= */ 0); + FormatHolder formatHolder = new FormatHolder(); + DecoderInputBuffer inputBuffer = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + SampleStream sampleStream = selectTracksOnMediaPeriodAndTriggerLoading(timeOffsetMediaPeriod); + sampleStream.readData(formatHolder, inputBuffer, FLAG_REQUIRE_FORMAT); + sampleStream.readData(formatHolder, inputBuffer, /* readFlags= */ 0); + long firstReadBufferTimeUs = inputBuffer.timeUs; + + timeOffsetMediaPeriod.updateTimeOffsetUs(/* timeOffsetUs= */ 2000); + sampleStream.readData(formatHolder, inputBuffer, /* readFlags= */ 0); + long secondReadBufferTimeUs = inputBuffer.timeUs; + long seekResultTimeUs = timeOffsetMediaPeriod.seekToUs(/* positionUs= */ 7000); + + assertThat(firstReadBufferTimeUs).isEqualTo(5000); + assertThat(secondReadBufferTimeUs).isEqualTo(14000); + verify(spyPeriod).seekToUs(5000); + assertThat(seekResultTimeUs).isEqualTo(7000); + } + @Test public void setEndPositionUs_isForwardedWithTimeOffset() throws Exception { FakeMediaPeriod fakeMediaPeriod =