diff --git a/RELEASENOTES.md b/RELEASENOTES.md index a0f121c8c8..87dee0fd54 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -12,6 +12,9 @@ * Fix potential `IndexOutOfBoundsException` caused by extractors reporting additional tracks after the initial preparation step ([#1476](https://github.com/androidx/media/issues/1476)). + * `Effects` in `ExoPlayer.setVideoEffect()` will receive the timestamps + with the renderer offset removed + ([#1098](https://github.com/androidx/media/issues/1098)). * Transformer: * Track Selection: * Extractors: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index af5a7002da..07038b3be4 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -75,6 +75,7 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecRenderer; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; +import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; @@ -177,6 +178,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer private int tunnelingAudioSessionId; /* package */ @Nullable OnFrameRenderedListenerV23 tunnelingOnFrameRenderedListener; @Nullable private VideoFrameMetadataListener frameMetadataListener; + private long startPositionUs; /** * @param context A context. @@ -414,6 +416,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer tunnelingAudioSessionId = C.AUDIO_SESSION_ID_UNSET; reportedVideoSize = null; rendererPriority = C.PRIORITY_PLAYBACK; + startPositionUs = C.TIME_UNSET; } // FrameTimingEvaluator methods @@ -714,6 +717,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer } } + @Override + protected void onStreamChanged( + Format[] formats, + long startPositionUs, + long offsetUs, + MediaSource.MediaPeriodId mediaPeriodId) + throws ExoPlaybackException { + super.onStreamChanged(formats, startPositionUs, offsetUs, mediaPeriodId); + if (this.startPositionUs == C.TIME_UNSET) { + this.startPositionUs = startPositionUs; + } + } + @Override protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { if (videoSink != null) { @@ -814,6 +830,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer super.onReset(); } finally { hasSetVideoSink = false; + startPositionUs = C.TIME_UNSET; if (placeholderSurface != null) { releasePlaceholderSurface(); } @@ -1446,8 +1463,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer * position) to the frame presentation time, in microseconds. */ protected long getBufferTimestampAdjustmentUs() { - // TODO - b/333514379: Make effect-enabled effect timestamp start from zero. - return 0; + return -startPositionUs; } private boolean maybeReleaseFrame( diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoTimestampConsistencyTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/VideoTimestampConsistencyTest.java similarity index 81% rename from libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoTimestampConsistencyTest.java rename to libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/VideoTimestampConsistencyTest.java index 05629033f0..e33f75543c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoTimestampConsistencyTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/VideoTimestampConsistencyTest.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package androidx.media3.transformer; +package androidx.media3.transformer.mh; import static androidx.media3.common.util.Util.usToMs; import static androidx.media3.transformer.AndroidTestUtil.JPG_SINGLE_PIXEL_ASSET; @@ -27,6 +27,18 @@ import android.view.SurfaceView; import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; import androidx.media3.effect.GlEffect; +import androidx.media3.exoplayer.ExoPlayer; +import androidx.media3.transformer.Composition; +import androidx.media3.transformer.CompositionPlayer; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.EditedMediaItemSequence; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.ExportTestResult; +import androidx.media3.transformer.InputTimestampRecordingShaderProgram; +import androidx.media3.transformer.PlayerTestListener; +import androidx.media3.transformer.SurfaceTestActivity; +import androidx.media3.transformer.Transformer; +import androidx.media3.transformer.TransformerAndroidTestRunner; import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; @@ -68,6 +80,7 @@ public class VideoTimestampConsistencyTest { private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); private final Context applicationContext = instrumentation.getContext().getApplicationContext(); + private ExoPlayer exoplayer; private CompositionPlayer compositionPlayer; private SurfaceView surfaceView; @@ -95,7 +108,8 @@ public class VideoTimestampConsistencyTest { .setFrameRate(30) .build(); - compareTimestamps(ImmutableList.of(image), IMAGE_TIMESTAMPS_US_500_MS_30_FPS); + compareTimestamps( + ImmutableList.of(image), IMAGE_TIMESTAMPS_US_500_MS_30_FPS, /* containsImage= */ true); } @Test @@ -105,7 +119,8 @@ public class VideoTimestampConsistencyTest { .setDurationUs(MP4_ASSET.videoDurationUs) .build(); - compareTimestamps(ImmutableList.of(video), MP4_ASSET_FRAME_TIMESTAMPS_US); + compareTimestamps( + ImmutableList.of(video), MP4_ASSET_FRAME_TIMESTAMPS_US, /* containsImage= */ false); } @Test @@ -138,7 +153,8 @@ public class VideoTimestampConsistencyTest { timestampUs -> ((MP4_ASSET.videoDurationUs - clippedStartUs) + timestampUs))) .build(); - compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps); + compareTimestamps( + ImmutableList.of(video1, video2), expectedTimestamps, /* containsImage= */ false); } @Test @@ -162,7 +178,8 @@ public class VideoTimestampConsistencyTest { timestampUs -> (MP4_ASSET.videoDurationUs + timestampUs))) .build(); - compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps); + compareTimestamps( + ImmutableList.of(video1, video2), expectedTimestamps, /* containsImage= */ false); } @Test @@ -198,7 +215,8 @@ public class VideoTimestampConsistencyTest { timestampUs -> (timestampUs + imageDurationUs))) .build(); - compareTimestamps(ImmutableList.of(image1, image2), expectedTimestamps); + compareTimestamps( + ImmutableList.of(image1, image2), expectedTimestamps, /* containsImage= */ true); } @Test @@ -227,7 +245,8 @@ public class VideoTimestampConsistencyTest { MP4_ASSET_FRAME_TIMESTAMPS_US, timestampUs -> (timestampUs + imageDurationUs))) .build(); - compareTimestamps(ImmutableList.of(image, video), expectedTimestamps); + compareTimestamps( + ImmutableList.of(image, video), expectedTimestamps, /* containsImage= */ true); } @Test @@ -257,7 +276,8 @@ public class VideoTimestampConsistencyTest { timestampUs -> (MP4_ASSET.videoDurationUs + timestampUs))) .build(); - compareTimestamps(ImmutableList.of(video, image), expectedTimestamps); + compareTimestamps( + ImmutableList.of(video, image), expectedTimestamps, /* containsImage= */ true); } @Test @@ -295,16 +315,26 @@ public class VideoTimestampConsistencyTest { timestampUs -> ((MP4_ASSET.videoDurationUs - clippedStartUs) + timestampUs))) .build(); - compareTimestamps(ImmutableList.of(video, image), expectedTimestamps); + compareTimestamps( + ImmutableList.of(video, image), expectedTimestamps, /* containsImage= */ true); } - private void compareTimestamps(List mediaItems, List expectedTimestamps) + private void compareTimestamps( + List mediaItems, List expectedTimestamps, boolean containsImage) throws Exception { ImmutableList timestampsFromCompositionPlayer = getTimestampsFromCompositionPlayer(mediaItems); ImmutableList timestampsFromTransformer = getTimestampsFromTransformer(mediaItems); assertThat(timestampsFromCompositionPlayer).isEqualTo(timestampsFromTransformer); + + if (!containsImage) { + // ExoPlayer doesn't support image playback with effects. + ImmutableList timestampsFromExoPlayer = + getTimestampsFromExoPlayer( + Lists.transform(mediaItems, editedMediaItem -> editedMediaItem.mediaItem)); + assertThat(timestampsFromCompositionPlayer).isEqualTo(timestampsFromExoPlayer); + } assertThat(timestampsFromTransformer).isEqualTo(expectedTimestamps); } @@ -318,6 +348,7 @@ public class VideoTimestampConsistencyTest { /* effects= */ ImmutableList.of( (GlEffect) (context, useHdr) -> timestampRecordingShaderProgram)); + @SuppressWarnings("unused") ExportTestResult result = new TransformerAndroidTestRunner.Builder( applicationContext, new Transformer.Builder(applicationContext).build()) @@ -365,6 +396,32 @@ public class VideoTimestampConsistencyTest { return timestampRecordingShaderProgram.getInputTimestampsUs(); } + private ImmutableList getTimestampsFromExoPlayer(List mediaItems) + throws Exception { + PlayerTestListener playerListener = new PlayerTestListener(TEST_TIMEOUT_MS); + InputTimestampRecordingShaderProgram timestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + + instrumentation.runOnMainSync( + () -> { + exoplayer = new ExoPlayer.Builder(applicationContext).build(); + // Set a surface on the player even though there is no UI on this test. We need a surface + // otherwise the player will skip/drop video frames. + exoplayer.setVideoSurfaceView(surfaceView); + exoplayer.addListener(playerListener); + exoplayer.setMediaItems(mediaItems); + exoplayer.setVideoEffects( + ImmutableList.of((GlEffect) (context, useHdr) -> timestampRecordingShaderProgram)); + exoplayer.prepare(); + exoplayer.play(); + }); + + playerListener.waitUntilPlayerEnded(); + instrumentation.runOnMainSync(() -> exoplayer.release()); + + return timestampRecordingShaderProgram.getInputTimestampsUs(); + } + private static ImmutableList prependVideoEffects( List editedMediaItems, List effects) { ImmutableList.Builder prependedItems = new ImmutableList.Builder<>(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java index 8a7fd8479b..e17ed9e6f3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java @@ -293,7 +293,7 @@ public final class EditedMediaItem { } /** Returns a {@link Builder} initialized with the values of this instance. */ - /* package */ Builder buildUpon() { + public Builder buildUpon() { return new Builder(this); }