From aed21e464e5aaff70956fc5d804d5517da16cd39 Mon Sep 17 00:00:00 2001 From: michaelkatz Date: Mon, 7 Aug 2023 10:04:50 +0000 Subject: [PATCH] Fix last sample detection issue for ClippingMediaSource If stream is not considered final and `ClippingMediaSource` is read to `endUs`, then `BaseRenderer.readSource` returns `C.RESULT_NOTHING_READ`. In that case, the `lastBufferInStreamPresentationTimeUs` is not set and the last frame is not rendered. PiperOrigin-RevId: 554418971 --- .../mediacodec/MediaCodecRenderer.java | 5 + .../video/MediaCodecVideoRendererTest.java | 225 ++++++++++++++++++ 2 files changed, 230 insertions(+) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 0566d5f91e..0279409c51 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -1306,6 +1306,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer { } if (result == C.RESULT_NOTHING_READ) { + if (hasReadStreamToEnd()) { + // Notify output queue of the last buffer's timestamp. + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; + } return false; } if (result == C.RESULT_FORMAT_READ) { @@ -1321,6 +1325,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { // We've read a buffer. if (buffer.isEndOfStream()) { + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; if (codecReconfigurationState == RECONFIGURATION_STATE_QUEUE_PENDING) { // We received a new format immediately before the end of the stream. We need to clear // the corresponding reconfiguration data from the current buffer, but re-write it into diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java index 8f77f80e80..83cf4cd401 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/MediaCodecVideoRendererTest.java @@ -46,6 +46,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.TrackGroup; import androidx.media3.common.VideoSize; import androidx.media3.common.util.Clock; import androidx.media3.decoder.CryptoInfo; @@ -61,8 +62,19 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; import androidx.media3.exoplayer.mediacodec.MediaCodecInfo; import androidx.media3.exoplayer.mediacodec.MediaCodecSelector; import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter; +import androidx.media3.exoplayer.source.ClippingMediaPeriod; +import androidx.media3.exoplayer.source.MediaPeriod; +import androidx.media3.exoplayer.source.MediaSource; +import androidx.media3.exoplayer.source.MediaSourceEventListener; +import androidx.media3.exoplayer.source.SampleStream; +import androidx.media3.exoplayer.source.TrackGroupArray; +import androidx.media3.exoplayer.trackselection.ExoTrackSelection; +import androidx.media3.exoplayer.trackselection.FixedTrackSelection; +import androidx.media3.exoplayer.upstream.Allocator; import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeMediaPeriod; import androidx.media3.test.utils.FakeSampleStream; +import androidx.media3.test.utils.robolectric.RobolectricUtil; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; @@ -71,6 +83,7 @@ import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; import org.junit.After; import org.junit.Before; @@ -98,6 +111,8 @@ public class MediaCodecVideoRendererTest { .setHeight(1080) .build(); + private static final TrackGroup TRACK_GROUP_H264 = new TrackGroup(VIDEO_H264); + private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO = MediaCodecInfo.newInstance( /* name= */ "h264-codec-hw", @@ -279,6 +294,216 @@ public class MediaCodecVideoRendererTest { assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2); } + @Test + public void + render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame() + throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + // Set up MediaPeriod with samples. + FakeMediaPeriod mediaPeriod = + new FakeMediaPeriod( + new TrackGroupArray(TRACK_GROUP_H264), + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* trackDataFactory= */ (format, mediaPeriodId) -> ImmutableList.of(), + new MediaSourceEventListener.EventDispatcher() + .withParameters( + /* windowIndex= */ 0, + new MediaSource.MediaPeriodId(/* periodUid= */ new Object())), + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* deferOnPrepared= */ false) { + @Override + protected FakeSampleStream createSampleStream( + Allocator allocator, + @Nullable MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher, + DrmSessionManager drmSessionManager, + DrmSessionEventListener.EventDispatcher drmEventDispatcher, + Format initialFormat, + List fakeSampleStreamItems) { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + allocator, + mediaSourceEventDispatcher, + drmSessionManager, + drmEventDispatcher, + initialFormat, + /* fakeSampleStreamItems= */ ImmutableList.of( + oneByteSample(/* timeUs= */ 90, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(0); + return fakeSampleStream; + } + }; + ClippingMediaPeriod clippingMediaPeriod = + new ClippingMediaPeriod( + mediaPeriod, + /* enableInitialDiscontinuity= */ true, + /* startUs= */ 0, + /* endUs= */ 100); + AtomicBoolean periodPrepared = new AtomicBoolean(); + clippingMediaPeriod.prepare( + new MediaPeriod.Callback() { + @Override + public void onPrepared(MediaPeriod mediaPeriod) { + periodPrepared.set(true); + } + + @Override + public void onContinueLoadingRequested(MediaPeriod source) { + clippingMediaPeriod.continueLoading(/* positionUs= */ 0); + } + }, + /* positionUs= */ 100); + RobolectricUtil.runMainLooperUntil(periodPrepared::get); + SampleStream[] sampleStreams = new SampleStream[1]; + clippingMediaPeriod.selectTracks( + new ExoTrackSelection[] {new FixedTrackSelection(TRACK_GROUP_H264, /* track= */ 0)}, + /* mayRetainStreamFlags= */ new boolean[] {false}, + sampleStreams, + /* streamResetFlags= */ new boolean[] {true}, + /* positionUs= */ 100); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + sampleStreams[0], + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 100, + /* offsetUs= */ 0); + + mediaCodecVideoRenderer.start(); + // Call to render should have read all samples up before endUs. + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000L); + assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue(); + // Following call to render should force-render last frame. + mediaCodecVideoRenderer.render(100, SystemClock.elapsedRealtime() * 1000L); + shadowOf(testMainLooper).idle(); + + verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong()); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1); + } + + @Test + public void render_withClippingMediaPeriodSetCurrentStreamFinal_rendersLastFrame() + throws Exception { + ArgumentCaptor argumentDecoderCounters = + ArgumentCaptor.forClass(DecoderCounters.class); + // Set up MediaPeriod with samples. + FakeMediaPeriod mediaPeriod = + new FakeMediaPeriod( + new TrackGroupArray(TRACK_GROUP_H264), + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* trackDataFactory= */ (format, mediaPeriodId) -> ImmutableList.of(), + new MediaSourceEventListener.EventDispatcher() + .withParameters( + /* windowIndex= */ 0, + new MediaSource.MediaPeriodId(/* periodUid= */ new Object())), + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* deferOnPrepared= */ false) { + @Override + protected FakeSampleStream createSampleStream( + Allocator allocator, + @Nullable MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher, + DrmSessionManager drmSessionManager, + DrmSessionEventListener.EventDispatcher drmEventDispatcher, + Format initialFormat, + List fakeSampleStreamItems) { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + allocator, + mediaSourceEventDispatcher, + drmSessionManager, + drmEventDispatcher, + initialFormat, + /* fakeSampleStreamItems= */ ImmutableList.of( + oneByteSample(/* timeUs= */ 90, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(0); + return fakeSampleStream; + } + }; + ClippingMediaPeriod clippingMediaPeriod = + new ClippingMediaPeriod( + mediaPeriod, + /* enableInitialDiscontinuity= */ true, + /* startUs= */ 0, + /* endUs= */ 100); + AtomicBoolean periodPrepared = new AtomicBoolean(); + clippingMediaPeriod.prepare( + new MediaPeriod.Callback() { + @Override + public void onPrepared(MediaPeriod mediaPeriod) { + periodPrepared.set(true); + } + + @Override + public void onContinueLoadingRequested(MediaPeriod source) { + clippingMediaPeriod.continueLoading(/* positionUs= */ 0); + } + }, + /* positionUs= */ 100); + RobolectricUtil.runMainLooperUntil(periodPrepared::get); + SampleStream[] sampleStreams = new SampleStream[1]; + clippingMediaPeriod.selectTracks( + new ExoTrackSelection[] {new FixedTrackSelection(TRACK_GROUP_H264, /* track= */ 0)}, + /* mayRetainStreamFlags= */ new boolean[] {false}, + sampleStreams, + /* streamResetFlags= */ new boolean[] {true}, + /* positionUs= */ 100); + mediaCodecVideoRenderer = + new MediaCodecVideoRenderer( + ApplicationProvider.getApplicationContext(), + new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3), + mediaCodecSelector, + /* allowedJoiningTimeMs= */ 0, + /* enableDecoderFallback= */ false, + /* eventHandler= */ new Handler(testMainLooper), + /* eventListener= */ eventListener, + /* maxDroppedFramesToNotify= */ 1); + mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface); + mediaCodecVideoRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {VIDEO_H264}, + sampleStreams[0], + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 100, + /* offsetUs= */ 0); + + mediaCodecVideoRenderer.start(); + mediaCodecVideoRenderer.setCurrentStreamFinal(); + // Call to render should have read all samples up before endUs. + mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000L); + assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue(); + // Following call to render should force-render last frame. + mediaCodecVideoRenderer.render(100, SystemClock.elapsedRealtime() * 1000L); + shadowOf(testMainLooper).idle(); + + verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong()); + verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture()); + assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1); + } + @Test public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception { FakeSampleStream fakeSampleStream =