From 56dd0f761d158c34370d842016e685fbc44a493f Mon Sep 17 00:00:00 2001 From: tonihei Date: Fri, 24 Mar 2023 20:39:40 +0000 Subject: [PATCH] Handle output format changes for empty sample streams correctly When MediaCodecRenderer is given an empty sample stream, it puts its output format change tracking in a bad state where we never process future stream changes because we are waiting for a sample that doesn't exist. We can fix this by: - Looping the pending output stream changes to see if we processed more than one change at once (this fixes the tracking for empty sample streams that are not the first in the queue). - Checking if none of the previous streams queued any samples in onStreamChanged to handle this in the same way as the case where we already output all samples (this fixes the problem when the empty sample stream comes first in the queue). - Also calling onProcessedStreamChange for the case above, which was missing previously. #minor-release PiperOrigin-RevId: 519226637 (cherry picked from commit b9790e69d7649d3399b9b1f920aa417ba4cc38c1) --- .../mediacodec/MediaCodecRenderer.java | 20 +++- .../mediacodec/MediaCodecRendererTest.java | 109 ++++++++++++++++++ 2 files changed, 123 insertions(+), 6 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 1263bad907..fddfea4e2a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -642,14 +642,22 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @Override protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) throws ExoPlaybackException { - if (outputStreamInfo.streamOffsetUs == C.TIME_UNSET - || (pendingOutputStreamChanges.isEmpty() - && lastProcessedOutputBufferTimeUs != C.TIME_UNSET - && lastProcessedOutputBufferTimeUs >= largestQueuedPresentationTimeUs)) { - // This is the first stream, or the previous has been fully output already. + if (outputStreamInfo.streamOffsetUs == C.TIME_UNSET) { + // This is the first stream. setOutputStreamInfo( new OutputStreamInfo( /* previousStreamLastBufferTimeUs= */ C.TIME_UNSET, startPositionUs, offsetUs)); + } else if (pendingOutputStreamChanges.isEmpty() + && (largestQueuedPresentationTimeUs == C.TIME_UNSET + || (lastProcessedOutputBufferTimeUs != C.TIME_UNSET + && lastProcessedOutputBufferTimeUs >= largestQueuedPresentationTimeUs))) { + // All previous streams have never queued any samples or have been fully output already. + setOutputStreamInfo( + new OutputStreamInfo( + /* previousStreamLastBufferTimeUs= */ C.TIME_UNSET, startPositionUs, offsetUs)); + if (outputStreamInfo.streamOffsetUs != C.TIME_UNSET) { + onProcessedStreamChange(); + } } else { pendingOutputStreamChanges.add( new OutputStreamInfo(largestQueuedPresentationTimeUs, startPositionUs, offsetUs)); @@ -1581,7 +1589,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { @CallSuper protected void onProcessedOutputBuffer(long presentationTimeUs) { lastProcessedOutputBufferTimeUs = presentationTimeUs; - if (!pendingOutputStreamChanges.isEmpty() + while (!pendingOutputStreamChanges.isEmpty() && presentationTimeUs >= pendingOutputStreamChanges.peek().previousStreamLastBufferTimeUs) { setOutputStreamInfo(pendingOutputStreamChanges.poll()); onProcessedStreamChange(); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index ac86669ee5..b27a3d3106 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -214,6 +214,115 @@ public class MediaCodecRendererTest { inOrder.verify(renderer).onProcessedOutputBuffer(600); } + @Test + public void + render_withReplaceStreamAfterInitialEmptySampleStream_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = createFakeSampleStream(format1 /* no samples */); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 0, /* offsetUs= */ 0); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + } + + @Test + public void + render_withReplaceStreamAfterIntermittentEmptySampleStream_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + Format format3 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(2000).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100); + FakeSampleStream fakeSampleStream2 = createFakeSampleStream(format2 /* no samples */); + FakeSampleStream fakeSampleStream3 = + createFakeSampleStream(format3, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 200, /* offsetUs= */ 200); + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format3}, fakeSampleStream3, /* startPositionUs= */ 200, /* offsetUs= */ 200); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(200); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(200); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format3), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + } + private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) { ImmutableList.Builder sampleListBuilder = ImmutableList.builder();