From c007068ddb204958ee500d6db13cd97975c1893b Mon Sep 17 00:00:00 2001 From: olly Date: Fri, 8 Apr 2022 17:05:39 +0100 Subject: [PATCH] Expect PresentationTime Discontinuity During Stream Transitions PiperOrigin-RevId: 440378974 --- .../main/java/androidx/media3/common/C.java | 8 +- .../java/androidx/media3/decoder/Buffer.java | 5 + .../media3/decoder/SimpleDecoder.java | 3 + .../exoplayer/audio/DecoderAudioRenderer.java | 15 +++ .../exoplayer/audio/DefaultAudioSink.java | 1 - .../audio/DecoderAudioRendererTest.java | 100 ++++++++++++++++++ .../mka/bear-flac-16bit.mka.audiosink.dump | 1 + .../mka/bear-flac-24bit.mka.audiosink.dump | 1 + 8 files changed, 131 insertions(+), 3 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/C.java b/libraries/common/src/main/java/androidx/media3/common/C.java index 24d4801d5c..da6e26853b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/C.java +++ b/libraries/common/src/main/java/androidx/media3/common/C.java @@ -585,8 +585,9 @@ public final class C { /** * Flags which can apply to a buffer containing a media sample. Possible flag values are {@link - * #BUFFER_FLAG_KEY_FRAME}, {@link #BUFFER_FLAG_END_OF_STREAM}, {@link #BUFFER_FLAG_LAST_SAMPLE}, - * {@link #BUFFER_FLAG_ENCRYPTED} and {@link #BUFFER_FLAG_DECODE_ONLY}. + * #BUFFER_FLAG_KEY_FRAME}, {@link #BUFFER_FLAG_END_OF_STREAM}, {@link #BUFFER_FLAG_FIRST_SAMPLE}, + * {@link #BUFFER_FLAG_LAST_SAMPLE}, {@link #BUFFER_FLAG_ENCRYPTED} and {@link + * #BUFFER_FLAG_DECODE_ONLY}. */ @UnstableApi @Documented @@ -597,6 +598,7 @@ public final class C { value = { BUFFER_FLAG_KEY_FRAME, BUFFER_FLAG_END_OF_STREAM, + BUFFER_FLAG_FIRST_SAMPLE, BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA, BUFFER_FLAG_LAST_SAMPLE, BUFFER_FLAG_ENCRYPTED, @@ -608,6 +610,8 @@ public final class C { /** Flag for empty buffers that signal that the end of the stream was reached. */ @UnstableApi public static final int BUFFER_FLAG_END_OF_STREAM = MediaCodec.BUFFER_FLAG_END_OF_STREAM; + /** Indicates that a buffer is known to contain the first media sample of the stream. */ + @UnstableApi public static final int BUFFER_FLAG_FIRST_SAMPLE = 1 << 27; // 0x08000000 /** Indicates that a buffer has supplemental data. */ @UnstableApi public static final int BUFFER_FLAG_HAS_SUPPLEMENTAL_DATA = 1 << 28; // 0x10000000 /** Indicates that a buffer is known to contain the last media sample of the stream. */ diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java index abbdba9c9b..cf2be91c65 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/Buffer.java @@ -34,6 +34,11 @@ public abstract class Buffer { return getFlag(C.BUFFER_FLAG_DECODE_ONLY); } + /** Returns whether the {@link C#BUFFER_FLAG_FIRST_SAMPLE} flag is set. */ + public final boolean isFirstSample() { + return getFlag(C.BUFFER_FLAG_FIRST_SAMPLE); + } + /** Returns whether the {@link C#BUFFER_FLAG_END_OF_STREAM} flag is set. */ public final boolean isEndOfStream() { return getFlag(C.BUFFER_FLAG_END_OF_STREAM); diff --git a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java index a9711841d7..58d2943da0 100644 --- a/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java +++ b/libraries/decoder/src/main/java/androidx/media3/decoder/SimpleDecoder.java @@ -235,6 +235,9 @@ public abstract class SimpleDecoder< if (inputBuffer.isDecodeOnly()) { outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); } + if (inputBuffer.isFirstSample()) { + outputBuffer.addFlag(C.BUFFER_FLAG_FIRST_SAMPLE); + } @Nullable E exception; try { exception = decode(inputBuffer, outputBuffer, resetDecoder); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java index 87c3666605..b1ccd35438 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java @@ -130,6 +130,7 @@ public abstract class DecoderAudioRenderer< private int encoderPadding; private boolean experimentalKeepAudioTrackOnSeek; + private boolean firstStreamSampleRead; @Nullable private T decoder; @@ -389,6 +390,9 @@ public abstract class DecoderAudioRenderer< decoderCounters.skippedOutputBufferCount += outputBuffer.skippedOutputBufferCount; audioSink.handleDiscontinuity(); } + if (outputBuffer.isFirstSample()) { + audioSink.handleDiscontinuity(); + } } if (outputBuffer.isEndOfStream()) { @@ -470,6 +474,10 @@ public abstract class DecoderAudioRenderer< inputBuffer = null; return false; } + if (!firstStreamSampleRead) { + firstStreamSampleRead = true; + inputBuffer.addFlag(C.BUFFER_FLAG_FIRST_SAMPLE); + } inputBuffer.flip(); inputBuffer.format = inputFormat; onQueueInputBuffer(inputBuffer); @@ -587,6 +595,13 @@ public abstract class DecoderAudioRenderer< } } + @Override + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) + throws ExoPlaybackException { + super.onStreamChanged(formats, startPositionUs, offsetUs); + firstStreamSampleRead = false; + } + @Override public void handleMessage(@MessageType int messageType, @Nullable Object message) throws ExoPlaybackException { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java index 70b54350f8..f69f5e7c2a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DefaultAudioSink.java @@ -887,7 +887,6 @@ public final class DefaultAudioSink implements AudioSink { @Override public void handleDiscontinuity() { - // Force resynchronization after a skipped buffer. startMediaTimeUsNeedsSync = true; } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java index f0009b7ceb..36d3d7d603 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java @@ -21,7 +21,12 @@ import static androidx.media3.exoplayer.RendererCapabilities.DECODER_SUPPORT_PRI import static androidx.media3.exoplayer.RendererCapabilities.TUNNELING_NOT_SUPPORTED; import static androidx.media3.exoplayer.RendererCapabilities.TUNNELING_SUPPORTED; import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; import static com.google.common.truth.Truth.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -46,6 +51,7 @@ import com.google.common.collect.ImmutableList; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.robolectric.annotation.Config; @@ -139,6 +145,100 @@ public class DecoderAudioRendererTest { verify(mockAudioSink, times(1)).reset(); } + @Test + public void firstSampleOfStreamSignalsDiscontinuityToAudioSink() throws Exception { + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.isEnded()).thenReturn(true); + InOrder inOrderAudioSink = inOrder(mockAudioSink); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_000), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + + audioRenderer.setCurrentStreamFinal(); + while (!audioRenderer.isEnded()) { + audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); + inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); + } + + @Test + public void firstSampleOfReplacementStreamSignalsDiscontinuityToAudioSink() throws Exception { + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.isEnded()).thenReturn(true); + InOrder inOrderAudioSink = inOrder(mockAudioSink); + FakeSampleStream fakeSampleStream1 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_000), + END_OF_STREAM_ITEM)); + fakeSampleStream1.writeData(/* startPositionUs= */ 0); + FakeSampleStream fakeSampleStream2 = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 1_000_000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 1_001_000), + END_OF_STREAM_ITEM)); + fakeSampleStream2.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + + while (!audioRenderer.hasReadStreamToEnd()) { + audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + audioRenderer.replaceStream( + new Format[] {FORMAT}, + fakeSampleStream2, + /* startPositionUs= */ 1_000_000, + /* offsetUs= */ 1_000_000); + audioRenderer.setCurrentStreamFinal(); + while (!audioRenderer.isEnded()) { + audioRenderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); + inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); + inOrderAudioSink.verify(mockAudioSink, times(1)).handleDiscontinuity(); + inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); + } + private static final class FakeDecoder extends SimpleDecoder { diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump index b7319b872b..bc71f0a24b 100644 --- a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump +++ b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-16bit.mka.audiosink.dump @@ -1,3 +1,4 @@ +discontinuity: config: pcmEncoding = 2 channelCount = 2 diff --git a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump index f425e7e2f2..cc55bb6e9a 100644 --- a/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump +++ b/libraries/test_data/src/test/assets/audiosinkdumps/mka/bear-flac-24bit.mka.audiosink.dump @@ -1,3 +1,4 @@ +discontinuity: config: pcmEncoding = 536870912 channelCount = 2