diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java index 15e71b840f..f925875a40 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/DecoderAudioRenderer.java @@ -165,6 +165,10 @@ public abstract class DecoderAudioRenderer< private final long[] pendingOutputStreamOffsetsUs; private int pendingOutputStreamOffsetCount; private boolean hasPendingReportedSkippedSilence; + private boolean isStarted; + private long largestQueuedPresentationTimeUs; + private long lastBufferInStreamPresentationTimeUs; + private long nextBufferToWritePresentationTimeUs; public DecoderAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -226,6 +230,9 @@ public abstract class DecoderAudioRenderer< audioTrackNeedsConfigure = true; setOutputStreamOffsetUs(C.TIME_UNSET); pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT]; + largestQueuedPresentationTimeUs = C.TIME_UNSET; + lastBufferInStreamPresentationTimeUs = C.TIME_UNSET; + nextBufferToWritePresentationTimeUs = C.TIME_UNSET; } @Override @@ -234,6 +241,23 @@ public abstract class DecoderAudioRenderer< return this; } + @Override + public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) { + if (nextBufferToWritePresentationTimeUs == C.TIME_UNSET) { + return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs); + } + long durationUs = + (long) + ((nextBufferToWritePresentationTimeUs - positionUs) + / (getPlaybackParameters() != null ? getPlaybackParameters().speed : 1.0f) + / 2); + if (isStarted) { + // Account for the elapsed time since the start of this iteration of the rendering loop. + durationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs; + } + return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs); + } + @Override public final @Capabilities int supportsFormat(Format format) { if (!MimeTypes.isAudio(format.sampleMimeType)) { @@ -279,6 +303,7 @@ public abstract class DecoderAudioRenderer< if (outputStreamEnded) { try { audioSink.playToEndOfStream(); + nextBufferToWritePresentationTimeUs = lastBufferInStreamPresentationTimeUs; } catch (AudioSink.WriteException e) { throw createRendererException( e, e.format, e.isRecoverable, PlaybackException.ERROR_CODE_AUDIO_TRACK_WRITE_FAILED); @@ -418,7 +443,6 @@ public abstract class DecoderAudioRenderer< processFirstSampleOfStream(); } } - if (outputBuffer.isEndOfStream()) { if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) { // We're waiting to re-initialize the decoder, and have now processed all final buffers. @@ -438,6 +462,7 @@ public abstract class DecoderAudioRenderer< } return false; } + nextBufferToWritePresentationTimeUs = C.TIME_UNSET; if (audioTrackNeedsConfigure) { Format outputFormat = @@ -464,6 +489,10 @@ public abstract class DecoderAudioRenderer< outputBuffer.release(); outputBuffer = null; return true; + } else { + // Downstream buffers are full, set nextBufferToWritePresentationTimeUs to the presentation + // time of the current 'to be written' sample. + nextBufferToWritePresentationTimeUs = outputBuffer.timeUs; } return false; @@ -516,6 +545,10 @@ public abstract class DecoderAudioRenderer< FormatHolder formatHolder = getFormatHolder(); switch (readSource(formatHolder, inputBuffer, /* readFlags= */ 0)) { case C.RESULT_NOTHING_READ: + if (hasReadStreamToEnd()) { + // Notify output queue of the last buffer's timestamp. + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; + } return false; case C.RESULT_FORMAT_READ: onInputFormatChanged(formatHolder); @@ -523,6 +556,7 @@ public abstract class DecoderAudioRenderer< case C.RESULT_BUFFER_READ: if (inputBuffer.isEndOfStream()) { inputStreamEnded = true; + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; decoder.queueInputBuffer(inputBuffer); inputBuffer = null; return false; @@ -531,6 +565,10 @@ public abstract class DecoderAudioRenderer< firstStreamSampleRead = true; inputBuffer.addFlag(C.BUFFER_FLAG_FIRST_SAMPLE); } + largestQueuedPresentationTimeUs = inputBuffer.timeUs; + if (hasReadStreamToEnd() || inputBuffer.isLastSample()) { + lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs; + } inputBuffer.flip(); inputBuffer.format = inputFormat; decoder.queueInputBuffer(inputBuffer); @@ -546,6 +584,7 @@ public abstract class DecoderAudioRenderer< private void processEndOfStream() throws AudioSink.WriteException { outputStreamEnded = true; audioSink.playToEndOfStream(); + nextBufferToWritePresentationTimeUs = lastBufferInStreamPresentationTimeUs; } private void flushDecoder() throws ExoPlaybackException { @@ -632,12 +671,14 @@ public abstract class DecoderAudioRenderer< @Override protected void onStarted() { audioSink.play(); + isStarted = true; } @Override protected void onStopped() { updateCurrentPosition(); audioSink.pause(); + isStarted = false; } @Override @@ -767,6 +808,8 @@ public abstract class DecoderAudioRenderer< outputBuffer = null; decoderReinitializationState = REINITIALIZATION_STATE_NONE; decoderReceivedBuffers = false; + largestQueuedPresentationTimeUs = C.TIME_UNSET; + lastBufferInStreamPresentationTimeUs = C.TIME_UNSET; if (decoder != null) { decoderCounters.decoderReleaseCount++; decoder.release(); diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java index 0360b1694a..aeba1c7b8a 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/audio/DecoderAudioRendererTest.java @@ -25,15 +25,18 @@ import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.longThat; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import android.os.SystemClock; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackParameters; import androidx.media3.common.util.Clock; import androidx.media3.decoder.CryptoConfig; import androidx.media3.decoder.DecoderException; @@ -46,9 +49,12 @@ import androidx.media3.exoplayer.drm.DrmSessionEventListener; import androidx.media3.exoplayer.drm.DrmSessionManager; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeClock; import androidx.media3.test.utils.FakeSampleStream; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.concurrent.CountDownLatch; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -70,28 +76,7 @@ public class DecoderAudioRendererTest { @Before public void setUp() throws Exception { MockitoAnnotations.initMocks(this); - audioRenderer = - new DecoderAudioRenderer(null, null, mockAudioSink) { - @Override - public String getName() { - return "TestAudioRenderer"; - } - - @Override - protected @C.FormatSupport int supportsFormatInternal(Format format) { - return FORMAT_HANDLED; - } - - @Override - protected FakeDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) { - return new FakeDecoder(); - } - - @Override - protected Format getOutputFormat(FakeDecoder decoder) { - return FORMAT; - } - }; + audioRenderer = createAudioRenderer(mockAudioSink); audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); } @@ -240,6 +225,254 @@ public class DecoderAudioRendererTest { inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt()); } + @Test + public void getDurationToProgressUs_withAudioSinkBuffersFull_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + CountDownLatch latchDecode = new CountDownLatch(4); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150000 us sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) + .thenReturn(false); + audioRenderer.start(); + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + } + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(75_000L); + } + + @Test + public void + getDurationToProgressUs_withAudioSinkBuffersFullAndDoublePlaybackSpeed_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.isEnded()).thenReturn(true); + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + PlaybackParameters playbackParametersWithDoubleSpeed = + new PlaybackParameters(/* speed= */ 2.0f); + when(mockAudioSink.getPlaybackParameters()).thenReturn(playbackParametersWithDoubleSpeed); + CountDownLatch latchDecode = new CountDownLatch(4); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + // Represents audio sink buffers being full when trying to write 150000 us sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) + .thenReturn(false); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + audioRenderer.start(); + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + } + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(37_500L); + } + + @Test + public void + getDurationToProgressUs_withAudioSinkBuffersFullAndPlaybackAdvancement_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.isEnded()).thenReturn(true); + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true); + CountDownLatch latchDecode = new CountDownLatch(4); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, fakeClock); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + // Represents audio sink buffers being full when trying to write 150000 us sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt())) + .thenReturn(false); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + audioRenderer.start(); + long rendererPositionElapsedRealtimeUs = SystemClock.elapsedRealtime() * 1000; + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + } + audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + + // Simulate playback progressing between render() and getDurationToProgressUs call + fakeClock.advanceTime(/* timeDiffMs= */ 10); + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, rendererPositionElapsedRealtimeUs); + + assertThat(durationToProgressUs).isEqualTo(65_000L); + } + + @Test + public void + getDurationToProgressUs_afterReadToEndOfStreamWithAudioSinkBuffersFull_returnsCalculatedDuration() + throws Exception { + when(mockAudioSink.isEnded()).thenReturn(true); + when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true); + when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT); + CountDownLatch latchDecode = new CountDownLatch(6); + ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink = + new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode); + audioRenderer = createAudioRenderer(countdownLatchAudioSink); + audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME), + oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME), + END_OF_STREAM_ITEM)); + // Mock that audio sink is full when trying to write final sample. + when(mockAudioSink.handleBuffer( + any(), longThat(presentationTimeUs -> presentationTimeUs == 250000), anyInt())) + .thenReturn(false); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + audioRenderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0, + new MediaSource.MediaPeriodId(new Object())); + // Represents audio sink buffers being full when trying to write 150000 us sample. + audioRenderer.start(); + while (latchDecode.getCount() != 0) { + audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + } + + long durationToProgressUs = + audioRenderer.getDurationToProgressUs( + /* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000); + + assertThat(durationToProgressUs).isEqualTo(125_000L); + } + + private static DecoderAudioRenderer createAudioRenderer(AudioSink audioSink) { + return new DecoderAudioRenderer(null, null, audioSink) { + @Override + public String getName() { + return "TestAudioRenderer"; + } + + @Override + protected @C.FormatSupport int supportsFormatInternal(Format format) { + return FORMAT_HANDLED; + } + + @Override + protected FakeDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) { + return new FakeDecoder(); + } + + @Override + protected Format getOutputFormat(FakeDecoder decoder) { + return FORMAT; + } + }; + } + private static final class FakeDecoder extends SimpleDecoder { @@ -276,4 +509,24 @@ public class DecoderAudioRendererTest { return null; } } + + private static final class ForwardingAudioSinkWithCountdownLatch extends ForwardingAudioSink { + + private final CountDownLatch latchDecode; + + public ForwardingAudioSinkWithCountdownLatch(AudioSink audioSink, CountDownLatch latchDecode) { + super(audioSink); + this.latchDecode = latchDecode; + } + + @Override + public boolean handleBuffer( + ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount) + throws InitializationException, WriteException { + if (latchDecode.getCount() > 0) { + latchDecode.countDown(); + } + return super.handleBuffer(buffer, presentationTimeUs, encodedAccessUnitCount); + } + } }