Implement getDurationToProgressUs for DecoderAudioRenderer

Add `DecoderAudioRenderer.getDurationToProgressUs()` so that `ExoPlayer`, if set with `experimentalSetDynamicSchedulingEnabled()`, will dynamically schedule its main work loop to when the `DecoderAudioRenderer` can make progress.

PiperOrigin-RevId: 689377247
This commit is contained in:
michaelkatz 2024-10-24 07:43:32 -07:00 committed by Copybara-Service
parent f181855c5e
commit 2f198c4c06
2 changed files with 319 additions and 23 deletions

View file

@ -165,6 +165,10 @@ public abstract class DecoderAudioRenderer<
private final long[] pendingOutputStreamOffsetsUs;
private int pendingOutputStreamOffsetCount;
private boolean hasPendingReportedSkippedSilence;
private boolean isStarted;
private long largestQueuedPresentationTimeUs;
private long lastBufferInStreamPresentationTimeUs;
private long nextBufferToWritePresentationTimeUs;
public DecoderAudioRenderer() {
this(/* eventHandler= */ null, /* eventListener= */ null);
@ -226,6 +230,9 @@ public abstract class DecoderAudioRenderer<
audioTrackNeedsConfigure = true;
setOutputStreamOffsetUs(C.TIME_UNSET);
pendingOutputStreamOffsetsUs = new long[MAX_PENDING_OUTPUT_STREAM_OFFSET_COUNT];
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
nextBufferToWritePresentationTimeUs = C.TIME_UNSET;
}
@Override
@ -234,6 +241,23 @@ public abstract class DecoderAudioRenderer<
return this;
}
@Override
public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) {
if (nextBufferToWritePresentationTimeUs == C.TIME_UNSET) {
return super.getDurationToProgressUs(positionUs, elapsedRealtimeUs);
}
long durationUs =
(long)
((nextBufferToWritePresentationTimeUs - positionUs)
/ (getPlaybackParameters() != null ? getPlaybackParameters().speed : 1.0f)
/ 2);
if (isStarted) {
// Account for the elapsed time since the start of this iteration of the rendering loop.
durationUs -= Util.msToUs(getClock().elapsedRealtime()) - elapsedRealtimeUs;
}
return max(DEFAULT_DURATION_TO_PROGRESS_US, durationUs);
}
@Override
public final @Capabilities int supportsFormat(Format format) {
if (!MimeTypes.isAudio(format.sampleMimeType)) {
@ -279,6 +303,7 @@ public abstract class DecoderAudioRenderer<
if (outputStreamEnded) {
try {
audioSink.playToEndOfStream();
nextBufferToWritePresentationTimeUs = lastBufferInStreamPresentationTimeUs;
} catch (AudioSink.WriteException e) {
throw createRendererException(
e, e.format, e.isRecoverable, PlaybackException.ERROR_CODE_AUDIO_TRACK_WRITE_FAILED);
@ -418,7 +443,6 @@ public abstract class DecoderAudioRenderer<
processFirstSampleOfStream();
}
}
if (outputBuffer.isEndOfStream()) {
if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) {
// We're waiting to re-initialize the decoder, and have now processed all final buffers.
@ -438,6 +462,7 @@ public abstract class DecoderAudioRenderer<
}
return false;
}
nextBufferToWritePresentationTimeUs = C.TIME_UNSET;
if (audioTrackNeedsConfigure) {
Format outputFormat =
@ -464,6 +489,10 @@ public abstract class DecoderAudioRenderer<
outputBuffer.release();
outputBuffer = null;
return true;
} else {
// Downstream buffers are full, set nextBufferToWritePresentationTimeUs to the presentation
// time of the current 'to be written' sample.
nextBufferToWritePresentationTimeUs = outputBuffer.timeUs;
}
return false;
@ -516,6 +545,10 @@ public abstract class DecoderAudioRenderer<
FormatHolder formatHolder = getFormatHolder();
switch (readSource(formatHolder, inputBuffer, /* readFlags= */ 0)) {
case C.RESULT_NOTHING_READ:
if (hasReadStreamToEnd()) {
// Notify output queue of the last buffer's timestamp.
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
}
return false;
case C.RESULT_FORMAT_READ:
onInputFormatChanged(formatHolder);
@ -523,6 +556,7 @@ public abstract class DecoderAudioRenderer<
case C.RESULT_BUFFER_READ:
if (inputBuffer.isEndOfStream()) {
inputStreamEnded = true;
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
decoder.queueInputBuffer(inputBuffer);
inputBuffer = null;
return false;
@ -531,6 +565,10 @@ public abstract class DecoderAudioRenderer<
firstStreamSampleRead = true;
inputBuffer.addFlag(C.BUFFER_FLAG_FIRST_SAMPLE);
}
largestQueuedPresentationTimeUs = inputBuffer.timeUs;
if (hasReadStreamToEnd() || inputBuffer.isLastSample()) {
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
}
inputBuffer.flip();
inputBuffer.format = inputFormat;
decoder.queueInputBuffer(inputBuffer);
@ -546,6 +584,7 @@ public abstract class DecoderAudioRenderer<
private void processEndOfStream() throws AudioSink.WriteException {
outputStreamEnded = true;
audioSink.playToEndOfStream();
nextBufferToWritePresentationTimeUs = lastBufferInStreamPresentationTimeUs;
}
private void flushDecoder() throws ExoPlaybackException {
@ -632,12 +671,14 @@ public abstract class DecoderAudioRenderer<
@Override
protected void onStarted() {
audioSink.play();
isStarted = true;
}
@Override
protected void onStopped() {
updateCurrentPosition();
audioSink.pause();
isStarted = false;
}
@Override
@ -767,6 +808,8 @@ public abstract class DecoderAudioRenderer<
outputBuffer = null;
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
decoderReceivedBuffers = false;
largestQueuedPresentationTimeUs = C.TIME_UNSET;
lastBufferInStreamPresentationTimeUs = C.TIME_UNSET;
if (decoder != null) {
decoderCounters.decoderReleaseCount++;
decoder.release();

View file

@ -25,15 +25,18 @@ import static com.google.common.truth.Truth.assertThat;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.longThat;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import android.os.SystemClock;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.PlaybackParameters;
import androidx.media3.common.util.Clock;
import androidx.media3.decoder.CryptoConfig;
import androidx.media3.decoder.DecoderException;
@ -46,9 +49,12 @@ import androidx.media3.exoplayer.drm.DrmSessionEventListener;
import androidx.media3.exoplayer.drm.DrmSessionManager;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.exoplayer.upstream.DefaultAllocator;
import androidx.media3.test.utils.FakeClock;
import androidx.media3.test.utils.FakeSampleStream;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.nio.ByteBuffer;
import java.util.concurrent.CountDownLatch;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -70,28 +76,7 @@ public class DecoderAudioRendererTest {
@Before
public void setUp() throws Exception {
MockitoAnnotations.initMocks(this);
audioRenderer =
new DecoderAudioRenderer<FakeDecoder>(null, null, mockAudioSink) {
@Override
public String getName() {
return "TestAudioRenderer";
}
@Override
protected @C.FormatSupport int supportsFormatInternal(Format format) {
return FORMAT_HANDLED;
}
@Override
protected FakeDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) {
return new FakeDecoder();
}
@Override
protected Format getOutputFormat(FakeDecoder decoder) {
return FORMAT;
}
};
audioRenderer = createAudioRenderer(mockAudioSink);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
}
@ -240,6 +225,254 @@ public class DecoderAudioRendererTest {
inOrderAudioSink.verify(mockAudioSink, times(2)).handleBuffer(any(), anyLong(), anyInt());
}
@Test
public void getDurationToProgressUs_withAudioSinkBuffersFull_returnsCalculatedDuration()
throws Exception {
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT);
CountDownLatch latchDecode = new CountDownLatch(4);
ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink =
new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode);
audioRenderer = createAudioRenderer(countdownLatchAudioSink);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ FORMAT,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME),
END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
audioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {FORMAT},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(new Object()));
// Represents audio sink buffers being full when trying to write 150000 us sample.
when(mockAudioSink.handleBuffer(
any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt()))
.thenReturn(false);
audioRenderer.start();
while (latchDecode.getCount() != 0) {
audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
}
audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
long durationToProgressUs =
audioRenderer.getDurationToProgressUs(
/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
assertThat(durationToProgressUs).isEqualTo(75_000L);
}
@Test
public void
getDurationToProgressUs_withAudioSinkBuffersFullAndDoublePlaybackSpeed_returnsCalculatedDuration()
throws Exception {
when(mockAudioSink.isEnded()).thenReturn(true);
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
PlaybackParameters playbackParametersWithDoubleSpeed =
new PlaybackParameters(/* speed= */ 2.0f);
when(mockAudioSink.getPlaybackParameters()).thenReturn(playbackParametersWithDoubleSpeed);
CountDownLatch latchDecode = new CountDownLatch(4);
ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink =
new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode);
audioRenderer = createAudioRenderer(countdownLatchAudioSink);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ FORMAT,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME),
END_OF_STREAM_ITEM));
// Represents audio sink buffers being full when trying to write 150000 us sample.
when(mockAudioSink.handleBuffer(
any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt()))
.thenReturn(false);
fakeSampleStream.writeData(/* startPositionUs= */ 0);
audioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {FORMAT},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(new Object()));
audioRenderer.start();
while (latchDecode.getCount() != 0) {
audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
}
audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
long durationToProgressUs =
audioRenderer.getDurationToProgressUs(
/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
assertThat(durationToProgressUs).isEqualTo(37_500L);
}
@Test
public void
getDurationToProgressUs_withAudioSinkBuffersFullAndPlaybackAdvancement_returnsCalculatedDuration()
throws Exception {
when(mockAudioSink.isEnded()).thenReturn(true);
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT);
FakeClock fakeClock = new FakeClock(/* initialTimeMs= */ 100, /* isAutoAdvancing= */ true);
CountDownLatch latchDecode = new CountDownLatch(4);
ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink =
new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode);
audioRenderer = createAudioRenderer(countdownLatchAudioSink);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, fakeClock);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ FORMAT,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME),
END_OF_STREAM_ITEM));
// Represents audio sink buffers being full when trying to write 150000 us sample.
when(mockAudioSink.handleBuffer(
any(), longThat(presentationTimeUs -> presentationTimeUs == 150000), anyInt()))
.thenReturn(false);
fakeSampleStream.writeData(/* startPositionUs= */ 0);
audioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {FORMAT},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(new Object()));
audioRenderer.start();
long rendererPositionElapsedRealtimeUs = SystemClock.elapsedRealtime() * 1000;
while (latchDecode.getCount() != 0) {
audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs);
}
audioRenderer.render(/* positionUs= */ 0, rendererPositionElapsedRealtimeUs);
// Simulate playback progressing between render() and getDurationToProgressUs call
fakeClock.advanceTime(/* timeDiffMs= */ 10);
long durationToProgressUs =
audioRenderer.getDurationToProgressUs(
/* positionUs= */ 0, rendererPositionElapsedRealtimeUs);
assertThat(durationToProgressUs).isEqualTo(65_000L);
}
@Test
public void
getDurationToProgressUs_afterReadToEndOfStreamWithAudioSinkBuffersFull_returnsCalculatedDuration()
throws Exception {
when(mockAudioSink.isEnded()).thenReturn(true);
when(mockAudioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
when(mockAudioSink.getPlaybackParameters()).thenReturn(PlaybackParameters.DEFAULT);
CountDownLatch latchDecode = new CountDownLatch(6);
ForwardingAudioSinkWithCountdownLatch countdownLatchAudioSink =
new ForwardingAudioSinkWithCountdownLatch(mockAudioSink, latchDecode);
audioRenderer = createAudioRenderer(countdownLatchAudioSink);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
/* mediaSourceEventDispatcher= */ null,
DrmSessionManager.DRM_UNSUPPORTED,
new DrmSessionEventListener.EventDispatcher(),
/* initialFormat= */ FORMAT,
ImmutableList.of(
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 50000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 100000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 150000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 200000, C.BUFFER_FLAG_KEY_FRAME),
oneByteSample(/* timeUs= */ 250000, C.BUFFER_FLAG_KEY_FRAME),
END_OF_STREAM_ITEM));
// Mock that audio sink is full when trying to write final sample.
when(mockAudioSink.handleBuffer(
any(), longThat(presentationTimeUs -> presentationTimeUs == 250000), anyInt()))
.thenReturn(false);
fakeSampleStream.writeData(/* startPositionUs= */ 0);
audioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {FORMAT},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ true,
/* startPositionUs= */ 0,
/* offsetUs= */ 0,
new MediaSource.MediaPeriodId(new Object()));
// Represents audio sink buffers being full when trying to write 150000 us sample.
audioRenderer.start();
while (latchDecode.getCount() != 0) {
audioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
}
long durationToProgressUs =
audioRenderer.getDurationToProgressUs(
/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
assertThat(durationToProgressUs).isEqualTo(125_000L);
}
private static DecoderAudioRenderer<FakeDecoder> createAudioRenderer(AudioSink audioSink) {
return new DecoderAudioRenderer<FakeDecoder>(null, null, audioSink) {
@Override
public String getName() {
return "TestAudioRenderer";
}
@Override
protected @C.FormatSupport int supportsFormatInternal(Format format) {
return FORMAT_HANDLED;
}
@Override
protected FakeDecoder createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) {
return new FakeDecoder();
}
@Override
protected Format getOutputFormat(FakeDecoder decoder) {
return FORMAT;
}
};
}
private static final class FakeDecoder
extends SimpleDecoder<DecoderInputBuffer, SimpleDecoderOutputBuffer, DecoderException> {
@ -276,4 +509,24 @@ public class DecoderAudioRendererTest {
return null;
}
}
private static final class ForwardingAudioSinkWithCountdownLatch extends ForwardingAudioSink {
private final CountDownLatch latchDecode;
public ForwardingAudioSinkWithCountdownLatch(AudioSink audioSink, CountDownLatch latchDecode) {
super(audioSink);
this.latchDecode = latchDecode;
}
@Override
public boolean handleBuffer(
ByteBuffer buffer, long presentationTimeUs, int encodedAccessUnitCount)
throws InitializationException, WriteException {
if (latchDecode.getCount() > 0) {
latchDecode.countDown();
}
return super.handleBuffer(buffer, presentationTimeUs, encodedAccessUnitCount);
}
}
}