sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-25 14:47:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Propagate gapless audio delay & padding.

Browse source 
MediaCodec does not need to be re-created in the
event of gapless metadata.

PiperOrigin-RevId: 318439694
This commit is contained in:
samrobinson 2020-06-26 10:58:34 +01:00 committed by Christos Tsilopoulos
parent 5a72b9452b
commit 81b0b53a37
6 changed files with 209 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
RELEASENOTES.md
View file

@ -170,6 +170,8 @@
([#7404](https://github.com/google/ExoPlayer/issues/7404)).
* Adjust input timestamps in `MediaCodecRenderer` to account for the
Codec2 MP3 decoder having lower timestamps on the output side.
* Propagate gapless audio metadata without the need to recreate the audio
decoders.
* DASH:
* Enable support for embedded CEA-708.
* HLS:

44
library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
View file

@ -310,16 +310,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
@Override
protected @KeepCodecResult int canKeepCodec(
MediaCodec codec, MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) {
// TODO: We currently rely on recreating the codec when encoder delay or padding is non-zero.
// Re-creating the codec is necessary to guarantee that onOutputMediaFormatChanged is called,
// which is where encoder delay and padding are propagated to the sink. We should find a better
// way to propagate these values, and then allow the codec to be re-used in cases where this
// would otherwise be possible.
if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize
|| oldFormat.encoderDelay != 0
|| oldFormat.encoderPadding != 0
|| newFormat.encoderDelay != 0
|| newFormat.encoderPadding != 0) {
if (getCodecMaxInputSize(codecInfo, newFormat) > codecMaxInputSize) {
return KEEP_CODEC_RESULT_NO;
} else if (codecInfo.isSeamlessAdaptationSupported(
oldFormat, newFormat, /* isNewFormatComplete= */ true)) {
@ -388,9 +379,14 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
}
@Override
protected void onOutputMediaFormatChanged(MediaCodec codec, MediaFormat outputMediaFormat)
throws ExoPlaybackException {
protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
configureOutput(outputFormat);
}
@Override
protected void configureOutput(Format outputFormat) throws ExoPlaybackException {
@C.Encoding int encoding;
MediaFormat mediaFormat;
int channelCount;
int sampleRate;
if (passthroughFormat != null) {
@ -398,18 +394,19 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
channelCount = passthroughFormat.channelCount;
sampleRate = passthroughFormat.sampleRate;
} else {
if (outputMediaFormat.containsKey(VIVO_BITS_PER_SAMPLE_KEY)) {
encoding = Util.getPcmEncoding(outputMediaFormat.getInteger(VIVO_BITS_PER_SAMPLE_KEY));
mediaFormat = getCodec().getOutputFormat();
if (mediaFormat.containsKey(VIVO_BITS_PER_SAMPLE_KEY)) {
encoding = Util.getPcmEncoding(mediaFormat.getInteger(VIVO_BITS_PER_SAMPLE_KEY));
} else {
encoding = getPcmEncoding(inputFormat);
encoding = getPcmEncoding(outputFormat);
}
channelCount = outputMediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
sampleRate = outputMediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
channelCount = mediaFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
sampleRate = mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
}
@Nullable int[] channelMap = null;
if (codecNeedsDiscardChannelsWorkaround && channelCount == 6 && inputFormat.channelCount < 6) {
channelMap = new int[inputFormat.channelCount];
for (int i = 0; i < inputFormat.channelCount; i++) {
if (codecNeedsDiscardChannelsWorkaround && channelCount == 6 && outputFormat.channelCount < 6) {
channelMap = new int[outputFormat.channelCount];
for (int i = 0; i < outputFormat.channelCount; i++) {
channelMap[i] = i;
}
}
@ -420,11 +417,10 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
sampleRate,
/* specifiedBufferSize= */ 0,
channelMap,
inputFormat.encoderDelay,
inputFormat.encoderPadding);
outputFormat.encoderDelay,
outputFormat.encoderPadding);
} catch (AudioSink.ConfigurationException e) {
// TODO(internal: b/145658993) Use outputFormat instead.
throw createRendererException(e, inputFormat);
throw createRendererException(e, outputFormat);
}
}

31
library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java
View file

@ -405,6 +405,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
private boolean waitingForFirstSampleInFormat;
private boolean pendingOutputEndOfStream;
@MediaCodecOperationMode private int mediaCodecOperationMode;
@Nullable private ExoPlaybackException pendingPlaybackException;
protected DecoderCounters decoderCounters;
private long outputStreamOffsetUs;
private int pendingOutputStreamOffsetCount;
@ -622,13 +623,25 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
return false;
}
/**
* Sets an exception to be re-thrown by render.
*
* @param exception The exception.
*/
protected void setPendingPlaybackException(ExoPlaybackException exception) {
pendingPlaybackException = exception;
}
/**
* Polls the pending output format queue for a given buffer timestamp. If a format is present, it
* is removed and returned. Otherwise returns {@code null}. Subclasses should only call this
* method if they are taking over responsibility for output format propagation (e.g., when using
* video tunneling).
*
* @throws ExoPlaybackException Thrown if an error occurs as a result of the output format change.
*/
protected final void updateOutputFormatForTime(long presentationTimeUs) {
protected final void updateOutputFormatForTime(long presentationTimeUs)
throws ExoPlaybackException {
@Nullable Format format = formatQueue.pollFloor(presentationTimeUs);
if (format != null) {
outputFormat = format;
@ -784,6 +797,12 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
pendingOutputEndOfStream = false;
processEndOfStream();
}
if (pendingPlaybackException != null) {
ExoPlaybackException playbackException = pendingPlaybackException;
pendingPlaybackException = null;
throw playbackException;
}
try {
if (outputStreamEnded) {
renderToEndOfStream();
@ -908,6 +927,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
codecInfo = null;
codecFormat = null;
codecHasOutputMediaFormat = false;
pendingPlaybackException = null;
codecOperatingRate = CODEC_OPERATING_RATE_UNSET;
codecAdaptationWorkaroundMode = ADAPTATION_WORKAROUND_MODE_NEVER;
codecNeedsReconfigureWorkaround = false;
@ -1490,8 +1510,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* <p>The default implementation is a no-op.
*
* @param outputFormat The new output {@link Format}.
* @throws ExoPlaybackException Thrown if an error occurs handling the new output format.
*/
protected void onOutputFormatChanged(Format outputFormat) {
protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
// Do nothing.
}
@ -1501,8 +1522,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* <p>The default implementation is a no-op.
*
* @param outputFormat The format to configure the output with.
* @throws ExoPlaybackException Thrown if an error occurs configuring the output.
*/
protected void configureOutput(Format outputFormat) {
protected void configureOutput(Format outputFormat) throws ExoPlaybackException {
// Do nothing.
}
@ -1538,8 +1560,9 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
* <p>The default implementation is a no-op.
*
* @param buffer The buffer to be queued.
* @throws ExoPlaybackException Thrown if an error occurs handling the input buffer.
*/
protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
// Do nothing.
}

13
library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java
View file

@ -643,11 +643,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
/**
* Called immediately before an input buffer is queued into the codec.
*
* <p>In tunneling mode for pre Marshmallow, the buffer is treated as if immediately output.
*
* @param buffer The buffer to be queued.
* @throws ExoPlaybackException Thrown if an error occurs handling the input buffer.
*/
@CallSuper
@Override
protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
// In tunneling mode the device may do frame rate conversion, so in general we can't keep track
// of the number of buffers in the codec.
if (!tunneling) {
@ -891,7 +894,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
/** Called when a buffer was processed in tunneling mode. */
protected void onProcessedTunneledBuffer(long presentationTimeUs) {
protected void onProcessedTunneledBuffer(long presentationTimeUs) throws ExoPlaybackException {
updateOutputFormatForTime(presentationTimeUs);
maybeNotifyVideoSizeChanged();
decoderCounters.renderedOutputBufferCount++;
@ -1808,7 +1811,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
if (presentationTimeUs == TUNNELING_EOS_PRESENTATION_TIME_US) {
onProcessedTunneledEndOfStream();
} else {
onProcessedTunneledBuffer(presentationTimeUs);
try {
onProcessedTunneledBuffer(presentationTimeUs);
} catch (ExoPlaybackException e) {
setPendingPlaybackException(e);
}
}
}
}

161
library/core/src/test/java/com/google/android/exoplayer2/audio/MediaCodecAudioRendererTest.java
View file

@ -15,6 +15,7 @@
*/
package com.google.android.exoplayer2.audio;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
@ -25,13 +26,12 @@ import android.os.SystemClock;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.ExoPlaybackException;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.RendererCapabilities;
import com.google.android.exoplayer2.RendererConfiguration;
import com.google.android.exoplayer2.drm.DrmSessionManager;
import com.google.android.exoplayer2.mediacodec.MediaCodecInfo;
import com.google.android.exoplayer2.mediacodec.MediaCodecSelector;
import com.google.android.exoplayer2.mediacodec.MediaCodecUtil.DecoderQueryException;
import com.google.android.exoplayer2.testutil.FakeSampleStream;
import com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem;
import com.google.android.exoplayer2.util.MimeTypes;
@ -61,6 +61,7 @@ public class MediaCodecAudioRendererTest {
.build();
private MediaCodecAudioRenderer mediaCodecAudioRenderer;
private MediaCodecSelector mediaCodecSelector;
@Mock private AudioSink audioSink;
@ -72,7 +73,7 @@ public class MediaCodecAudioRendererTest {
when(audioSink.handleBuffer(any(), anyLong(), anyInt())).thenReturn(true);
MediaCodecSelector mediaCodecSelector =
mediaCodecSelector =
new MediaCodecSelector() {
@Override
public List<MediaCodecInfo> getDecoderInfos(
@ -98,17 +99,13 @@ public class MediaCodecAudioRendererTest {
/* enableDecoderFallback= */ false,
/* eventHandler= */ null,
/* eventListener= */ null,
audioSink) {
@Override
protected int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format)
throws DecoderQueryException {
return RendererCapabilities.create(FORMAT_HANDLED);
}
};
audioSink);
}
@Test
public void render_configuresAudioSink() throws Exception {
public void render_configuresAudioSink_afterFormatChange() throws Exception {
Format changedFormat = AUDIO_AAC.buildUpon().setSampleRate(48_000).setEncoderDelay(400).build();
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
/* format= */ AUDIO_AAC,
@ -117,11 +114,17 @@ public class MediaCodecAudioRendererTest {
/* firstSampleTimeUs= */ 0,
/* timeUsIncrement= */ 50,
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(changedFormat),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
FakeSampleStreamItem.END_OF_STREAM_ITEM);
mediaCodecAudioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC},
new Format[] {AUDIO_AAC, changedFormat},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
@ -148,5 +151,139 @@ public class MediaCodecAudioRendererTest {
/* outputChannels= */ null,
AUDIO_AAC.encoderDelay,
AUDIO_AAC.encoderPadding);
verify(audioSink)
.configure(
changedFormat.pcmEncoding,
changedFormat.channelCount,
changedFormat.sampleRate,
/* specifiedBufferSize= */ 0,
/* outputChannels= */ null,
changedFormat.encoderDelay,
changedFormat.encoderPadding);
}
@Test
public void render_configuresAudioSink_afterGaplessFormatChange() throws Exception {
Format changedFormat =
AUDIO_AAC.buildUpon().setEncoderDelay(400).setEncoderPadding(232).build();
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
/* format= */ AUDIO_AAC,
DrmSessionManager.DUMMY,
/* eventDispatcher= */ null,
/* firstSampleTimeUs= */ 0,
/* timeUsIncrement= */ 50,
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(changedFormat),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
FakeSampleStreamItem.END_OF_STREAM_ITEM);
mediaCodecAudioRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC, changedFormat},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ false,
/* offsetUs */ 0);
mediaCodecAudioRenderer.start();
mediaCodecAudioRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
mediaCodecAudioRenderer.render(/* positionUs= */ 250, SystemClock.elapsedRealtime() * 1000);
mediaCodecAudioRenderer.setCurrentStreamFinal();
int positionUs = 500;
do {
mediaCodecAudioRenderer.render(positionUs, SystemClock.elapsedRealtime() * 1000);
positionUs += 250;
} while (!mediaCodecAudioRenderer.isEnded());
verify(audioSink)
.configure(
AUDIO_AAC.pcmEncoding,
AUDIO_AAC.channelCount,
AUDIO_AAC.sampleRate,
/* specifiedBufferSize= */ 0,
/* outputChannels= */ null,
AUDIO_AAC.encoderDelay,
AUDIO_AAC.encoderPadding);
verify(audioSink)
.configure(
changedFormat.pcmEncoding,
changedFormat.channelCount,
changedFormat.sampleRate,
/* specifiedBufferSize= */ 0,
/* outputChannels= */ null,
changedFormat.encoderDelay,
changedFormat.encoderPadding);
}
@Test
public void render_throwsExoPlaybackExceptionJustOnce_whenSet() throws Exception {
MediaCodecAudioRenderer exceptionThrowingRenderer =
new MediaCodecAudioRenderer(
ApplicationProvider.getApplicationContext(),
mediaCodecSelector,
/* eventHandler= */ null,
/* eventListener= */ null) {
@Override
protected void onOutputFormatChanged(Format outputFormat) throws ExoPlaybackException {
super.onOutputFormatChanged(outputFormat);
if (!outputFormat.equals(AUDIO_AAC)) {
setPendingPlaybackException(
ExoPlaybackException.createForRenderer(
new AudioSink.ConfigurationException("Test"),
"rendererName",
/* rendererIndex= */ 0,
outputFormat,
FORMAT_HANDLED));
}
}
};
Format changedFormat = AUDIO_AAC.buildUpon().setSampleRate(32_000).build();
FakeSampleStream fakeSampleStream =
new FakeSampleStream(
/* format= */ AUDIO_AAC,
DrmSessionManager.DUMMY,
/* eventDispatcher= */ null,
/* firstSampleTimeUs= */ 0,
/* timeUsIncrement= */ 50,
new FakeSampleStreamItem(new byte[] {0}, C.BUFFER_FLAG_KEY_FRAME),
FakeSampleStreamItem.END_OF_STREAM_ITEM);
exceptionThrowingRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC, changedFormat},
fakeSampleStream,
/* positionUs= */ 0,
/* joining= */ false,
/* mayRenderStartOfStream= */ false,
/* offsetUs */ 0);
exceptionThrowingRenderer.start();
exceptionThrowingRenderer.render(/* positionUs= */ 0, SystemClock.elapsedRealtime() * 1000);
exceptionThrowingRenderer.render(/* positionUs= */ 250, SystemClock.elapsedRealtime() * 1000);
// Simulating the exception being thrown when not traceable back to render.
exceptionThrowingRenderer.onOutputFormatChanged(changedFormat);
assertThrows(
ExoPlaybackException.class,
() ->
exceptionThrowingRenderer.render(
/* positionUs= */ 500, SystemClock.elapsedRealtime() * 1000));
// Doesn't throw an exception because it's cleared after being thrown in the previous call to
// render.
exceptionThrowingRenderer.render(/* positionUs= */ 750, SystemClock.elapsedRealtime() * 1000);
}
}

2
playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DebugRenderersFactory.java
View file

@ -142,7 +142,7 @@ import java.util.ArrayList;
}
@Override
protected void onQueueInputBuffer(DecoderInputBuffer buffer) {
protected void onQueueInputBuffer(DecoderInputBuffer buffer) throws ExoPlaybackException {
super.onQueueInputBuffer(buffer);
insertTimestamp(buffer.timeUs);
maybeShiftTimestampsList();