sjs/media
1
0
Fork 0
mirror of https://github.com/samsonjs/media.git synced 2026-04-22 14:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Pass correct formats to AudioSink

Browse source 
The renderers are currently constructing formats that consist of their
input format with added PCM encoding. Such formats are not self-consistent,
and this only works because DefaultAudioSink ignores the rest of the
format if the format has a PCM encoding. It would not work if the sink
implementation checked the MIME type, for example, which wouldn't be a
strange or incorrect thing for it to do.

The more correct approach is to construct a new format that properly
represents the PCM that will be provided to the sink.

This change also renames supportsOutput to supportsFormat, because
AudioSink itself has both an input and an output side, and this method
is actually evaluating support on the input side of the sink.

PiperOrigin-RevId: 320396089
This commit is contained in:
olly 2020-07-09 15:56:16 +01:00 committed by kim-vde
parent 02f8cdf1d9
commit e8596428c0
10 changed files with 79 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java
View file

@ -28,6 +28,7 @@ import com.google.android.exoplayer2.drm.ExoMediaCrypto;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
/** Decodes and renders audio using FFmpeg. */
@ -143,13 +144,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
private boolean isOutputSupported(Format inputFormat) {
return shouldUseFloatOutput(inputFormat)
|| supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build());
|| sinkSupportsFormat(inputFormat, C.ENCODING_PCM_16BIT);
}
private boolean shouldUseFloatOutput(Format inputFormat) {
Assertions.checkNotNull(inputFormat.sampleMimeType);
if (!enableFloatOutput
|| !supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build())) {
if (!enableFloatOutput || !sinkSupportsFormat(inputFormat, C.ENCODING_PCM_FLOAT)) {
return false;
}
switch (inputFormat.sampleMimeType) {
@ -167,4 +167,12 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer {
}
}
/**
* Returns whether the renderer's {@link AudioSink} supports the PCM format that will be output
* from the decoder for the given input format and requested output encoding.
*/
private boolean sinkSupportsFormat(Format inputFormat, @C.PcmEncoding int pcmEncoding) {
return sinkSupportsFormat(
Util.getPcmFormat(pcmEncoding, inputFormat.channelCount, inputFormat.sampleRate));
}
}

26
extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/LibflacAudioRenderer.java
View file

@ -85,22 +85,23 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer {
|| !MimeTypes.AUDIO_FLAC.equalsIgnoreCase(format.sampleMimeType)) {
return FORMAT_UNSUPPORTED_TYPE;
}
// Compute the PCM encoding that the FLAC decoder will output.
@C.PcmEncoding int pcmEncoding;
// Compute the format that the FLAC decoder will output.
Format outputFormat;
if (format.initializationData.isEmpty()) {
// The initialization data might not be set if the format was obtained from a manifest (e.g.
// for DASH playbacks) rather than directly from the media. In this case we assume
// ENCODING_PCM_16BIT. If the actual encoding is different then playback will still succeed as
// long as the AudioSink supports it, which will always be true when using DefaultAudioSink.
pcmEncoding = C.ENCODING_PCM_16BIT;
outputFormat =
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate);
} else {
int streamMetadataOffset =
FlacConstants.STREAM_MARKER_SIZE + FlacConstants.METADATA_BLOCK_HEADER_SIZE;
FlacStreamMetadata streamMetadata =
new FlacStreamMetadata(format.initializationData.get(0), streamMetadataOffset);
pcmEncoding = Util.getPcmEncoding(streamMetadata.bitsPerSample);
outputFormat = getOutputFormat(streamMetadata);
}
if (!supportsOutput(format.buildUpon().setEncoding(pcmEncoding).build())) {
if (!sinkSupportsFormat(outputFormat)) {
return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (format.drmInitData != null && format.exoMediaCryptoType == null) {
return FORMAT_UNSUPPORTED_DRM;
@ -122,12 +123,13 @@ public final class LibflacAudioRenderer extends DecoderAudioRenderer {
@Override
protected Format getOutputFormat() {
Assertions.checkNotNull(streamMetadata);
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(streamMetadata.channels)
.setSampleRate(streamMetadata.sampleRate)
.setEncoding(Util.getPcmEncoding(streamMetadata.bitsPerSample))
.build();
return getOutputFormat(Assertions.checkNotNull(streamMetadata));
}
private static Format getOutputFormat(FlacStreamMetadata streamMetadata) {
return Util.getPcmFormat(
Util.getPcmEncoding(streamMetadata.bitsPerSample),
streamMetadata.channels,
streamMetadata.sampleRate);
}
}

11
extensions/opus/src/main/java/com/google/android/exoplayer2/ext/opus/LibopusAudioRenderer.java
View file

@ -25,6 +25,7 @@ import com.google.android.exoplayer2.audio.DecoderAudioRenderer;
import com.google.android.exoplayer2.drm.ExoMediaCrypto;
import com.google.android.exoplayer2.util.MimeTypes;
import com.google.android.exoplayer2.util.TraceUtil;
import com.google.android.exoplayer2.util.Util;
/** Decodes and renders audio using the native Opus decoder. */
public class LibopusAudioRenderer extends DecoderAudioRenderer {
@ -69,7 +70,8 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
if (!OpusLibrary.isAvailable()
|| !MimeTypes.AUDIO_OPUS.equalsIgnoreCase(format.sampleMimeType)) {
return FORMAT_UNSUPPORTED_TYPE;
} else if (!supportsOutput(format.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build())) {
} else if (!sinkSupportsFormat(
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate))) {
return FORMAT_UNSUPPORTED_SUBTYPE;
} else if (!drmIsSupported) {
return FORMAT_UNSUPPORTED_DRM;
@ -99,11 +101,6 @@ public class LibopusAudioRenderer extends DecoderAudioRenderer {
@Override
protected Format getOutputFormat() {
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(channelCount)
.setSampleRate(sampleRate)
.setEncoding(C.ENCODING_PCM_16BIT)
.build();
return Util.getPcmFormat(C.ENCODING_PCM_16BIT, channelCount, sampleRate);
}
}

24
library/common/src/main/java/com/google/android/exoplayer2/util/Util.java
View file

@ -1431,14 +1431,30 @@ public final class Util {
return split(codecs.trim(), "(\\s*,\\s*)");
}
/**
* Gets a PCM {@link Format} with the specified parameters.
*
* @param pcmEncoding The {@link C.PcmEncoding}.
* @param channels The number of channels, or {@link Format#NO_VALUE} if unknown.
* @param sampleRate The sample rate in Hz, or {@link Format#NO_VALUE} if unknown.
* @return The PCM format.
*/
public static Format getPcmFormat(@C.PcmEncoding int pcmEncoding, int channels, int sampleRate) {
return new Format.Builder()
.setSampleMimeType(MimeTypes.AUDIO_RAW)
.setChannelCount(channels)
.setSampleRate(sampleRate)
.setEncoding(pcmEncoding)
.build();
}
/**
* Converts a sample bit depth to a corresponding PCM encoding constant.
*
* @param bitDepth The bit depth. Supported values are 8, 16, 24 and 32.
* @return The corresponding encoding. One of {@link C#ENCODING_PCM_8BIT},
* {@link C#ENCODING_PCM_16BIT}, {@link C#ENCODING_PCM_24BIT} and
* {@link C#ENCODING_PCM_32BIT}. If the bit depth is unsupported then
* {@link C#ENCODING_INVALID} is returned.
* @return The corresponding encoding. One of {@link C#ENCODING_PCM_8BIT}, {@link
* C#ENCODING_PCM_16BIT}, {@link C#ENCODING_PCM_24BIT} and {@link C#ENCODING_PCM_32BIT}. If
* the bit depth is unsupported then {@link C#ENCODING_INVALID} is returned.
*/
@C.PcmEncoding
public static int getPcmEncoding(int bitDepth) {

8
library/core/src/main/java/com/google/android/exoplayer2/audio/AudioSink.java
View file

@ -184,12 +184,12 @@ public interface AudioSink {
void setListener(Listener listener);
/**
* Returns whether the sink supports the audio format.
* Returns whether the sink supports a given {@link Format}.
*
* @param format The format of the audio.
* @return Whether the sink supports the audio format.
* @param format The format.
* @return Whether the sink supports the format.
*/
boolean supportsOutput(Format format);
boolean supportsFormat(Format format);
/**
* Returns the playback position in the stream starting at zero, in microseconds, or

8
library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java
View file

@ -211,12 +211,12 @@ public abstract class DecoderAudioRenderer extends BaseRenderer implements Media
protected abstract int supportsFormatInternal(Format format);
/**
* Returns whether the sink supports the audio format.
* Returns whether the renderer's {@link AudioSink} supports a given {@link Format}.
*
* @see AudioSink#supportsOutput(Format)
* @see AudioSink#supportsFormat(Format)
*/
protected final boolean supportsOutput(Format format) {
return audioSink.supportsOutput(format);
protected final boolean sinkSupportsFormat(Format format) {
return audioSink.supportsFormat(format);
}
@Override

4
library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java
View file

@ -419,7 +419,7 @@ public final class DefaultAudioSink implements AudioSink {
}
@Override
public boolean supportsOutput(Format format) {
public boolean supportsFormat(Format format) {
if (format.encoding == C.ENCODING_INVALID) {
return false;
}
@ -473,7 +473,7 @@ public final class DefaultAudioSink implements AudioSink {
boolean useFloatOutput =
enableFloatOutput
&& Util.isEncodingHighResolutionPcm(inputFormat.encoding)
&& supportsOutput(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build());
&& supportsFormat(inputFormat.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build());
AudioProcessor[] availableAudioProcessors =
useFloatOutput ? toFloatPcmAvailableAudioProcessors : toIntPcmAvailableAudioProcessors;
if (processingEnabled) {

4
library/core/src/main/java/com/google/android/exoplayer2/audio/ForwardingAudioSink.java
View file

@ -35,8 +35,8 @@ public class ForwardingAudioSink implements AudioSink {
}
@Override
public boolean supportsOutput(Format format) {
return sink.supportsOutput(format);
public boolean supportsFormat(Format format) {
return sink.supportsFormat(format);
}
@Override

20
library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java
View file

@ -220,17 +220,21 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
int tunnelingSupport = Util.SDK_INT >= 21 ? TUNNELING_SUPPORTED : TUNNELING_NOT_SUPPORTED;
boolean formatHasDrm = format.drmInitData != null || format.exoMediaCryptoType != null;
boolean supportsFormatDrm = supportsFormatDrm(format);
// In passthrough mode, if a DRM is present we need to use a passthrough decoder to
// decrypt the content. For passthrough of clear content we don't need a decoder at all.
// In passthrough mode, if the format needs decryption then we need to use a passthrough
// decoder. Else we don't don't need a decoder at all.
if (supportsFormatDrm
&& usePassthrough(format)
&& (!formatHasDrm || MediaCodecUtil.getPassthroughDecoderInfo() != null)) {
return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_NOT_SEAMLESS, tunnelingSupport);
}
if ((MimeTypes.AUDIO_RAW.equals(format.sampleMimeType) && !audioSink.supportsOutput(format))
|| !audioSink.supportsOutput(
format.buildUpon().setEncoding(C.ENCODING_PCM_16BIT).build())) {
// Assume the decoder outputs 16-bit PCM, unless the input is raw.
// If the input is PCM then it will be passed directly to the sink. Hence the sink must support
// the input format directly.
if (MimeTypes.AUDIO_RAW.equals(format.sampleMimeType) && !audioSink.supportsFormat(format)) {
return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE);
}
// For all other input formats, we expect the decoder to output 16-bit PCM.
if (!audioSink.supportsFormat(
Util.getPcmFormat(C.ENCODING_PCM_16BIT, format.channelCount, format.sampleRate))) {
return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE);
}
List<MediaCodecInfo> decoderInfos =
@ -446,7 +450,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
.setChannelCount(Format.NO_VALUE)
.setEncoding(C.ENCODING_E_AC3_JOC)
.build();
if (audioSink.supportsOutput(eAc3JocFormat)) {
if (audioSink.supportsFormat(eAc3JocFormat)) {
return C.ENCODING_E_AC3_JOC;
}
// E-AC3 receivers can decode JOC streams, but in 2-D rather than 3-D, so try to fall back.
@ -455,7 +459,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
@C.Encoding int encoding = MimeTypes.getEncoding(mimeType, format.codecs);
Format passthroughFormat = format.buildUpon().setEncoding(encoding).build();
if (audioSink.supportsOutput(passthroughFormat)) {
if (audioSink.supportsFormat(passthroughFormat)) {
return encoding;
} else {
return C.ENCODING_INVALID;

12
library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java
View file

@ -206,25 +206,25 @@ public final class DefaultAudioSinkTest {
@Config(minSdk = OLDEST_SDK, maxSdk = 20)
@Test
public void doesNotSupportFloatOutputBeforeApi21() {
public void doesNotSupportFloatPcmBeforeApi21() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsOutput(floatFormat)).isFalse();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isFalse();
}
@Config(minSdk = 21, maxSdk = TARGET_SDK)
@Test
public void supportsFloatOutputFromApi21() {
public void supportsFloatPcmFromApi21() {
Format floatFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_PCM_FLOAT).build();
assertThat(defaultAudioSink.supportsOutput(floatFormat)).isTrue();
assertThat(defaultAudioSink.supportsFormat(floatFormat)).isTrue();
}
@Test
public void audioSinkWithAacAudioCapabilitiesWithoutOffload_doesNotSupportAacOutput() {
public void audioSinkWithAacAudioCapabilitiesWithoutOffload_doesNotSupportAac() {
DefaultAudioSink defaultAudioSink =
new DefaultAudioSink(
new AudioCapabilities(new int[] {C.ENCODING_AAC_LC}, 2), new AudioProcessor[0]);
Format aacLcFormat = STEREO_44_1_FORMAT.buildUpon().setEncoding(C.ENCODING_AAC_LC).build();
assertThat(defaultAudioSink.supportsOutput(aacLcFormat)).isFalse();
assertThat(defaultAudioSink.supportsFormat(aacLcFormat)).isFalse();
}
private void configureDefaultAudioSink(int channelCount) throws AudioSink.ConfigurationException {