From d2f807ebae7f7f8554e089a2a285b29940b0c0e9 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Tue, 25 Jan 2022 13:57:18 +0530 Subject: [PATCH 001/274] Add support for RTSP MPEG4 Added MPEG4 RTP packet reader and added support for MPEG4 playback through RTSP Change-Id: I57c9a61b18471dbd2c368177ebfb89ee662f995b --- .../common/util/CodecSpecificDataUtil.java | 86 +++++++++ .../exoplayer/rtsp/RtpPayloadFormat.java | 4 + .../media3/exoplayer/rtsp/RtspMediaTrack.java | 27 +++ .../DefaultRtpPayloadReaderFactory.java | 2 + .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 166 ++++++++++++++++++ 5 files changed, 285 insertions(+) create mode 100644 libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index b83620df38..a8a19aee8a 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -31,6 +31,13 @@ public final class CodecSpecificDataUtil { private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS = new String[] {"", "A", "B", "C"}; + // MP4V-ES + private static final int VISUAL_OBJECT_LAYER = 1; + private static final int VISUAL_OBJECT_LAYER_START = 0x20; + private static final int EXTENDED_PAR = 0x0F; + private static final int RECTANGULAR = 0x00; + private static final int FINE_GRANULARITY_SCALABLE = 0x12; + /** * Parses an ALAC AudioSpecificConfig (i.e. an ALACSpecificConfig). @@ -72,6 +79,85 @@ public final class CodecSpecificDataUtil { && initializationData.get(0)[0] == 1; } + /** + * Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2 + * + * @param videoSpecificConfig A byte array containing the MPEG-4 Visual configuration information + * to parse. + * @return A pair consisting of the width and the height. + */ + public static Pair parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) { + int offset = 0; + boolean foundVOL = false; + ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig); + while (offset + 3 < videoSpecificConfig.length) { + if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER + || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) { + scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2); + offset++; + continue; + } + foundVOL = true; + break; + } + + Assertions.checkArgument(foundVOL); + + ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); + scdScratchBits.skipBits((offset + 4) * 8); + scdScratchBits.skipBits(1); // random_accessible_vol + + int videoObjectTypeIndication = scdScratchBits.readBits(8); + Assertions.checkArgument(videoObjectTypeIndication != FINE_GRANULARITY_SCALABLE); + + if (scdScratchBits.readBit()) { // object_layer_identifier + scdScratchBits.skipBits(4); // video_object_layer_verid + scdScratchBits.skipBits(3); // video_object_layer_priority + } + + int aspectRatioInfo = scdScratchBits.readBits(4); + if (aspectRatioInfo == EXTENDED_PAR) { + scdScratchBits.skipBits(8); // par_width + scdScratchBits.skipBits(8); // par_height + } + + if (scdScratchBits.readBit()) { // vol_control_parameters + scdScratchBits.skipBits(2); // chroma_format + scdScratchBits.skipBits(1); // low_delay + if (scdScratchBits.readBit()) { // vbv_parameters + scdScratchBits.skipBits(79); + } + } + + int videoObjectLayerShape = scdScratchBits.readBits(2); + Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR); + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int vopTimeIncrementResolution = scdScratchBits.readBits(16); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + if (scdScratchBits.readBit()) { // fixed_vop_rate + Assertions.checkArgument(vopTimeIncrementResolution > 0); + --vopTimeIncrementResolution; + int numBits = 0; + while (vopTimeIncrementResolution > 0) { + ++numBits; + vopTimeIncrementResolution >>= 1; + } + scdScratchBits.skipBits(numBits); // fixed_vop_time_increment + } + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerWidth = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerHeight = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + scdScratchBits.skipBits(1); // interlaced + + return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight); + } + /** * Builds an RFC 6381 AVC codec string using the provided parameters. * diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 4c4521e682..f44c68ba0a 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -38,6 +38,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_AC3 = "AC3"; private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; + private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; private static final String RTP_MEDIA_H264 = "H264"; /** Returns whether the format of a {@link MediaDescription} is supported. */ @@ -45,6 +46,7 @@ public final class RtpPayloadFormat { switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) { case RTP_MEDIA_AC3: case RTP_MEDIA_H264: + case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_MPEG4_GENERIC: return true; default: @@ -65,6 +67,8 @@ public final class RtpPayloadFormat { return MimeTypes.AUDIO_AC3; case RTP_MEDIA_H264: return MimeTypes.VIDEO_H264; + case RTP_MEDIA_MPEG4_VIDEO: + return MimeTypes.VIDEO_MP4V; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 5b6b9a4607..02733aadcc 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -25,6 +25,7 @@ import static androidx.media3.extractor.NalUnitUtil.NAL_START_CODE; import android.net.Uri; import android.util.Base64; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -44,10 +45,14 @@ import com.google.common.collect.ImmutableMap; // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; + private static final String PARAMETER_CONFIG = "config"; + /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; /** Prefix for the RFC6381 codecs string for AVC formats. */ private static final String H264_CODECS_PREFIX = "avc1."; + /** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */ + private static final String MPEG4_CODECS_PREFIX = "mp4v"; private static final String GENERIC_CONTROL_ATTR = "*"; @@ -116,6 +121,10 @@ import com.google.common.collect.ImmutableMap; checkArgument(!fmtpParameters.isEmpty()); processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate); break; + case MimeTypes.VIDEO_MP4V: + checkArgument(!fmtpParameters.isEmpty()); + processMPEG4FmtpAttribute(formatBuilder, fmtpParameters); + break; case MimeTypes.VIDEO_H264: checkArgument(!fmtpParameters.isEmpty()); processH264FmtpAttribute(formatBuilder, fmtpParameters); @@ -160,6 +169,24 @@ import com.google.common.collect.ImmutableMap; AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + private static void processMPEG4FmtpAttribute( + Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { + @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG); + if (configInput != null) { + byte[] csd = Util.getBytesFromHexString(configInput); + ImmutableList initializationData = ImmutableList.of(csd); + formatBuilder.setInitializationData(initializationData); + Pair dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd); + formatBuilder.setWidth(dimensions.first); + formatBuilder.setHeight(dimensions.second); + } + @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); + if (profileLevel == null) { + profileLevel = "1"; // default + } + formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel); + } + private static void processH264FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS)); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 8fe084c131..0d58957a34 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -38,6 +38,8 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; return new RtpAacReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); + case MimeTypes.VIDEO_MP4V: + return new RtpMPEG4Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java new file mode 100644 index 0000000000..8d22cd82f3 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import com.google.common.primitives.Bytes; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * RFC6416 for more details. + */ +/* package */ final class RtpMPEG4Reader implements RtpPayloadReader { + private static final String TAG = "RtpMPEG4Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + /** + * VOP unit type. + */ + private static final int I_VOP = 0; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + + private int previousSequenceNumber; + + private long startTimeOffsetUs; + + private int sampleLength; + + File output = null; + + FileOutputStream outputStream = null; + + /** Creates an instance. */ + public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + sampleLength = 0; + try { + output = new File("/data/local/tmp/" + "mpeg4v_es.out"); + outputStream = new FileOutputStream(output); + } catch (IOException e) { + //do nothing; + } + } + + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket"); + } + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) { + Log.e(TAG, "Packet loss"); + } + checkStateNotNull(trackOutput); + + int limit = data.bytesLeft(); + trackOutput.sampleData(data, limit); + sampleLength += limit; + parseVopType(data); + + // Write the video sample + if (outputStream != null) { + try { + outputStream.write(data.getData()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP + // packet(or only RTP packet) of a VOP. When multiple VOPs are carried + // in the same RTP packet, the marker bit is set to 1. + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null); + sampleLength = 0; + } + + previousSequenceNumber = sequenceNumber; + } + + /** + * Parses VOP Coding type + * + * Sets {@link #bufferFlags} according to the VOP Coding type. + */ + private void parseVopType(ParsableByteArray data) { + // search for VOP_START_CODE (00 00 01 B6) + byte[] inputData = data.getData(); + byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6}; + int vopStartCodePos = Bytes.indexOf(inputData, startCode); + if (vopStartCodePos != -1) { + data.setPosition(vopStartCodePos + 4); + int vopType = data.peekUnsignedByte() >> 6; + bufferFlags = getBufferFlagsFromVopType(vopType); + } + } + + @C.BufferFlags + private static int getBufferFlagsFromVopType(int vopType) { + return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + sampleLength = 0; + } +} From 743437e34fe50f5ebd56b262c7e8080a2a98f128 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Mon, 31 Jan 2022 13:36:37 +0530 Subject: [PATCH 002/274] Clean up RtpMpeg4Reader --- .../common/util/CodecSpecificDataUtil.java | 21 ++++++++---------- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 22 ------------------- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index a8a19aee8a..b4b872a6da 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -36,7 +36,6 @@ public final class CodecSpecificDataUtil { private static final int VISUAL_OBJECT_LAYER_START = 0x20; private static final int EXTENDED_PAR = 0x0F; private static final int RECTANGULAR = 0x00; - private static final int FINE_GRANULARITY_SCALABLE = 0x12; /** * Parses an ALAC AudioSpecificConfig (i.e. an 0); + Assertions.checkArgument(vopTimeIncrementResolution > 0, "Invalid input"); --vopTimeIncrementResolution; int numBits = 0; while (vopTimeIncrementResolution > 0) { @@ -147,11 +144,11 @@ public final class CodecSpecificDataUtil { scdScratchBits.skipBits(numBits); // fixed_vop_time_increment } - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit int videoObjectLayerWidth = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit int videoObjectLayerHeight = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit scdScratchBits.skipBits(1); // interlaced diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index 8d22cd82f3..a34c1e14f7 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -27,9 +27,6 @@ import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; import com.google.common.primitives.Bytes; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** @@ -59,22 +56,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private int sampleLength; - File output = null; - - FileOutputStream outputStream = null; - /** Creates an instance. */ public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { this.payloadFormat = payloadFormat; firstReceivedTimestamp = C.TIME_UNSET; previousSequenceNumber = C.INDEX_UNSET; sampleLength = 0; - try { - output = new File("/data/local/tmp/" + "mpeg4v_es.out"); - outputStream = new FileOutputStream(output); - } catch (IOException e) { - //do nothing; - } } private static long toSampleUs( @@ -110,15 +97,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sampleLength += limit; parseVopType(data); - // Write the video sample - if (outputStream != null) { - try { - outputStream.write(data.getData()); - } catch (IOException e) { - e.printStackTrace(); - } - } - // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP // packet(or only RTP packet) of a VOP. When multiple VOPs are carried // in the same RTP packet, the marker bit is set to 1. From dfef2d13872d7950f54a805a1a18ad83e6c510dd Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Tue, 8 Feb 2022 17:02:59 +0530 Subject: [PATCH 003/274] Some minor cleanup in RTPMpeg4Reader --- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 4 +- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 41 +++++++++---------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 02733aadcc..d0cc763720 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -45,7 +45,7 @@ import com.google.common.collect.ImmutableMap; // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; - private static final String PARAMETER_CONFIG = "config"; + private static final String PARAMETER_MP4V_CONFIG = "config"; /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; @@ -171,7 +171,7 @@ import com.google.common.collect.ImmutableMap; private static void processMPEG4FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { - @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG); + @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); if (configInput != null) { byte[] csd = Util.getBytesFromHexString(configInput); ImmutableList initializationData = ImmutableList.of(csd); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index a34c1e14f7..a3dacd0f89 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -30,7 +30,7 @@ import com.google.common.primitives.Bytes; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** - * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * Parses an MPEG4 byte stream carried on RTP packets, and extracts MPEG4 Access Units. Refer to * RFC6416 for more details. */ /* package */ final class RtpMPEG4Reader implements RtpPayloadReader { @@ -44,16 +44,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final int I_VOP = 0; private final RtpPayloadFormat payloadFormat; - private @MonotonicNonNull TrackOutput trackOutput; @C.BufferFlags private int bufferFlags; - private long firstReceivedTimestamp; - private int previousSequenceNumber; - private long startTimeOffsetUs; - private int sampleLength; /** Creates an instance. */ @@ -64,15 +59,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; sampleLength = 0; } - private static long toSampleUs( - long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { - return startTimeOffsetUs - + Util.scaleLargeTimestamp( - (rtpTimestamp - firstReceivedRtpTimestamp), - /* multiplier= */ C.MICROS_PER_SECOND, - /* divisor= */ MEDIA_CLOCK_FREQUENCY); - } - @Override public void createTracks(ExtractorOutput extractorOutput, int trackId) { trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); @@ -113,6 +99,15 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; previousSequenceNumber = sequenceNumber; } + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + sampleLength = 0; + } + + // Internal methods. + /** * Parses VOP Coding type * @@ -130,15 +125,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + @C.BufferFlags private static int getBufferFlagsFromVopType(int vopType) { return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; } - - @Override - public void seek(long nextRtpTimestamp, long timeUs) { - firstReceivedTimestamp = nextRtpTimestamp; - startTimeOffsetUs = timeUs; - sampleLength = 0; - } } From e7567d2072dd3716bce7c7ff6b77b2d6184fd035 Mon Sep 17 00:00:00 2001 From: Manisha Jajoo Date: Wed, 9 Feb 2022 21:35:11 +0530 Subject: [PATCH 004/274] Fix review comments in RtpMPEG4Reader --- .../common/util/CodecSpecificDataUtil.java | 79 ++++++++++--------- .../media3/exoplayer/rtsp/RtspMediaTrack.java | 15 ++-- .../exoplayer/rtsp/reader/RtpMPEG4Reader.java | 45 ++++++----- 3 files changed, 73 insertions(+), 66 deletions(-) diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index b4b872a6da..821a7a2ba6 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -15,6 +15,8 @@ */ package androidx.media3.common.util; +import static androidx.media3.common.util.Assertions.checkArgument; + import android.util.Pair; import androidx.annotation.Nullable; import androidx.media3.common.C; @@ -85,14 +87,15 @@ public final class CodecSpecificDataUtil { * to parse. * @return A pair consisting of the width and the height. */ - public static Pair parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) { + public static Pair getVideoResolutionFromMpeg4VideoConfig( + byte[] videoSpecificConfig) { int offset = 0; boolean foundVOL = false; - ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig); + ParsableByteArray scratchBytes = new ParsableByteArray(videoSpecificConfig); while (offset + 3 < videoSpecificConfig.length) { - if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER + if (scratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) { - scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2); + scratchBytes.setPosition(scratchBytes.getPosition() - 2); offset++; continue; } @@ -100,57 +103,59 @@ public final class CodecSpecificDataUtil { break; } - Assertions.checkArgument(foundVOL, "Invalid input. VOL not found"); + checkArgument(foundVOL, "Invalid input: VOL not found."); - ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); - scdScratchBits.skipBits((offset + 4) * 8); - scdScratchBits.skipBits(1); // random_accessible_vol - scdScratchBits.skipBits(8); // video_object_type_indication + ParsableBitArray scratchBits = new ParsableBitArray(videoSpecificConfig); + // Skip the start codecs from the bitstream + scratchBits.skipBits((offset + 4) * 8); + scratchBits.skipBits(1); // random_accessible_vol + scratchBits.skipBits(8); // video_object_type_indication - if (scdScratchBits.readBit()) { // object_layer_identifier - scdScratchBits.skipBits(4); // video_object_layer_verid - scdScratchBits.skipBits(3); // video_object_layer_priority + if (scratchBits.readBit()) { // object_layer_identifier + scratchBits.skipBits(4); // video_object_layer_verid + scratchBits.skipBits(3); // video_object_layer_priority } - int aspectRatioInfo = scdScratchBits.readBits(4); + int aspectRatioInfo = scratchBits.readBits(4); if (aspectRatioInfo == EXTENDED_PAR) { - scdScratchBits.skipBits(8); // par_width - scdScratchBits.skipBits(8); // par_height + scratchBits.skipBits(8); // par_width + scratchBits.skipBits(8); // par_height } - if (scdScratchBits.readBit()) { // vol_control_parameters - scdScratchBits.skipBits(2); // chroma_format - scdScratchBits.skipBits(1); // low_delay - if (scdScratchBits.readBit()) { // vbv_parameters - scdScratchBits.skipBits(79); + if (scratchBits.readBit()) { // vol_control_parameters + scratchBits.skipBits(2); // chroma_format + scratchBits.skipBits(1); // low_delay + if (scratchBits.readBit()) { // vbv_parameters + scratchBits.skipBits(79); } } - int videoObjectLayerShape = scdScratchBits.readBits(2); - Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR, "Unsupported feature"); + int videoObjectLayerShape = scratchBits.readBits(2); + checkArgument( + videoObjectLayerShape == RECTANGULAR, "Only supports rectangular video object layer shape"); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int vopTimeIncrementResolution = scdScratchBits.readBits(16); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit + checkArgument(scratchBits.readBit()); // marker_bit + int vopTimeIncrementResolution = scratchBits.readBits(16); + checkArgument(scratchBits.readBit()); // marker_bit - if (scdScratchBits.readBit()) { // fixed_vop_rate - Assertions.checkArgument(vopTimeIncrementResolution > 0, "Invalid input"); - --vopTimeIncrementResolution; - int numBits = 0; + if (scratchBits.readBit()) { // fixed_vop_rate + checkArgument(vopTimeIncrementResolution > 0); + vopTimeIncrementResolution--; + int numBitsToSkip = 0; while (vopTimeIncrementResolution > 0) { - ++numBits; + numBitsToSkip++; vopTimeIncrementResolution >>= 1; } - scdScratchBits.skipBits(numBits); // fixed_vop_time_increment + scratchBits.skipBits(numBitsToSkip); // fixed_vop_time_increment } - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int videoObjectLayerWidth = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit - int videoObjectLayerHeight = scdScratchBits.readBits(13); - Assertions.checkArgument(scdScratchBits.readBit(), "Invalid input"); // marker_bit + checkArgument(scratchBits.readBit()); // marker_bit + int videoObjectLayerWidth = scratchBits.readBits(13); + checkArgument(scratchBits.readBit()); // marker_bit + int videoObjectLayerHeight = scratchBits.readBits(13); + checkArgument(scratchBits.readBit()); // marker_bit - scdScratchBits.skipBits(1); // interlaced + scratchBits.skipBits(1); // interlaced return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight); } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index d0cc763720..f8edb33311 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -174,17 +174,14 @@ import com.google.common.collect.ImmutableMap; @Nullable String configInput = fmtpAttributes.get(PARAMETER_MP4V_CONFIG); if (configInput != null) { byte[] csd = Util.getBytesFromHexString(configInput); - ImmutableList initializationData = ImmutableList.of(csd); - formatBuilder.setInitializationData(initializationData); - Pair dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd); - formatBuilder.setWidth(dimensions.first); - formatBuilder.setHeight(dimensions.second); + formatBuilder.setInitializationData(ImmutableList.of(csd)); + Pair resolution = + CodecSpecificDataUtil.getVideoResolutionFromMpeg4VideoConfig(csd); + formatBuilder.setWidth(resolution.first); + formatBuilder.setHeight(resolution.second); } @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); - if (profileLevel == null) { - profileLevel = "1"; // default - } - formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel); + formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + (profileLevel == null ? "1" : profileLevel)); } private static void processH264FmtpAttribute( diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java index a3dacd0f89..8154b9379b 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -23,6 +23,7 @@ import androidx.media3.common.ParserException; import androidx.media3.common.util.Log; import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; import androidx.media3.extractor.ExtractorOutput; import androidx.media3.extractor.TrackOutput; @@ -38,9 +39,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private static final long MEDIA_CLOCK_FREQUENCY = 90_000; - /** - * VOP unit type. - */ + /** VOP unit type. */ private static final int I_VOP = 0; private final RtpPayloadFormat payloadFormat; @@ -66,22 +65,31 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } @Override - public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { - Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket"); - } + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {} @Override public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) throws ParserException { - if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) { - Log.e(TAG, "Packet loss"); - } checkStateNotNull(trackOutput); + // Check that this packet is in the sequence of the previous packet. + if (previousSequenceNumber != C.INDEX_UNSET) { + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (sequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d." + + " Dropping packet.", + expectedSequenceNumber, sequenceNumber)); + return; + } + } + // Parse VOP Type and get the buffer flags int limit = data.bytesLeft(); trackOutput.sampleData(data, limit); + if (sampleLength == 0) bufferFlags = getBufferFlagsFromVop(data); sampleLength += limit; - parseVopType(data); // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP // packet(or only RTP packet) of a VOP. When multiple VOPs are carried @@ -95,7 +103,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null); sampleLength = 0; } - previousSequenceNumber = sequenceNumber; } @@ -109,20 +116,23 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; // Internal methods. /** - * Parses VOP Coding type + * Parses VOP Coding type. * * Sets {@link #bufferFlags} according to the VOP Coding type. */ - private void parseVopType(ParsableByteArray data) { + @C.BufferFlags + private static int getBufferFlagsFromVop(ParsableByteArray data) { + int flags = 0; // search for VOP_START_CODE (00 00 01 B6) byte[] inputData = data.getData(); - byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6}; + byte[] startCode = new byte[] {0x0, 0x0, 0x1, (byte) 0xB6}; int vopStartCodePos = Bytes.indexOf(inputData, startCode); if (vopStartCodePos != -1) { data.setPosition(vopStartCodePos + 4); int vopType = data.peekUnsignedByte() >> 6; - bufferFlags = getBufferFlagsFromVopType(vopType); + flags = vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; } + return flags; } private static long toSampleUs( @@ -133,9 +143,4 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* multiplier= */ C.MICROS_PER_SECOND, /* divisor= */ MEDIA_CLOCK_FREQUENCY); } - - @C.BufferFlags - private static int getBufferFlagsFromVopType(int vopType) { - return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; - } } From da947c0c743f5e08d3c2e0a889c0a55a0cee08d9 Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Tue, 8 Feb 2022 17:28:07 +0000 Subject: [PATCH 005/274] Remove FfmpegVideoRenderer from the 2.17.0 release branch This class is not ready to be released, and only exists in the dev-v2 branch. --- .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 6 +- .../ext/ffmpeg/FfmpegVideoRenderer.java | 135 ------------------ .../ffmpeg/DefaultRenderersFactoryTest.java | 9 +- library/core/proguard-rules.txt | 4 - .../exoplayer2/DefaultRenderersFactory.java | 26 ---- 5 files changed, 2 insertions(+), 178 deletions(-) delete mode 100644 extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java index 97f11225bb..abd184b59e 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -42,7 +42,7 @@ public final class FfmpegLibrary { /** * Override the names of the FFmpeg native libraries. If an application wishes to call this * method, it must do so before calling any other method defined by this class, and before - * instantiating a {@link FfmpegAudioRenderer} or {@link FfmpegVideoRenderer} instance. + * instantiating a {@link FfmpegAudioRenderer} instance. * * @param libraries The names of the FFmpeg native libraries. */ @@ -140,10 +140,6 @@ public final class FfmpegLibrary { return "pcm_mulaw"; case MimeTypes.AUDIO_ALAW: return "pcm_alaw"; - case MimeTypes.VIDEO_H264: - return "h264"; - case MimeTypes.VIDEO_H265: - return "hevc"; default: return null; } diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java deleted file mode 100644 index e074b87a21..0000000000 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (C) 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.DISCARD_REASON_MIME_TYPE_CHANGED; -import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_NO; -import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION; - -import android.os.Handler; -import android.view.Surface; -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.RendererCapabilities; -import com.google.android.exoplayer2.decoder.CryptoConfig; -import com.google.android.exoplayer2.decoder.Decoder; -import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.decoder.DecoderReuseEvaluation; -import com.google.android.exoplayer2.decoder.VideoDecoderOutputBuffer; -import com.google.android.exoplayer2.util.TraceUtil; -import com.google.android.exoplayer2.util.Util; -import com.google.android.exoplayer2.video.DecoderVideoRenderer; -import com.google.android.exoplayer2.video.VideoRendererEventListener; - -// TODO: Remove the NOTE below. -/** - * NOTE: This class if under development and is not yet functional. - * - *

Decodes and renders video using FFmpeg. - */ -public final class FfmpegVideoRenderer extends DecoderVideoRenderer { - - private static final String TAG = "FfmpegVideoRenderer"; - - /** - * Creates a new instance. - * - * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer - * can attempt to seamlessly join an ongoing playback. - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between - * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. - */ - public FfmpegVideoRenderer( - long allowedJoiningTimeMs, - @Nullable Handler eventHandler, - @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { - super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); - // TODO: Implement. - } - - @Override - public String getName() { - return TAG; - } - - @Override - @RendererCapabilities.Capabilities - public final int supportsFormat(Format format) { - // TODO: Remove this line and uncomment the implementation below. - return C.FORMAT_UNSUPPORTED_TYPE; - /* - String mimeType = Assertions.checkNotNull(format.sampleMimeType); - if (!FfmpegLibrary.isAvailable() || !MimeTypes.isVideo(mimeType)) { - return FORMAT_UNSUPPORTED_TYPE; - } else if (!FfmpegLibrary.supportsFormat(format.sampleMimeType)) { - return RendererCapabilities.create(FORMAT_UNSUPPORTED_SUBTYPE); - } else if (format.exoMediaCryptoType != null) { - return RendererCapabilities.create(FORMAT_UNSUPPORTED_DRM); - } else { - return RendererCapabilities.create( - FORMAT_HANDLED, - ADAPTIVE_SEAMLESS, - TUNNELING_NOT_SUPPORTED); - } - */ - } - - @SuppressWarnings("nullness:return") - @Override - protected Decoder - createDecoder(Format format, @Nullable CryptoConfig cryptoConfig) - throws FfmpegDecoderException { - TraceUtil.beginSection("createFfmpegVideoDecoder"); - // TODO: Implement, remove the SuppressWarnings annotation, and update the return type to use - // the concrete type of the decoder (probably FfmepgVideoDecoder). - TraceUtil.endSection(); - return null; - } - - @Override - protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) - throws FfmpegDecoderException { - // TODO: Implement. - } - - @Override - protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) { - // TODO: Uncomment the implementation below. - /* - if (decoder != null) { - decoder.setOutputMode(outputMode); - } - */ - } - - @Override - protected DecoderReuseEvaluation canReuseDecoder( - String decoderName, Format oldFormat, Format newFormat) { - boolean sameMimeType = Util.areEqual(oldFormat.sampleMimeType, newFormat.sampleMimeType); - // TODO: Ability to reuse the decoder may be MIME type dependent. - return new DecoderReuseEvaluation( - decoderName, - oldFormat, - newFormat, - sameMimeType ? REUSE_RESULT_YES_WITHOUT_RECONFIGURATION : REUSE_RESULT_NO, - sameMimeType ? 0 : DISCARD_REASON_MIME_TYPE_CHANGED); - } -} diff --git a/extensions/ffmpeg/src/test/java/com/google/android/exoplayer2/ext/ffmpeg/DefaultRenderersFactoryTest.java b/extensions/ffmpeg/src/test/java/com/google/android/exoplayer2/ext/ffmpeg/DefaultRenderersFactoryTest.java index cc8ca5487e..fa4c6809aa 100644 --- a/extensions/ffmpeg/src/test/java/com/google/android/exoplayer2/ext/ffmpeg/DefaultRenderersFactoryTest.java +++ b/extensions/ffmpeg/src/test/java/com/google/android/exoplayer2/ext/ffmpeg/DefaultRenderersFactoryTest.java @@ -22,8 +22,7 @@ import org.junit.Test; import org.junit.runner.RunWith; /** - * Unit test for {@link DefaultRenderersFactoryTest} with {@link FfmpegAudioRenderer} and {@link - * FfmpegVideoRenderer}. + * Unit test for {@link DefaultRenderersFactoryTest} with {@link FfmpegAudioRenderer}. */ @RunWith(AndroidJUnit4.class) public final class DefaultRenderersFactoryTest { @@ -33,10 +32,4 @@ public final class DefaultRenderersFactoryTest { DefaultRenderersFactoryAsserts.assertExtensionRendererCreated( FfmpegAudioRenderer.class, C.TRACK_TYPE_AUDIO); } - - @Test - public void createRenderers_instantiatesFfmpegVideoRenderer() { - DefaultRenderersFactoryAsserts.assertExtensionRendererCreated( - FfmpegVideoRenderer.class, C.TRACK_TYPE_VIDEO); - } } diff --git a/library/core/proguard-rules.txt b/library/core/proguard-rules.txt index fc6787e09d..ebe0c271ef 100644 --- a/library/core/proguard-rules.txt +++ b/library/core/proguard-rules.txt @@ -9,10 +9,6 @@ -keepclassmembers class com.google.android.exoplayer2.ext.av1.Libgav1VideoRenderer { (long, android.os.Handler, com.google.android.exoplayer2.video.VideoRendererEventListener, int); } --dontnote com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer --keepclassmembers class com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer { - (long, android.os.Handler, com.google.android.exoplayer2.video.VideoRendererEventListener, int); -} -dontnote com.google.android.exoplayer2.ext.opus.LibopusAudioRenderer -keepclassmembers class com.google.android.exoplayer2.ext.opus.LibopusAudioRenderer { (android.os.Handler, com.google.android.exoplayer2.audio.AudioRendererEventListener, com.google.android.exoplayer2.audio.AudioSink); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java index 0d1c126dc5..1fcf65e40d 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java @@ -453,32 +453,6 @@ public class DefaultRenderersFactory implements RenderersFactory { // The extension is present, but instantiation failed. throw new RuntimeException("Error instantiating AV1 extension", e); } - - try { - // Full class names used for constructor args so the LINT rule triggers if any of them move. - Class clazz = - Class.forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer"); - Constructor constructor = - clazz.getConstructor( - long.class, - android.os.Handler.class, - com.google.android.exoplayer2.video.VideoRendererEventListener.class, - int.class); - Renderer renderer = - (Renderer) - constructor.newInstance( - allowedVideoJoiningTimeMs, - eventHandler, - eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); - out.add(extensionRendererIndex++, renderer); - Log.i(TAG, "Loaded FfmpegVideoRenderer."); - } catch (ClassNotFoundException e) { - // Expected if the app was built without the extension. - } catch (Exception e) { - // The extension is present, but instantiation failed. - throw new RuntimeException("Error instantiating FFmpeg extension", e); - } } /** From e350f9ce1f68d0de9b022da45c257d47331b8931 Mon Sep 17 00:00:00 2001 From: ibaker Date: Mon, 31 Jan 2022 17:08:15 +0000 Subject: [PATCH 006/274] Publish the ImaServerSideAdInsertionMediaSource Issue: google/ExoPlayer#8213 #minor-release PiperOrigin-RevId: 425381474 --- RELEASENOTES.md | 3 + .../ImaServerSideAdInsertionMediaSource.java | 1126 +++++++++++++++++ .../ServerSideAdInsertionStreamRequest.java | 476 +++++++ ...erverSideAdInsertionStreamRequestTest.java | 148 +++ 4 files changed, 1753 insertions(+) create mode 100644 extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaServerSideAdInsertionMediaSource.java create mode 100644 extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequest.java create mode 100644 extensions/ima/src/test/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequestTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index c6f478c67c..0b4fb8b296 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -83,6 +83,9 @@ ([#9615](https://github.com/google/ExoPlayer/issues/9615)). * Enforce playback speed of 1.0 during ad playback ([#9018](https://github.com/google/ExoPlayer/issues/9018)). + * Add support for + [IMA Dynamic Ad Insertion (DAI)](https://support.google.com/admanager/answer/6147120) + ([#8213](https://github.com/google/ExoPlayer/issues/8213)). * DASH: * Support the `forced-subtitle` track role ([#9727](https://github.com/google/ExoPlayer/issues/9727)). diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaServerSideAdInsertionMediaSource.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaServerSideAdInsertionMediaSource.java new file mode 100644 index 0000000000..0c2aa28003 --- /dev/null +++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaServerSideAdInsertionMediaSource.java @@ -0,0 +1,1126 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ima; + +import static com.google.android.exoplayer2.ext.ima.ImaUtil.expandAdGroupPlaceholder; +import static com.google.android.exoplayer2.ext.ima.ImaUtil.splitAdPlaybackStateForPeriods; +import static com.google.android.exoplayer2.ext.ima.ImaUtil.updateAdDurationAndPropagate; +import static com.google.android.exoplayer2.ext.ima.ImaUtil.updateAdDurationInAdGroup; +import static com.google.android.exoplayer2.source.ads.ServerSideAdInsertionUtil.addAdGroupToAdPlaybackState; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Assertions.checkState; +import static com.google.android.exoplayer2.util.Util.secToUs; +import static com.google.android.exoplayer2.util.Util.sum; +import static com.google.android.exoplayer2.util.Util.usToMs; +import static java.lang.Math.min; + +import android.content.Context; +import android.net.Uri; +import android.os.Handler; +import android.view.ViewGroup; +import androidx.annotation.MainThread; +import androidx.annotation.Nullable; +import com.google.ads.interactivemedia.v3.api.Ad; +import com.google.ads.interactivemedia.v3.api.AdDisplayContainer; +import com.google.ads.interactivemedia.v3.api.AdErrorEvent; +import com.google.ads.interactivemedia.v3.api.AdErrorEvent.AdErrorListener; +import com.google.ads.interactivemedia.v3.api.AdEvent; +import com.google.ads.interactivemedia.v3.api.AdEvent.AdEventListener; +import com.google.ads.interactivemedia.v3.api.AdPodInfo; +import com.google.ads.interactivemedia.v3.api.AdsLoader.AdsLoadedListener; +import com.google.ads.interactivemedia.v3.api.AdsManager; +import com.google.ads.interactivemedia.v3.api.AdsManagerLoadedEvent; +import com.google.ads.interactivemedia.v3.api.AdsRenderingSettings; +import com.google.ads.interactivemedia.v3.api.CompanionAdSlot; +import com.google.ads.interactivemedia.v3.api.CuePoint; +import com.google.ads.interactivemedia.v3.api.ImaSdkFactory; +import com.google.ads.interactivemedia.v3.api.ImaSdkSettings; +import com.google.ads.interactivemedia.v3.api.StreamDisplayContainer; +import com.google.ads.interactivemedia.v3.api.StreamManager; +import com.google.ads.interactivemedia.v3.api.StreamRequest; +import com.google.ads.interactivemedia.v3.api.player.VideoProgressUpdate; +import com.google.ads.interactivemedia.v3.api.player.VideoStreamPlayer; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.Player; +import com.google.android.exoplayer2.Timeline; +import com.google.android.exoplayer2.drm.DrmSessionManagerProvider; +import com.google.android.exoplayer2.metadata.Metadata; +import com.google.android.exoplayer2.metadata.emsg.EventMessage; +import com.google.android.exoplayer2.metadata.id3.TextInformationFrame; +import com.google.android.exoplayer2.source.CompositeMediaSource; +import com.google.android.exoplayer2.source.DefaultMediaSourceFactory; +import com.google.android.exoplayer2.source.ForwardingTimeline; +import com.google.android.exoplayer2.source.MediaPeriod; +import com.google.android.exoplayer2.source.MediaSource; +import com.google.android.exoplayer2.source.ads.AdPlaybackState; +import com.google.android.exoplayer2.source.ads.ServerSideAdInsertionMediaSource; +import com.google.android.exoplayer2.source.ads.ServerSideAdInsertionMediaSource.AdPlaybackStateUpdater; +import com.google.android.exoplayer2.source.ads.ServerSideAdInsertionUtil; +import com.google.android.exoplayer2.ui.AdOverlayInfo; +import com.google.android.exoplayer2.ui.AdViewProvider; +import com.google.android.exoplayer2.upstream.Allocator; +import com.google.android.exoplayer2.upstream.LoadErrorHandlingPolicy; +import com.google.android.exoplayer2.upstream.Loader; +import com.google.android.exoplayer2.upstream.Loader.LoadErrorAction; +import com.google.android.exoplayer2.upstream.Loader.Loadable; +import com.google.android.exoplayer2.upstream.TransferListener; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.ConditionVariable; +import com.google.android.exoplayer2.util.Util; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * MediaSource for IMA server side inserted ad streams. + * + *

TODO(bachinger) add code snippet from PlayerActivity + */ +public final class ImaServerSideAdInsertionMediaSource extends CompositeMediaSource { + + /** + * Factory for creating {@link ImaServerSideAdInsertionMediaSource + * ImaServerSideAdInsertionMediaSources}. + * + *

Apps can use the {@link ImaServerSideAdInsertionMediaSource.Factory} to customized the + * {@link DefaultMediaSourceFactory} that is used to build a player: + * + *

TODO(bachinger) add code snippet from PlayerActivity + */ + public static final class Factory implements MediaSource.Factory { + + private final AdsLoader adsLoader; + private final MediaSource.Factory contentMediaSourceFactory; + + /** + * Creates a new factory for {@link ImaServerSideAdInsertionMediaSource + * ImaServerSideAdInsertionMediaSources}. + * + * @param adsLoader The {@link AdsLoader}. + * @param contentMediaSourceFactory The content media source factory to create content sources. + */ + public Factory(AdsLoader adsLoader, MediaSource.Factory contentMediaSourceFactory) { + this.adsLoader = adsLoader; + this.contentMediaSourceFactory = contentMediaSourceFactory; + } + + @Override + public MediaSource.Factory setLoadErrorHandlingPolicy( + @Nullable LoadErrorHandlingPolicy loadErrorHandlingPolicy) { + contentMediaSourceFactory.setLoadErrorHandlingPolicy(loadErrorHandlingPolicy); + return this; + } + + @Override + public MediaSource.Factory setDrmSessionManagerProvider( + @Nullable DrmSessionManagerProvider drmSessionManagerProvider) { + contentMediaSourceFactory.setDrmSessionManagerProvider(drmSessionManagerProvider); + return this; + } + + @Override + public int[] getSupportedTypes() { + return contentMediaSourceFactory.getSupportedTypes(); + } + + @Override + public MediaSource createMediaSource(MediaItem mediaItem) { + Player player = checkNotNull(adsLoader.player); + StreamPlayer streamPlayer = new StreamPlayer(player, mediaItem); + ImaSdkFactory imaSdkFactory = ImaSdkFactory.getInstance(); + StreamDisplayContainer streamDisplayContainer = + createStreamDisplayContainer(imaSdkFactory, adsLoader.configuration, streamPlayer); + com.google.ads.interactivemedia.v3.api.AdsLoader imaAdsLoader = + imaSdkFactory.createAdsLoader( + adsLoader.context, adsLoader.configuration.imaSdkSettings, streamDisplayContainer); + ImaServerSideAdInsertionMediaSource mediaSource = + new ImaServerSideAdInsertionMediaSource( + mediaItem, + player, + imaAdsLoader, + streamPlayer, + contentMediaSourceFactory, + adsLoader.configuration.applicationAdEventListener, + adsLoader.configuration.applicationAdErrorListener); + adsLoader.addMediaSourceResources(mediaSource, streamPlayer, imaAdsLoader); + return mediaSource; + } + } + + /** An ads loader for IMA server side ad insertion streams. */ + public static final class AdsLoader { + + /** Builder for building an {@link AdsLoader}. */ + public static final class Builder { + + private final Context context; + private final AdViewProvider adViewProvider; + + @Nullable private ImaSdkSettings imaSdkSettings; + @Nullable private AdEventListener adEventListener; + @Nullable private AdErrorEvent.AdErrorListener adErrorListener; + private ImmutableList companionAdSlots; + + /** + * Creates an instance. + * + * @param context A context. + * @param adViewProvider A provider for {@link ViewGroup} instances. + */ + public Builder(Context context, AdViewProvider adViewProvider) { + this.context = context; + this.adViewProvider = adViewProvider; + companionAdSlots = ImmutableList.of(); + } + + /** + * Sets the IMA SDK settings. + * + *

If this method is not called the default settings will be used. + * + * @param imaSdkSettings The {@link ImaSdkSettings}. + * @return This builder, for convenience. + */ + public AdsLoader.Builder setImaSdkSettings(ImaSdkSettings imaSdkSettings) { + this.imaSdkSettings = imaSdkSettings; + return this; + } + + /** + * Sets the optional {@link AdEventListener} that will be passed to {@link + * AdsManager#addAdEventListener(AdEventListener)}. + * + * @param adEventListener The ad event listener. + * @return This builder, for convenience. + */ + public AdsLoader.Builder setAdEventListener(AdEventListener adEventListener) { + this.adEventListener = adEventListener; + return this; + } + + /** + * Sets the optional {@link AdErrorEvent.AdErrorListener} that will be passed to {@link + * AdsManager#addAdErrorListener(AdErrorEvent.AdErrorListener)}. + * + * @param adErrorListener The {@link AdErrorEvent.AdErrorListener}. + * @return This builder, for convenience. + */ + public AdsLoader.Builder setAdErrorListener(AdErrorEvent.AdErrorListener adErrorListener) { + this.adErrorListener = adErrorListener; + return this; + } + + /** + * Sets the slots to use for companion ads, if they are present in the loaded ad. + * + * @param companionAdSlots The slots to use for companion ads. + * @return This builder, for convenience. + * @see AdDisplayContainer#setCompanionSlots(Collection) + */ + public AdsLoader.Builder setCompanionAdSlots(Collection companionAdSlots) { + this.companionAdSlots = ImmutableList.copyOf(companionAdSlots); + return this; + } + + /** Returns a new {@link AdsLoader}. */ + public AdsLoader build() { + @Nullable ImaSdkSettings imaSdkSettings = this.imaSdkSettings; + if (imaSdkSettings == null) { + imaSdkSettings = ImaSdkFactory.getInstance().createImaSdkSettings(); + imaSdkSettings.setLanguage(Util.getSystemLanguageCodes()[0]); + } + ImaUtil.ServerSideAdInsertionConfiguration configuration = + new ImaUtil.ServerSideAdInsertionConfiguration( + adViewProvider, + imaSdkSettings, + adEventListener, + adErrorListener, + companionAdSlots, + imaSdkSettings.isDebugMode()); + return new AdsLoader(context, configuration); + } + } + + private final ImaUtil.ServerSideAdInsertionConfiguration configuration; + private final Context context; + private final Map + mediaSourceResources; + + @Nullable private Player player; + + private AdsLoader(Context context, ImaUtil.ServerSideAdInsertionConfiguration configuration) { + this.context = context.getApplicationContext(); + this.configuration = configuration; + mediaSourceResources = new HashMap<>(); + } + + /** + * Sets the player. + * + *

This method needs to be called before adding server side ad insertion media items to the + * player. + */ + public void setPlayer(Player player) { + this.player = player; + } + + public void addMediaSourceResources( + ImaServerSideAdInsertionMediaSource mediaSource, + StreamPlayer streamPlayer, + com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader) { + mediaSourceResources.put(mediaSource, new MediaSourceResourceHolder(streamPlayer, adsLoader)); + } + + /** Releases resources when the ads loader is no longer needed. */ + public void release() { + for (MediaSourceResourceHolder resourceHolder : mediaSourceResources.values()) { + resourceHolder.streamPlayer.release(); + resourceHolder.adsLoader.release(); + } + mediaSourceResources.clear(); + } + + private static final class MediaSourceResourceHolder { + public final StreamPlayer streamPlayer; + public final com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader; + + private MediaSourceResourceHolder( + StreamPlayer streamPlayer, com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader) { + this.streamPlayer = streamPlayer; + this.adsLoader = adsLoader; + } + } + } + + private final MediaItem mediaItem; + private final Player player; + private final MediaSource.Factory contentMediaSourceFactory; + private final com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader; + @Nullable private final AdEventListener applicationAdEventListener; + @Nullable private final AdErrorListener applicationAdErrorListener; + private final ServerSideAdInsertionStreamRequest streamRequest; + private final StreamPlayer streamPlayer; + private final Handler mainHandler; + private final ComponentListener componentListener; + + @Nullable private Loader loader; + @Nullable private StreamManager streamManager; + @Nullable private ServerSideAdInsertionMediaSource serverSideAdInsertionMediaSource; + @Nullable private IOException loadError; + private @MonotonicNonNull Timeline contentTimeline; + private AdPlaybackState adPlaybackState; + + private ImaServerSideAdInsertionMediaSource( + MediaItem mediaItem, + Player player, + com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader, + StreamPlayer streamPlayer, + MediaSource.Factory contentMediaSourceFactory, + @Nullable AdEventListener applicationAdEventListener, + @Nullable AdErrorEvent.AdErrorListener applicationAdErrorListener) { + this.mediaItem = mediaItem; + this.player = player; + this.adsLoader = adsLoader; + this.streamPlayer = streamPlayer; + this.contentMediaSourceFactory = contentMediaSourceFactory; + this.applicationAdEventListener = applicationAdEventListener; + this.applicationAdErrorListener = applicationAdErrorListener; + componentListener = new ComponentListener(); + adPlaybackState = AdPlaybackState.NONE; + mainHandler = Util.createHandlerForCurrentLooper(); + Uri streamRequestUri = checkNotNull(mediaItem.localConfiguration).uri; + streamRequest = ServerSideAdInsertionStreamRequest.fromUri(streamRequestUri); + } + + @Override + public MediaItem getMediaItem() { + return mediaItem; + } + + @Override + public void prepareSourceInternal(@Nullable TransferListener mediaTransferListener) { + super.prepareSourceInternal(mediaTransferListener); + if (loader == null) { + Loader loader = new Loader("ImaServerSideAdInsertionMediaSource"); + player.addListener(componentListener); + StreamManagerLoadable streamManagerLoadable = + new StreamManagerLoadable( + adsLoader, + streamRequest.getStreamRequest(), + streamPlayer, + applicationAdErrorListener, + streamRequest.loadVideoTimeoutMs); + loader.startLoading( + streamManagerLoadable, + new StreamManagerLoadableCallback(), + /* defaultMinRetryCount= */ 0); + this.loader = loader; + } + } + + @Override + protected void onChildSourceInfoRefreshed( + Void id, MediaSource mediaSource, Timeline newTimeline) { + refreshSourceInfo( + new ForwardingTimeline(newTimeline) { + @Override + public Window getWindow( + int windowIndex, Window window, long defaultPositionProjectionUs) { + newTimeline.getWindow(windowIndex, window, defaultPositionProjectionUs); + window.mediaItem = mediaItem; + return window; + } + }); + } + + @Override + public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long startPositionUs) { + return checkNotNull(serverSideAdInsertionMediaSource) + .createPeriod(id, allocator, startPositionUs); + } + + @Override + public void releasePeriod(MediaPeriod mediaPeriod) { + checkNotNull(serverSideAdInsertionMediaSource).releasePeriod(mediaPeriod); + } + + @Override + public void maybeThrowSourceInfoRefreshError() throws IOException { + super.maybeThrowSourceInfoRefreshError(); + if (loadError != null) { + IOException loadError = this.loadError; + this.loadError = null; + throw loadError; + } + } + + @Override + protected void releaseSourceInternal() { + super.releaseSourceInternal(); + if (loader != null) { + loader.release(); + player.removeListener(componentListener); + mainHandler.post(() -> setStreamManager(/* streamManager= */ null)); + loader = null; + } + } + + // Internal methods (called on the main thread). + + @MainThread + private void setStreamManager(@Nullable StreamManager streamManager) { + if (this.streamManager == streamManager) { + return; + } + if (this.streamManager != null) { + if (applicationAdEventListener != null) { + this.streamManager.removeAdEventListener(applicationAdEventListener); + } + if (applicationAdErrorListener != null) { + this.streamManager.removeAdErrorListener(applicationAdErrorListener); + } + this.streamManager.removeAdEventListener(componentListener); + this.streamManager.destroy(); + this.streamManager = null; + } + this.streamManager = streamManager; + if (streamManager != null) { + streamManager.addAdEventListener(componentListener); + if (applicationAdEventListener != null) { + streamManager.addAdEventListener(applicationAdEventListener); + } + if (applicationAdErrorListener != null) { + streamManager.addAdErrorListener(applicationAdErrorListener); + } + } + } + + @MainThread + private void setAdPlaybackState(AdPlaybackState adPlaybackState) { + if (adPlaybackState.equals(this.adPlaybackState)) { + return; + } + this.adPlaybackState = adPlaybackState; + invalidateServerSideAdInsertionAdPlaybackState(); + } + + @MainThread + private void setContentTimeline(Timeline contentTimeline) { + if (contentTimeline.equals(this.contentTimeline)) { + return; + } + this.contentTimeline = contentTimeline; + invalidateServerSideAdInsertionAdPlaybackState(); + } + + @MainThread + private void invalidateServerSideAdInsertionAdPlaybackState() { + if (!adPlaybackState.equals(AdPlaybackState.NONE) && contentTimeline != null) { + ImmutableMap splitAdPlaybackStates = + splitAdPlaybackStateForPeriods(adPlaybackState, contentTimeline); + streamPlayer.setAdPlaybackStates(streamRequest.adsId, splitAdPlaybackStates, contentTimeline); + checkNotNull(serverSideAdInsertionMediaSource).setAdPlaybackStates(splitAdPlaybackStates); + } + } + + // Internal methods (called on the playback thread). + + private void setContentUri(Uri contentUri) { + if (serverSideAdInsertionMediaSource != null) { + return; + } + ServerSideAdInsertionMediaSource serverSideAdInsertionMediaSource = + new ServerSideAdInsertionMediaSource( + contentMediaSourceFactory.createMediaSource(MediaItem.fromUri(contentUri)), + componentListener); + this.serverSideAdInsertionMediaSource = serverSideAdInsertionMediaSource; + if (streamRequest.isLiveStream()) { + AdPlaybackState liveAdPlaybackState = + new AdPlaybackState(streamRequest.adsId) + .withNewAdGroup(/* adGroupIndex= */ 0, /* adGroupTimeUs= */ C.TIME_END_OF_SOURCE) + .withIsServerSideInserted(/* adGroupIndex= */ 0, true); + mainHandler.post(() -> setAdPlaybackState(liveAdPlaybackState)); + } + prepareChildSource(/* id= */ null, serverSideAdInsertionMediaSource); + } + + // Static methods. + + private static AdPlaybackState setVodAdGroupPlaceholders( + List cuePoints, AdPlaybackState adPlaybackState) { + for (int i = 0; i < cuePoints.size(); i++) { + CuePoint cuePoint = cuePoints.get(i); + adPlaybackState = + addAdGroupToAdPlaybackState( + adPlaybackState, + /* fromPositionUs= */ secToUs(cuePoint.getStartTime()), + /* contentResumeOffsetUs= */ 0, + /* adDurationsUs...= */ secToUs(cuePoint.getEndTime() - cuePoint.getStartTime())); + } + return adPlaybackState; + } + + private static AdPlaybackState setVodAdInPlaceholder(Ad ad, AdPlaybackState adPlaybackState) { + AdPodInfo adPodInfo = ad.getAdPodInfo(); + // Handle post rolls that have a podIndex of -1. + int adGroupIndex = + adPodInfo.getPodIndex() == -1 ? adPlaybackState.adGroupCount - 1 : adPodInfo.getPodIndex(); + AdPlaybackState.AdGroup adGroup = adPlaybackState.getAdGroup(adGroupIndex); + int adIndexInAdGroup = adPodInfo.getAdPosition() - 1; + if (adGroup.count < adPodInfo.getTotalAds()) { + adPlaybackState = + expandAdGroupPlaceholder( + adGroupIndex, + /* adGroupDurationUs= */ secToUs(adPodInfo.getMaxDuration()), + adIndexInAdGroup, + /* adDurationUs= */ secToUs(ad.getDuration()), + /* adsInAdGroupCount= */ adPodInfo.getTotalAds(), + adPlaybackState); + } else if (adIndexInAdGroup < adGroup.count - 1) { + adPlaybackState = + updateAdDurationInAdGroup( + adGroupIndex, + adIndexInAdGroup, + /* adDurationUs= */ secToUs(ad.getDuration()), + adPlaybackState); + } + return adPlaybackState; + } + + private static AdPlaybackState addLiveAdBreak( + Ad ad, long currentPeriodPositionUs, AdPlaybackState adPlaybackState) { + AdPodInfo adPodInfo = ad.getAdPodInfo(); + long adDurationUs = secToUs(ad.getDuration()); + int adIndexInAdGroup = adPodInfo.getAdPosition() - 1; + + // TODO(b/208398934) Support seeking backwards. + if (adIndexInAdGroup == 0 || adPlaybackState.adGroupCount == 1) { + // First ad of group. Create a new group with all ads. + long[] adDurationsUs = + updateAdDurationAndPropagate( + new long[adPodInfo.getTotalAds()], + adIndexInAdGroup, + adDurationUs, + secToUs(adPodInfo.getMaxDuration())); + adPlaybackState = + addAdGroupToAdPlaybackState( + adPlaybackState, + /* fromPositionUs= */ currentPeriodPositionUs, + /* contentResumeOffsetUs= */ sum(adDurationsUs), + /* adDurationsUs...= */ adDurationsUs); + } else { + int adGroupIndex = adPlaybackState.adGroupCount - 2; + adPlaybackState = + updateAdDurationInAdGroup(adGroupIndex, adIndexInAdGroup, adDurationUs, adPlaybackState); + AdPlaybackState.AdGroup adGroup = adPlaybackState.getAdGroup(adGroupIndex); + return adPlaybackState.withContentResumeOffsetUs( + adGroupIndex, min(adGroup.contentResumeOffsetUs, sum(adGroup.durationsUs))); + } + return adPlaybackState; + } + + private static AdPlaybackState skipAd(Ad ad, AdPlaybackState adPlaybackState) { + AdPodInfo adPodInfo = ad.getAdPodInfo(); + int adGroupIndex = adPodInfo.getPodIndex(); + // IMA SDK always returns index starting at 1. + int adIndexInAdGroup = adPodInfo.getAdPosition() - 1; + return adPlaybackState.withSkippedAd(adGroupIndex, adIndexInAdGroup); + } + + private final class ComponentListener + implements AdEvent.AdEventListener, Player.Listener, AdPlaybackStateUpdater { + + // Implement Player.Listener. + + @Override + public void onPositionDiscontinuity( + Player.PositionInfo oldPosition, + Player.PositionInfo newPosition, + @Player.DiscontinuityReason int reason) { + if (reason != Player.DISCONTINUITY_REASON_AUTO_TRANSITION) { + // Only auto transitions within the same or to the next media item are of interest. + return; + } + + if (mediaItem.equals(oldPosition.mediaItem) && !mediaItem.equals(newPosition.mediaItem)) { + // Playback automatically transitioned to the next media item. Notify the SDK. + streamPlayer.onContentCompleted(); + } + + if (!mediaItem.equals(oldPosition.mediaItem) + || !mediaItem.equals(newPosition.mediaItem) + || !streamRequest.adsId.equals( + player + .getCurrentTimeline() + .getPeriodByUid(checkNotNull(newPosition.periodUid), new Timeline.Period()) + .getAdsId())) { + // Discontinuity not within this ad media source. + return; + } + + if (oldPosition.adGroupIndex != C.INDEX_UNSET && newPosition.adGroupIndex == C.INDEX_UNSET) { + AdPlaybackState newAdPlaybackState = adPlaybackState; + for (int i = 0; i <= oldPosition.adIndexInAdGroup; i++) { + int state = newAdPlaybackState.getAdGroup(oldPosition.adGroupIndex).states[i]; + if (state != AdPlaybackState.AD_STATE_SKIPPED + && state != AdPlaybackState.AD_STATE_ERROR) { + newAdPlaybackState = + newAdPlaybackState.withPlayedAd( + oldPosition.adGroupIndex, /* adIndexInAdGroup= */ i); + } + } + setAdPlaybackState(newAdPlaybackState); + } + } + + @Override + public void onMetadata(Metadata metadata) { + if (!isCurrentAdPlaying(player, mediaItem, streamRequest.adsId)) { + return; + } + for (int i = 0; i < metadata.length(); i++) { + Metadata.Entry entry = metadata.get(i); + if (entry instanceof TextInformationFrame) { + TextInformationFrame textFrame = (TextInformationFrame) entry; + if ("TXXX".equals(textFrame.id)) { + streamPlayer.triggerUserTextReceived(textFrame.value); + } + } else if (entry instanceof EventMessage) { + EventMessage eventMessage = (EventMessage) entry; + String eventMessageValue = new String(eventMessage.messageData); + streamPlayer.triggerUserTextReceived(eventMessageValue); + } + } + } + + @Override + public void onPlaybackStateChanged(@Player.State int state) { + if (state == Player.STATE_ENDED + && isCurrentAdPlaying(player, mediaItem, streamRequest.adsId)) { + streamPlayer.onContentCompleted(); + } + } + + @Override + public void onVolumeChanged(float volume) { + if (!isCurrentAdPlaying(player, mediaItem, streamRequest.adsId)) { + return; + } + int volumePct = (int) Math.floor(volume * 100); + streamPlayer.onContentVolumeChanged(volumePct); + } + + // Implement AdEvent.AdEventListener. + + @MainThread + @Override + public void onAdEvent(AdEvent event) { + AdPlaybackState newAdPlaybackState = adPlaybackState; + switch (event.getType()) { + case CUEPOINTS_CHANGED: + // CUEPOINTS_CHANGED event is firing multiple times with the same queue points. + if (!streamRequest.isLiveStream() && newAdPlaybackState.equals(AdPlaybackState.NONE)) { + newAdPlaybackState = + setVodAdGroupPlaceholders( + checkNotNull(streamManager).getCuePoints(), + new AdPlaybackState(streamRequest.adsId)); + } + break; + case LOADED: + if (streamRequest.isLiveStream()) { + Timeline timeline = player.getCurrentTimeline(); + Timeline.Window window = + timeline.getWindow(player.getCurrentMediaItemIndex(), new Timeline.Window()); + if (window.lastPeriodIndex > window.firstPeriodIndex) { + // multi-period live not integrated + return; + } + long positionInWindowUs = + timeline.getPeriod(player.getCurrentPeriodIndex(), new Timeline.Period()) + .positionInWindowUs; + long currentPeriodPosition = + Util.msToUs(player.getCurrentPosition()) - positionInWindowUs; + newAdPlaybackState = + addLiveAdBreak( + event.getAd(), + currentPeriodPosition, + newAdPlaybackState.equals(AdPlaybackState.NONE) + ? new AdPlaybackState(streamRequest.adsId) + : newAdPlaybackState); + } else { + newAdPlaybackState = setVodAdInPlaceholder(event.getAd(), newAdPlaybackState); + } + break; + case SKIPPED: + if (!streamRequest.isLiveStream()) { + newAdPlaybackState = skipAd(event.getAd(), newAdPlaybackState); + } + break; + default: + // Do nothing. + break; + } + setAdPlaybackState(newAdPlaybackState); + } + + // Implement AdPlaybackStateUpdater (called on the playback thread). + + @Override + public boolean onAdPlaybackStateUpdateRequested(Timeline contentTimeline) { + mainHandler.post(() -> setContentTimeline(contentTimeline)); + // Defer source refresh to ad playback state update for VOD. Refresh immediately when live + // with single period. + return !streamRequest.isLiveStream() || contentTimeline.getPeriodCount() > 1; + } + } + + private final class StreamManagerLoadableCallback + implements Loader.Callback { + + @Override + public void onLoadCompleted( + StreamManagerLoadable loadable, long elapsedRealtimeMs, long loadDurationMs) { + mainHandler.post(() -> setStreamManager(checkNotNull(loadable.getStreamManager()))); + setContentUri(checkNotNull(loadable.getContentUri())); + } + + @Override + public void onLoadCanceled( + StreamManagerLoadable loadable, + long elapsedRealtimeMs, + long loadDurationMs, + boolean released) { + // We only cancel when the loader is released. + checkState(released); + } + + @Override + public LoadErrorAction onLoadError( + StreamManagerLoadable loadable, + long elapsedRealtimeMs, + long loadDurationMs, + IOException error, + int errorCount) { + loadError = error; + return Loader.DONT_RETRY; + } + } + + /** Loads the {@link StreamManager} and the content URI. */ + private static class StreamManagerLoadable + implements Loadable, AdsLoadedListener, AdErrorListener { + + private final com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader; + private final StreamRequest request; + private final StreamPlayer streamPlayer; + @Nullable private final AdErrorListener adErrorListener; + private final int loadVideoTimeoutMs; + private final ConditionVariable conditionVariable; + + @Nullable private volatile StreamManager streamManager; + @Nullable private volatile Uri contentUri; + private volatile boolean cancelled; + private volatile boolean error; + @Nullable private volatile String errorMessage; + private volatile int errorCode; + + /** Creates an instance. */ + private StreamManagerLoadable( + com.google.ads.interactivemedia.v3.api.AdsLoader adsLoader, + StreamRequest request, + StreamPlayer streamPlayer, + @Nullable AdErrorListener adErrorListener, + int loadVideoTimeoutMs) { + this.adsLoader = adsLoader; + this.request = request; + this.streamPlayer = streamPlayer; + this.adErrorListener = adErrorListener; + this.loadVideoTimeoutMs = loadVideoTimeoutMs; + conditionVariable = new ConditionVariable(); + errorCode = -1; + } + + /** Returns the DAI content URI or null if not yet available. */ + @Nullable + public Uri getContentUri() { + return contentUri; + } + + /** Returns the stream manager or null if not yet loaded. */ + @Nullable + public StreamManager getStreamManager() { + return streamManager; + } + + // Implement Loadable. + + @Override + public void load() throws IOException { + try { + // SDK will call loadUrl on stream player for SDK once manifest uri is available. + streamPlayer.setStreamLoadListener( + (streamUri, subtitles) -> { + contentUri = Uri.parse(streamUri); + conditionVariable.open(); + }); + if (adErrorListener != null) { + adsLoader.addAdErrorListener(adErrorListener); + } + adsLoader.addAdsLoadedListener(this); + adsLoader.addAdErrorListener(this); + adsLoader.requestStream(request); + while (contentUri == null && !cancelled && !error) { + try { + conditionVariable.block(); + } catch (InterruptedException e) { + /* Do nothing. */ + } + } + if (error && contentUri == null) { + throw new IOException(errorMessage + " [errorCode: " + errorCode + "]"); + } + } finally { + adsLoader.removeAdsLoadedListener(this); + adsLoader.removeAdErrorListener(this); + if (adErrorListener != null) { + adsLoader.removeAdErrorListener(adErrorListener); + } + } + } + + @Override + public void cancelLoad() { + cancelled = true; + } + + // AdsLoader.AdsLoadedListener implementation. + + @MainThread + @Override + public void onAdsManagerLoaded(AdsManagerLoadedEvent event) { + StreamManager streamManager = event.getStreamManager(); + if (streamManager == null) { + error = true; + errorMessage = "streamManager is null after ads manager has been loaded"; + conditionVariable.open(); + return; + } + AdsRenderingSettings adsRenderingSettings = + ImaSdkFactory.getInstance().createAdsRenderingSettings(); + adsRenderingSettings.setLoadVideoTimeout(loadVideoTimeoutMs); + // After initialization completed the streamUri will be reported to the streamPlayer. + streamManager.init(adsRenderingSettings); + this.streamManager = streamManager; + } + + // AdErrorEvent.AdErrorListener implementation. + + @MainThread + @Override + public void onAdError(AdErrorEvent adErrorEvent) { + error = true; + if (adErrorEvent.getError() != null) { + @Nullable String errorMessage = adErrorEvent.getError().getMessage(); + if (errorMessage != null) { + this.errorMessage = errorMessage.replace('\n', ' '); + } + errorCode = adErrorEvent.getError().getErrorCodeNumber(); + } + conditionVariable.open(); + } + } + + /** + * Receives the content URI from the SDK and sends back in-band media metadata and playback + * progression data to the SDK. + */ + private static final class StreamPlayer implements VideoStreamPlayer { + + /** A listener to listen for the stream URI loaded by the SDK. */ + public interface StreamLoadListener { + /** + * Loads a stream with dynamic ad insertion given the stream url and subtitles array. The + * subtitles array is only used in VOD streams. + * + *

Each entry in the subtitles array is a HashMap that corresponds to a language. Each map + * will have a "language" key with a two letter language string value, a "language name" to + * specify the set of subtitles if multiple sets exist for the same language, and one or more + * subtitle key/value pairs. Here's an example the map for English: + * + *

"language" -> "en" "language_name" -> "English" "webvtt" -> + * "https://example.com/vtt/en.vtt" "ttml" -> "https://example.com/ttml/en.ttml" + */ + void onLoadStream(String streamUri, List> subtitles); + } + + private final List callbacks; + private final Player player; + private final MediaItem mediaItem; + private final Timeline.Window window; + private final Timeline.Period period; + + private ImmutableMap adPlaybackStates; + @Nullable private Timeline contentTimeline; + @Nullable private Object adsId; + @Nullable private StreamLoadListener streamLoadListener; + + /** Creates an instance. */ + public StreamPlayer(Player player, MediaItem mediaItem) { + this.player = player; + this.mediaItem = mediaItem; + callbacks = new ArrayList<>(/* initialCapacity= */ 1); + adPlaybackStates = ImmutableMap.of(); + window = new Timeline.Window(); + period = new Timeline.Period(); + } + + /** Registers the ad playback states matching to the given content timeline. */ + public void setAdPlaybackStates( + Object adsId, + ImmutableMap adPlaybackStates, + Timeline contentTimeline) { + this.adsId = adsId; + this.adPlaybackStates = adPlaybackStates; + this.contentTimeline = contentTimeline; + } + + /** Sets the {@link StreamLoadListener} to be called when the SSAI content URI was loaded. */ + public void setStreamLoadListener(StreamLoadListener listener) { + streamLoadListener = Assertions.checkNotNull(listener); + } + + /** Called when the content has completed playback. */ + public void onContentCompleted() { + for (VideoStreamPlayer.VideoStreamPlayerCallback callback : callbacks) { + callback.onContentComplete(); + } + } + + /** Called when the content player changed the volume. */ + public void onContentVolumeChanged(int volumePct) { + for (VideoStreamPlayer.VideoStreamPlayerCallback callback : callbacks) { + callback.onVolumeChanged(volumePct); + } + } + + /** Releases the player. */ + public void release() { + callbacks.clear(); + adsId = null; + adPlaybackStates = ImmutableMap.of(); + contentTimeline = null; + streamLoadListener = null; + } + + // Implements VolumeProvider. + + @Override + public int getVolume() { + return (int) Math.floor(player.getVolume() * 100); + } + + // Implement ContentProgressProvider. + + @Override + public VideoProgressUpdate getContentProgress() { + if (!isCurrentAdPlaying(player, mediaItem, adsId)) { + return VideoProgressUpdate.VIDEO_TIME_NOT_READY; + } else if (adPlaybackStates.isEmpty()) { + return new VideoProgressUpdate(/* currentTimeMs= */ 0, /* durationMs= */ C.TIME_UNSET); + } + + Timeline timeline = player.getCurrentTimeline(); + int currentPeriodIndex = player.getCurrentPeriodIndex(); + timeline.getPeriod(currentPeriodIndex, period, /* setIds= */ true); + timeline.getWindow(player.getCurrentMediaItemIndex(), window); + + // We need the period of the content timeline because its period UIDs are the key used in the + // ad playback state map. The period UIDs of the public timeline are different (masking). + Timeline.Period contentPeriod = + checkNotNull(contentTimeline) + .getPeriod( + currentPeriodIndex - window.firstPeriodIndex, + new Timeline.Period(), + /* setIds= */ true); + AdPlaybackState adPlaybackState = checkNotNull(adPlaybackStates.get(contentPeriod.uid)); + + long streamPositionMs = + usToMs(ServerSideAdInsertionUtil.getStreamPositionUs(player, adPlaybackState)); + if (window.windowStartTimeMs != C.TIME_UNSET) { + // Add the time since epoch at start of the window for live streams. + streamPositionMs += window.windowStartTimeMs + period.getPositionInWindowMs(); + } else if (currentPeriodIndex > window.firstPeriodIndex) { + // Add the end position of the previous period in the underlying stream. + checkNotNull(contentTimeline) + .getPeriod( + currentPeriodIndex - window.firstPeriodIndex - 1, + contentPeriod, + /* setIds= */ true); + streamPositionMs += usToMs(contentPeriod.positionInWindowUs + contentPeriod.durationUs); + } + return new VideoProgressUpdate( + streamPositionMs, + checkNotNull(contentTimeline).getWindow(/* windowIndex= */ 0, window).getDurationMs()); + } + + // Implement VideoStreamPlayer. + + @Override + public void loadUrl(String url, List> subtitles) { + if (streamLoadListener != null) { + // SDK provided manifest url, notify the listener. + streamLoadListener.onLoadStream(url, subtitles); + } + } + + @Override + public void addCallback(VideoStreamPlayer.VideoStreamPlayerCallback callback) { + callbacks.add(callback); + } + + @Override + public void removeCallback(VideoStreamPlayer.VideoStreamPlayerCallback callback) { + callbacks.remove(callback); + } + + @Override + public void onAdBreakStarted() { + // Do nothing. + } + + @Override + public void onAdBreakEnded() { + // Do nothing. + } + + @Override + public void onAdPeriodStarted() { + // Do nothing. + } + + @Override + public void onAdPeriodEnded() { + // Do nothing. + } + + @Override + public void pause() { + // Do nothing. + } + + @Override + public void resume() { + // Do nothing. + } + + @Override + public void seek(long timeMs) { + // Do nothing. + } + + // Internal methods. + + private void triggerUserTextReceived(String userText) { + for (VideoStreamPlayer.VideoStreamPlayerCallback callback : callbacks) { + callback.onUserTextReceived(userText); + } + } + } + + private static boolean isCurrentAdPlaying( + Player player, MediaItem mediaItem, @Nullable Object adsId) { + if (player.getPlaybackState() == Player.STATE_IDLE) { + return false; + } + Timeline.Period period = new Timeline.Period(); + player.getCurrentTimeline().getPeriod(player.getCurrentPeriodIndex(), period); + return (period.isPlaceholder && mediaItem.equals(player.getCurrentMediaItem())) + || (adsId != null && adsId.equals(period.getAdsId())); + } + + private static StreamDisplayContainer createStreamDisplayContainer( + ImaSdkFactory imaSdkFactory, + ImaUtil.ServerSideAdInsertionConfiguration config, + StreamPlayer streamPlayer) { + StreamDisplayContainer container = + ImaSdkFactory.createStreamDisplayContainer( + checkNotNull(config.adViewProvider.getAdViewGroup()), streamPlayer); + container.setCompanionSlots(config.companionAdSlots); + registerFriendlyObstructions(imaSdkFactory, container, config.adViewProvider); + return container; + } + + private static void registerFriendlyObstructions( + ImaSdkFactory imaSdkFactory, + StreamDisplayContainer container, + AdViewProvider adViewProvider) { + for (int i = 0; i < adViewProvider.getAdOverlayInfos().size(); i++) { + AdOverlayInfo overlayInfo = adViewProvider.getAdOverlayInfos().get(i); + container.registerFriendlyObstruction( + imaSdkFactory.createFriendlyObstruction( + overlayInfo.view, + ImaUtil.getFriendlyObstructionPurpose(overlayInfo.purpose), + overlayInfo.reasonDetail != null ? overlayInfo.reasonDetail : "Unknown reason")); + } + } +} diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequest.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequest.java new file mode 100644 index 0000000000..6a52991489 --- /dev/null +++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequest.java @@ -0,0 +1,476 @@ +/* + * Copyright (C) 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ima; + +import static com.google.android.exoplayer2.util.Assertions.checkArgument; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; +import static com.google.android.exoplayer2.util.Assertions.checkState; + +import android.net.Uri; +import android.text.TextUtils; +import androidx.annotation.Nullable; +import com.google.ads.interactivemedia.v3.api.ImaSdkFactory; +import com.google.ads.interactivemedia.v3.api.StreamRequest; +import com.google.ads.interactivemedia.v3.api.StreamRequest.StreamFormat; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.C.ContentType; +import com.google.common.base.Objects; +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; + +/** Stream request data for an IMA DAI stream. */ +/* package */ final class ServerSideAdInsertionStreamRequest { + + /** The default timeout for loading the video URI, in milliseconds. */ + public static final int DEFAULT_LOAD_VIDEO_TIMEOUT_MS = 10_000; + + /** Builds a {@link ServerSideAdInsertionStreamRequest}. */ + public static final class Builder { + + @Nullable private String adsId; + @Nullable private String assetKey; + @Nullable private String apiKey; + @Nullable private String contentSourceId; + @Nullable private String videoId; + @Nullable private String manifestSuffix; + @Nullable private String contentUrl; + @Nullable private String authToken; + @Nullable private String streamActivityMonitorId; + private ImmutableMap adTagParameters; + @ContentType public int format = C.TYPE_HLS; + private int loadVideoTimeoutMs; + + /** Creates a new instance. */ + public Builder() { + adTagParameters = ImmutableMap.of(); + loadVideoTimeoutMs = DEFAULT_LOAD_VIDEO_TIMEOUT_MS; + } + + /** + * An opaque identifier for associated ad playback state, or {@code null} if the {@link + * #setAssetKey(String) asset key} (for live) or {@link #setVideoId(String) video id} (for VOD) + * should be used as the ads identifier. + * + * @param adsId The ads identifier. + * @return This instance, for convenience. + */ + public Builder setAdsId(String adsId) { + this.adsId = adsId; + return this; + } + + /** + * The stream request asset key used for live streams. + * + * @param assetKey Live stream asset key. + * @return This instance, for convenience. + */ + public Builder setAssetKey(@Nullable String assetKey) { + this.assetKey = assetKey; + return this; + } + + /** + * Sets the stream request authorization token. Used in place of {@link #setApiKey(String) the + * API key} for stricter content authorization. The publisher can control individual content + * streams authorizations based on this token. + * + * @param authToken Live stream authorization token. + * @return This instance, for convenience. + */ + public Builder setAuthToken(@Nullable String authToken) { + this.authToken = authToken; + return this; + } + + /** + * The stream request content source ID used for on-demand streams. + * + * @param contentSourceId VOD stream content source id. + * @return This instance, for convenience. + */ + public Builder setContentSourceId(@Nullable String contentSourceId) { + this.contentSourceId = contentSourceId; + return this; + } + + /** + * The stream request video ID used for on-demand streams. + * + * @param videoId VOD stream video id. + * @return This instance, for convenience. + */ + public Builder setVideoId(@Nullable String videoId) { + this.videoId = videoId; + return this; + } + + /** + * Sets the format of the stream request. + * + * @param format VOD or live stream type. + * @return This instance, for convenience. + */ + public Builder setFormat(@ContentType int format) { + checkArgument(format == C.TYPE_DASH || format == C.TYPE_HLS); + this.format = format; + return this; + } + + /** + * The stream request API key. This is used for content authentication. The API key is provided + * to the publisher to unlock their content. It's a security measure used to verify the + * applications that are attempting to access the content. + * + * @param apiKey Stream api key. + * @return This instance, for convenience. + */ + public Builder setApiKey(@Nullable String apiKey) { + this.apiKey = apiKey; + return this; + } + + /** + * Sets the ID to be used to debug the stream with the stream activity monitor. This is used to + * provide a convenient way to allow publishers to find a stream log in the stream activity + * monitor tool. + * + * @param streamActivityMonitorId ID for debugging the stream with the stream activity monitor. + * @return This instance, for convenience. + */ + public Builder setStreamActivityMonitorId(@Nullable String streamActivityMonitorId) { + this.streamActivityMonitorId = streamActivityMonitorId; + return this; + } + + /** + * Sets the overridable ad tag parameters on the stream request. Supply targeting parameters to your + * stream provides more information. + * + *

You can use the dai-ot and dai-ov parameters for stream variant preference. See Override Stream Variant Parameters + * for more information. + * + * @param adTagParameters A map of extra parameters to pass to the ad server. + * @return This instance, for convenience. + */ + public Builder setAdTagParameters(Map adTagParameters) { + this.adTagParameters = ImmutableMap.copyOf(adTagParameters); + return this; + } + + /** + * Sets the optional stream manifest's suffix, which will be appended to the stream manifest's + * URL. The provided string must be URL-encoded and must not include a leading question mark. + * + * @param manifestSuffix Stream manifest's suffix. + * @return This instance, for convenience. + */ + public Builder setManifestSuffix(@Nullable String manifestSuffix) { + this.manifestSuffix = manifestSuffix; + return this; + } + + /** + * Specifies the deep link to the content's screen. If provided, this parameter is passed to the + * OM SDK. See Android + * documentation for more information. + * + * @param contentUrl Deep link to the content's screen. + * @return This instance, for convenience. + */ + public Builder setContentUrl(@Nullable String contentUrl) { + this.contentUrl = contentUrl; + return this; + } + + /** + * Sets the duration after which resolving the video URI should time out, in milliseconds. + * + *

The default is {@link #DEFAULT_LOAD_VIDEO_TIMEOUT_MS} milliseconds. + * + * @param loadVideoTimeoutMs The timeout after which to give up resolving the video URI. + * @return This instance, for convenience. + */ + public Builder setLoadVideoTimeoutMs(int loadVideoTimeoutMs) { + this.loadVideoTimeoutMs = loadVideoTimeoutMs; + return this; + } + + /** + * Builds a {@link ServerSideAdInsertionStreamRequest} with the builder's current values. + * + * @return The build {@link ServerSideAdInsertionStreamRequest}. + * @throws IllegalStateException If request has missing or invalid inputs. + */ + public ServerSideAdInsertionStreamRequest build() { + checkState( + (TextUtils.isEmpty(assetKey) + && !TextUtils.isEmpty(contentSourceId) + && !TextUtils.isEmpty(videoId)) + || (!TextUtils.isEmpty(assetKey) + && TextUtils.isEmpty(contentSourceId) + && TextUtils.isEmpty(videoId))); + @Nullable String adsId = this.adsId; + if (adsId == null) { + adsId = assetKey != null ? assetKey : checkNotNull(videoId); + } + return new ServerSideAdInsertionStreamRequest( + adsId, + assetKey, + apiKey, + contentSourceId, + videoId, + adTagParameters, + manifestSuffix, + contentUrl, + authToken, + streamActivityMonitorId, + format, + loadVideoTimeoutMs); + } + } + + private static final String SCHEME = "imadai"; + private static final String ADS_ID = "adsId"; + private static final String ASSET_KEY = "assetKey"; + private static final String API_KEY = "apiKey"; + private static final String CONTENT_SOURCE_ID = "contentSourceId"; + private static final String VIDEO_ID = "videoId"; + private static final String AD_TAG_PARAMETERS = "adTagParameters"; + private static final String MANIFEST_SUFFIX = "manifestSuffix"; + private static final String CONTENT_URL = "contentUrl"; + private static final String AUTH_TOKEN = "authToken"; + private static final String STREAM_ACTIVITY_MONITOR_ID = "streamActivityMonitorId"; + private static final String FORMAT = "format"; + private static final String LOAD_VIDEO_TIMEOUT_MS = "loadVideoTimeoutMs"; + + public final String adsId; + @Nullable public final String assetKey; + @Nullable public final String apiKey; + @Nullable public final String contentSourceId; + @Nullable public final String videoId; + public final ImmutableMap adTagParameters; + @Nullable public final String manifestSuffix; + @Nullable public final String contentUrl; + @Nullable public final String authToken; + @Nullable public final String streamActivityMonitorId; + @ContentType public int format = C.TYPE_HLS; + public final int loadVideoTimeoutMs; + + private ServerSideAdInsertionStreamRequest( + String adsId, + @Nullable String assetKey, + @Nullable String apiKey, + @Nullable String contentSourceId, + @Nullable String videoId, + ImmutableMap adTagParameters, + @Nullable String manifestSuffix, + @Nullable String contentUrl, + @Nullable String authToken, + @Nullable String streamActivityMonitorId, + @ContentType int format, + int loadVideoTimeoutMs) { + this.adsId = adsId; + this.assetKey = assetKey; + this.apiKey = apiKey; + this.contentSourceId = contentSourceId; + this.videoId = videoId; + this.adTagParameters = adTagParameters; + this.manifestSuffix = manifestSuffix; + this.contentUrl = contentUrl; + this.authToken = authToken; + this.streamActivityMonitorId = streamActivityMonitorId; + this.format = format; + this.loadVideoTimeoutMs = loadVideoTimeoutMs; + } + + /** Returns whether this request is for a live stream or false if it is a VOD stream. */ + public boolean isLiveStream() { + return !TextUtils.isEmpty(assetKey); + } + + /** Returns the corresponding {@link StreamRequest}. */ + @SuppressWarnings("nullness") // Required for making nullness test pass for library_with_ima_sdk. + public StreamRequest getStreamRequest() { + StreamRequest streamRequest; + if (!TextUtils.isEmpty(assetKey)) { + streamRequest = ImaSdkFactory.getInstance().createLiveStreamRequest(assetKey, apiKey); + } else { + streamRequest = + ImaSdkFactory.getInstance() + .createVodStreamRequest(checkNotNull(contentSourceId), checkNotNull(videoId), apiKey); + } + if (format == C.TYPE_DASH) { + streamRequest.setFormat(StreamFormat.DASH); + } else if (format == C.TYPE_HLS) { + streamRequest.setFormat(StreamFormat.HLS); + } + // Optional params. + streamRequest.setAdTagParameters(adTagParameters); + if (manifestSuffix != null) { + streamRequest.setManifestSuffix(manifestSuffix); + } + if (contentUrl != null) { + streamRequest.setContentUrl(contentUrl); + } + if (authToken != null) { + streamRequest.setAuthToken(authToken); + } + if (streamActivityMonitorId != null) { + streamRequest.setStreamActivityMonitorId(streamActivityMonitorId); + } + return streamRequest; + } + + /** Returns a corresponding {@link Uri}. */ + public Uri toUri() { + Uri.Builder dataUriBuilder = new Uri.Builder(); + dataUriBuilder.scheme(SCHEME); + dataUriBuilder.appendQueryParameter(ADS_ID, adsId); + if (loadVideoTimeoutMs != DEFAULT_LOAD_VIDEO_TIMEOUT_MS) { + dataUriBuilder.appendQueryParameter( + LOAD_VIDEO_TIMEOUT_MS, String.valueOf(loadVideoTimeoutMs)); + } + if (assetKey != null) { + dataUriBuilder.appendQueryParameter(ASSET_KEY, assetKey); + } + if (apiKey != null) { + dataUriBuilder.appendQueryParameter(API_KEY, apiKey); + } + if (contentSourceId != null) { + dataUriBuilder.appendQueryParameter(CONTENT_SOURCE_ID, contentSourceId); + } + if (videoId != null) { + dataUriBuilder.appendQueryParameter(VIDEO_ID, videoId); + } + if (manifestSuffix != null) { + dataUriBuilder.appendQueryParameter(MANIFEST_SUFFIX, manifestSuffix); + } + if (contentUrl != null) { + dataUriBuilder.appendQueryParameter(CONTENT_URL, contentUrl); + } + if (authToken != null) { + dataUriBuilder.appendQueryParameter(AUTH_TOKEN, authToken); + } + if (streamActivityMonitorId != null) { + dataUriBuilder.appendQueryParameter(STREAM_ACTIVITY_MONITOR_ID, streamActivityMonitorId); + } + if (!adTagParameters.isEmpty()) { + Uri.Builder adTagParametersUriBuilder = new Uri.Builder(); + for (Map.Entry entry : adTagParameters.entrySet()) { + adTagParametersUriBuilder.appendQueryParameter(entry.getKey(), entry.getValue()); + } + dataUriBuilder.appendQueryParameter( + AD_TAG_PARAMETERS, adTagParametersUriBuilder.build().toString()); + } + dataUriBuilder.appendQueryParameter(FORMAT, String.valueOf(format)); + return dataUriBuilder.build(); + } + + @Override + public boolean equals(@Nullable Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ServerSideAdInsertionStreamRequest)) { + return false; + } + ServerSideAdInsertionStreamRequest that = (ServerSideAdInsertionStreamRequest) o; + return format == that.format + && loadVideoTimeoutMs == that.loadVideoTimeoutMs + && Objects.equal(adsId, that.adsId) + && Objects.equal(assetKey, that.assetKey) + && Objects.equal(apiKey, that.apiKey) + && Objects.equal(contentSourceId, that.contentSourceId) + && Objects.equal(videoId, that.videoId) + && Objects.equal(adTagParameters, that.adTagParameters) + && Objects.equal(manifestSuffix, that.manifestSuffix) + && Objects.equal(contentUrl, that.contentUrl) + && Objects.equal(authToken, that.authToken) + && Objects.equal(streamActivityMonitorId, that.streamActivityMonitorId); + } + + @Override + public int hashCode() { + return Objects.hashCode( + adsId, + assetKey, + apiKey, + contentSourceId, + videoId, + adTagParameters, + manifestSuffix, + contentUrl, + authToken, + streamActivityMonitorId, + loadVideoTimeoutMs, + format); + } + + /** + * Creates a {@link ServerSideAdInsertionStreamRequest} for the given URI. + * + * @param uri The URI. + * @return An {@link ServerSideAdInsertionStreamRequest} for the given URI. + * @throws IllegalStateException If uri has missing or invalid inputs. + */ + public static ServerSideAdInsertionStreamRequest fromUri(Uri uri) { + ServerSideAdInsertionStreamRequest.Builder request = + new ServerSideAdInsertionStreamRequest.Builder(); + if (!SCHEME.equals(uri.getScheme())) { + throw new IllegalArgumentException("Invalid scheme."); + } + request.setAdsId(checkNotNull(uri.getQueryParameter(ADS_ID))); + request.setAssetKey(uri.getQueryParameter(ASSET_KEY)); + request.setApiKey(uri.getQueryParameter(API_KEY)); + request.setContentSourceId(uri.getQueryParameter(CONTENT_SOURCE_ID)); + request.setVideoId(uri.getQueryParameter(VIDEO_ID)); + request.setManifestSuffix(uri.getQueryParameter(MANIFEST_SUFFIX)); + request.setContentUrl(uri.getQueryParameter(CONTENT_URL)); + request.setAuthToken(uri.getQueryParameter(AUTH_TOKEN)); + request.setStreamActivityMonitorId(uri.getQueryParameter(STREAM_ACTIVITY_MONITOR_ID)); + String adsLoaderTimeoutUs = uri.getQueryParameter(LOAD_VIDEO_TIMEOUT_MS); + request.setLoadVideoTimeoutMs( + TextUtils.isEmpty(adsLoaderTimeoutUs) + ? DEFAULT_LOAD_VIDEO_TIMEOUT_MS + : Integer.parseInt(adsLoaderTimeoutUs)); + String formatValue = uri.getQueryParameter(FORMAT); + if (!TextUtils.isEmpty(formatValue)) { + request.setFormat(Integer.parseInt(formatValue)); + } + Map adTagParameters; + String adTagParametersValue; + String singleAdTagParameterValue; + if (uri.getQueryParameter(AD_TAG_PARAMETERS) != null) { + adTagParameters = new HashMap<>(); + adTagParametersValue = uri.getQueryParameter(AD_TAG_PARAMETERS); + if (!TextUtils.isEmpty(adTagParametersValue)) { + Uri adTagParametersUri = Uri.parse(adTagParametersValue); + for (String paramName : adTagParametersUri.getQueryParameterNames()) { + singleAdTagParameterValue = adTagParametersUri.getQueryParameter(paramName); + if (!TextUtils.isEmpty(singleAdTagParameterValue)) { + adTagParameters.put(paramName, singleAdTagParameterValue); + } + } + } + request.setAdTagParameters(adTagParameters); + } + return request.build(); + } +} diff --git a/extensions/ima/src/test/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequestTest.java b/extensions/ima/src/test/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequestTest.java new file mode 100644 index 0000000000..79e2f90da5 --- /dev/null +++ b/extensions/ima/src/test/java/com/google/android/exoplayer2/ext/ima/ServerSideAdInsertionStreamRequestTest.java @@ -0,0 +1,148 @@ +/* + * Copyright 2021 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ima; + +import static com.google.common.truth.Truth.assertThat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.util.HashMap; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link ServerSideAdInsertionStreamRequest}. */ +@RunWith(AndroidJUnit4.class) +public final class ServerSideAdInsertionStreamRequestTest { + + private static final String ADS_ID = "testAdsId"; + private static final String ASSET_KEY = "testAssetKey"; + private static final String API_KEY = "testApiKey"; + private static final String CONTENT_SOURCE_ID = "testContentSourceId"; + private static final String VIDEO_ID = "testVideoId"; + private static final String MANIFEST_SUFFIX = "testManifestSuffix"; + private static final String CONTENT_URL = + "http://google.com/contentUrl?queryParamName=queryParamValue"; + private static final String AUTH_TOKEN = "testAuthToken"; + private static final String STREAM_ACTIVITY_MONITOR_ID = "testStreamActivityMonitorId"; + private static final int ADS_LOADER_TIMEOUT_MS = 2; + private static final int FORMAT_DASH = 0; + private static final int FORMAT_HLS = 2; + private static final Map adTagParameters = new HashMap<>(); + + static { + adTagParameters.put("param1", "value1"); + adTagParameters.put("param2", "value2"); + } + + @Test + public void build_live_correctUriAndParsing() { + ServerSideAdInsertionStreamRequest.Builder builder = + new ServerSideAdInsertionStreamRequest.Builder(); + builder.setAdsId(ADS_ID); + builder.setAssetKey(ASSET_KEY); + builder.setApiKey(API_KEY); + builder.setManifestSuffix(MANIFEST_SUFFIX); + builder.setContentUrl(CONTENT_URL); + builder.setAuthToken(AUTH_TOKEN); + builder.setStreamActivityMonitorId(STREAM_ACTIVITY_MONITOR_ID); + builder.setFormat(FORMAT_HLS); + builder.setAdTagParameters(adTagParameters); + builder.setLoadVideoTimeoutMs(ADS_LOADER_TIMEOUT_MS); + ServerSideAdInsertionStreamRequest streamRequest = builder.build(); + + ServerSideAdInsertionStreamRequest requestAfterConversions = + ServerSideAdInsertionStreamRequest.fromUri(streamRequest.toUri()); + + assertThat(streamRequest).isEqualTo(requestAfterConversions); + } + + @Test + public void build_vod_correctUriAndParsing() { + ServerSideAdInsertionStreamRequest.Builder builder = + new ServerSideAdInsertionStreamRequest.Builder(); + builder.setAdsId(ADS_ID); + builder.setApiKey(API_KEY); + builder.setContentSourceId(CONTENT_SOURCE_ID); + builder.setVideoId(VIDEO_ID); + builder.setManifestSuffix(MANIFEST_SUFFIX); + builder.setContentUrl(CONTENT_URL); + builder.setAuthToken(AUTH_TOKEN); + builder.setStreamActivityMonitorId(STREAM_ACTIVITY_MONITOR_ID); + builder.setFormat(FORMAT_DASH); + builder.setAdTagParameters(adTagParameters); + builder.setLoadVideoTimeoutMs(ADS_LOADER_TIMEOUT_MS); + ServerSideAdInsertionStreamRequest streamRequest = builder.build(); + + ServerSideAdInsertionStreamRequest requestAfterConversions = + ServerSideAdInsertionStreamRequest.fromUri(streamRequest.toUri()); + + assertThat(requestAfterConversions).isEqualTo(streamRequest); + } + + @Test + public void build_vodWithNoAdsId_usesVideoIdAsDefault() { + ServerSideAdInsertionStreamRequest.Builder builder = + new ServerSideAdInsertionStreamRequest.Builder(); + builder.setContentSourceId(CONTENT_SOURCE_ID); + builder.setVideoId(VIDEO_ID); + + ServerSideAdInsertionStreamRequest streamRequest = builder.build(); + + assertThat(streamRequest.adsId).isEqualTo(VIDEO_ID); + assertThat(streamRequest.toUri().getQueryParameter("adsId")).isEqualTo(VIDEO_ID); + } + + @Test + public void build_liveWithNoAdsId_usesAssetKeyAsDefault() { + ServerSideAdInsertionStreamRequest.Builder builder = + new ServerSideAdInsertionStreamRequest.Builder(); + builder.setAssetKey(ASSET_KEY); + + ServerSideAdInsertionStreamRequest streamRequest = builder.build(); + + assertThat(streamRequest.adsId).isEqualTo(ASSET_KEY); + assertThat(streamRequest.toUri().getQueryParameter("adsId")).isEqualTo(ASSET_KEY); + } + + @Test + public void build_assetKeyWithVideoId_throwsIllegalStateException() { + ServerSideAdInsertionStreamRequest.Builder requestBuilder = + new ServerSideAdInsertionStreamRequest.Builder(); + requestBuilder.setAssetKey(ASSET_KEY); + requestBuilder.setVideoId(VIDEO_ID); + + Assert.assertThrows(IllegalStateException.class, requestBuilder::build); + } + + @Test + public void build_assetKeyWithContentSource_throwsIllegalStateException() { + ServerSideAdInsertionStreamRequest.Builder requestBuilder = + new ServerSideAdInsertionStreamRequest.Builder(); + requestBuilder.setAssetKey(ASSET_KEY); + requestBuilder.setContentSourceId(CONTENT_SOURCE_ID); + + Assert.assertThrows(IllegalStateException.class, requestBuilder::build); + } + + @Test + public void build_withoutContentSourceAndVideoIdOrAssetKey_throwsIllegalStateException() { + ServerSideAdInsertionStreamRequest.Builder requestBuilder = + new ServerSideAdInsertionStreamRequest.Builder(); + + Assert.assertThrows(IllegalStateException.class, requestBuilder::build); + } +} From 035e16937fa9ebe440efc07d0064564bdb297fa7 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Tue, 1 Feb 2022 10:24:43 +0000 Subject: [PATCH 007/274] Add support for experimenting with HDR - Add a checkbox in the demo app to enable experimental HDR editing. - Add an `experimental_` method to `TransformationRequest` to enable HDR editing. - Add fragment/vertex shaders for the experimental HDR pipeline. The main difference compared to the existing shaders is that we sample from the decoder in YUV rather than RGB (because the YUV -> RGB conversion in the graphics driver is not precisely defined, so we need to do this to get consistent results), which requires the use of ES 3, and then do a crude YUV -> RGB conversion in the shader (ignoring the input color primaries for now). - When HDR editing is enabled, we force using `FrameEditor` (no passthrough) to avoid the need to select another edit operation, and use the new shaders. The `EGLContext` and `EGLSurface` also need to be set up differently for this path. PiperOrigin-RevId: 425570639 --- .../ConfigurationActivity.java | 21 +++- .../transformerdemo/TransformerActivity.java | 2 + .../res/layout/configuration_activity.xml | 10 ++ .../src/main/res/values/strings.xml | 1 + .../android/exoplayer2/util/GlUtil.java | 112 ++++++++++++++---- .../FrameEditorDataProcessingTest.java | 1 + .../transformer/FrameEditorTest.java | 4 +- ...lsl => fragment_shader_copy_external.glsl} | 0 ...fragment_shader_copy_external_yuv_es3.glsl | 29 +++++ ...glsl => vertex_shader_transformation.glsl} | 0 .../vertex_shader_transformation_es3.glsl | 23 ++++ .../exoplayer2/transformer/FrameEditor.java | 71 +++++++++-- .../transformer/TransformationRequest.java | 40 ++++++- .../transformer/TransformerVideoRenderer.java | 3 + .../VideoTranscodingSamplePipeline.java | 4 +- 15 files changed, 274 insertions(+), 47 deletions(-) rename library/transformer/src/main/assets/shaders/{fragment_shader.glsl => fragment_shader_copy_external.glsl} (100%) create mode 100644 library/transformer/src/main/assets/shaders/fragment_shader_copy_external_yuv_es3.glsl rename library/transformer/src/main/assets/shaders/{vertex_shader.glsl => vertex_shader_transformation.glsl} (100%) create mode 100644 library/transformer/src/main/assets/shaders/vertex_shader_transformation_es3.glsl diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java index a0f96d8b69..3381935df4 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java @@ -54,6 +54,7 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; + public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; private static final String[] INPUT_URIS = { "https://html5demos.com/assets/dizzy.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/android-block-1080-hevc.mp4", @@ -69,6 +70,7 @@ public final class ConfigurationActivity extends AppCompatActivity { private static final String SAME_AS_INPUT_OPTION = "same as input"; private @MonotonicNonNull Button chooseFileButton; + private @MonotonicNonNull TextView chosenFileTextView; private @MonotonicNonNull CheckBox removeAudioCheckbox; private @MonotonicNonNull CheckBox removeVideoCheckbox; private @MonotonicNonNull CheckBox flattenForSlowMotionCheckbox; @@ -78,7 +80,7 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner translateSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; - private @MonotonicNonNull TextView chosenFileTextView; + private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; private int inputUriPosition; @Override @@ -151,6 +153,8 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateSpinner = findViewById(R.id.rotate_spinner); rotateSpinner.setAdapter(rotateAdapter); rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "90", "180"); + + enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); } @Override @@ -178,7 +182,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "translateSpinner", "scaleSpinner", - "rotateSpinner" + "rotateSpinner", + "enableHdrEditingCheckBox" }) private void startTransformation(View view) { Intent transformerIntent = new Intent(this, TransformerActivity.class); @@ -216,6 +221,7 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedRotate)) { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } + bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); transformerIntent.putExtras(bundle); @Nullable Uri intentUri = getIntent().getData(); @@ -247,7 +253,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "translateSpinner", "scaleSpinner", - "rotateSpinner" + "rotateSpinner", + "enableHdrEditingCheckBox" }) private void onRemoveAudio(View view) { if (((CheckBox) view).isChecked()) { @@ -265,7 +272,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "translateSpinner", "scaleSpinner", - "rotateSpinner" + "rotateSpinner", + "enableHdrEditingCheckBox" }) private void onRemoveVideo(View view) { if (((CheckBox) view).isChecked()) { @@ -282,7 +290,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "translateSpinner", "scaleSpinner", - "rotateSpinner" + "rotateSpinner", + "enableHdrEditingCheckBox" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { audioMimeSpinner.setEnabled(isAudioEnabled); @@ -291,6 +300,7 @@ public final class ConfigurationActivity extends AppCompatActivity { translateSpinner.setEnabled(isVideoEnabled); scaleSpinner.setEnabled(isVideoEnabled); rotateSpinner.setEnabled(isVideoEnabled); + enableHdrEditingCheckBox.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); @@ -298,5 +308,6 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.translate).setEnabled(isVideoEnabled); findViewById(R.id.scale).setEnabled(isVideoEnabled); findViewById(R.id.rotate).setEnabled(isVideoEnabled); + findViewById(R.id.hdr_editing).setEnabled(isVideoEnabled); } } diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index 888ddd3edc..8220bfe498 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -213,6 +213,8 @@ public final class TransformerActivity extends AppCompatActivity { if (!transformationMatrix.isIdentity()) { requestBuilder.setTransformationMatrix(transformationMatrix); } + requestBuilder.experimental_setEnableHdrEditing( + bundle.getBoolean(ConfigurationActivity.ENABLE_HDR_EDITING)); transformerBuilder .setTransformationRequest(requestBuilder.build()) .setRemoveAudio(bundle.getBoolean(ConfigurationActivity.SHOULD_REMOVE_AUDIO)) diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index a9a9410a35..c3403a6269 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -164,6 +164,16 @@ android:layout_gravity="right|center_vertical" android:gravity="right" /> + + + +