From a4f9f9487b559ff5fd737edae147b0eda7e747c3 Mon Sep 17 00:00:00 2001 From: kimvde Date: Mon, 16 Jan 2023 19:10:26 +0000 Subject: [PATCH] Add the possility to shift frame timestamps in SampleConsumer This is needed for constrained multi-asset to shift the timestamps of the media items that are not the first in the sequence. PiperOrigin-RevId: 502409923 --- .../android/exoplayer2/util/FrameInfo.java | 113 ++++++++++++++++-- .../exoplayer2/util/FrameProcessor.java | 5 +- .../video/MediaCodecVideoRenderer.java | 9 +- ...EffectsFrameProcessorFrameReleaseTest.java | 4 +- .../GlEffectsFrameProcessorPixelTest.java | 10 +- .../effect/ExternalTextureManager.java | 5 +- .../effect/GlEffectsFrameProcessor.java | 24 ++-- .../transformer/VideoSamplePipeline.java | 6 +- 8 files changed, 132 insertions(+), 44 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameInfo.java b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameInfo.java index 865bfec67e..164abd9cf4 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameInfo.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameInfo.java @@ -17,8 +17,95 @@ package com.google.android.exoplayer2.util; import static com.google.android.exoplayer2.util.Assertions.checkArgument; +import com.google.errorprone.annotations.CanIgnoreReturnValue; + /** Value class specifying information about a decoded video frame. */ public class FrameInfo { + + /** A builder for {@link FrameInfo} instances. */ + public static final class Builder { + + private int width; + private int height; + private float pixelWidthHeightRatio; + private long streamOffsetUs; + private long offsetToAddUs; + + /** + * Creates an instance with default values. + * + * @param width The frame width, in pixels. + * @param height The frame height, in pixels. + */ + public Builder(int width, int height) { + this.width = width; + this.height = height; + pixelWidthHeightRatio = 1; + } + + /** Creates an instance with the values of the provided {@link FrameInfo}. */ + public Builder(FrameInfo frameInfo) { + width = frameInfo.width; + height = frameInfo.height; + pixelWidthHeightRatio = frameInfo.pixelWidthHeightRatio; + streamOffsetUs = frameInfo.streamOffsetUs; + offsetToAddUs = frameInfo.offsetToAddUs; + } + + /** Sets the frame width, in pixels. */ + @CanIgnoreReturnValue + public Builder setWidth(int width) { + this.width = width; + return this; + } + + /** Sets the frame height, in pixels. */ + @CanIgnoreReturnValue + public Builder setHeight(int height) { + this.height = height; + return this; + } + + /** + * Sets the ratio of width over height for each pixel. + * + *

The default value is {@code 1}. + */ + @CanIgnoreReturnValue + public Builder setPixelWidthHeightRatio(float pixelWidthHeightRatio) { + this.pixelWidthHeightRatio = pixelWidthHeightRatio; + return this; + } + + /** + * Sets the {@linkplain FrameInfo#streamOffsetUs stream offset}, in microseconds. + * + *

The default value is {@code 0}. + */ + @CanIgnoreReturnValue + public Builder setStreamOffsetUs(long streamOffsetUs) { + this.streamOffsetUs = streamOffsetUs; + return this; + } + + /** + * Sets the {@linkplain FrameInfo#offsetToAddUs offset to add} to the frame presentation + * timestamp, in microseconds. + * + *

The default value is {@code 0}. + */ + @CanIgnoreReturnValue + public Builder setOffsetToAddUs(long offsetToAddUs) { + this.offsetToAddUs = offsetToAddUs; + return this; + } + + /** Builds a {@link FrameInfo} instance. */ + public FrameInfo build() { + return new FrameInfo(width, height, pixelWidthHeightRatio, streamOffsetUs, offsetToAddUs); + } + } + /** The width of the frame, in pixels. */ public final int width; /** The height of the frame, in pixels. */ @@ -29,23 +116,24 @@ public class FrameInfo { * An offset in microseconds that is part of the input timestamps and should be ignored for * processing but added back to the output timestamps. * - *

The offset stays constant within a stream but changes in between streams to ensure that - * frame timestamps are always monotonically increasing. + *

The offset stays constant within a stream. If the first timestamp of the next stream is less + * than or equal to the last timestamp of the current stream (including the {@linkplain + * #offsetToAddUs} offset to add), the stream offset must be updated between the streams to ensure + * that the offset frame timestamps are always monotonically increasing. */ public final long streamOffsetUs; + /** + * The offset that must be added to the frame presentation timestamp, in microseconds. + * + *

This offset is not part of the input timestamps. It is added to the frame timestamps before + * processing, and is retained in the output timestamps. + */ + public final long offsetToAddUs; // TODO(b/227624622): Add color space information for HDR. - /** - * Creates a new instance. - * - * @param width The width of the frame, in pixels. - * @param height The height of the frame, in pixels. - * @param pixelWidthHeightRatio The ratio of width over height for each pixel. - * @param streamOffsetUs An offset in microseconds that is part of the input timestamps and should - * be ignored for processing but added back to the output timestamps. - */ - public FrameInfo(int width, int height, float pixelWidthHeightRatio, long streamOffsetUs) { + private FrameInfo( + int width, int height, float pixelWidthHeightRatio, long streamOffsetUs, long offsetToAddUs) { checkArgument(width > 0, "width must be positive, but is: " + width); checkArgument(height > 0, "height must be positive, but is: " + height); @@ -53,5 +141,6 @@ public class FrameInfo { this.height = height; this.pixelWidthHeightRatio = pixelWidthHeightRatio; this.streamOffsetUs = streamOffsetUs; + this.offsetToAddUs = offsetToAddUs; } } diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java index c63fd5e65f..7d30e5da05 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/FrameProcessor.java @@ -135,8 +135,9 @@ public interface FrameProcessor { *

Pixels are expanded using the {@link FrameInfo#pixelWidthHeightRatio} so that the output * frames' pixels have a ratio of 1. * - *

The caller should update {@link FrameInfo#streamOffsetUs} when switching input streams to - * ensure that frame timestamps are always monotonically increasing. + *

The caller should update {@link FrameInfo#streamOffsetUs} when switching to an input stream + * whose first frame timestamp is less than or equal to the last timestamp received. This stream + * offset should ensure that frame timestamps are monotonically increasing. * *

Can be called on any thread. */ diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index 56cc4af916..adf150d21a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -2060,11 +2060,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { public void setInputFormat(Format inputFormat) { checkNotNull(frameProcessor) .setInputFrameInfo( - new FrameInfo( - inputFormat.width, - inputFormat.height, - inputFormat.pixelWidthHeightRatio, - renderer.getOutputStreamOffsetUs())); + new FrameInfo.Builder(inputFormat.width, inputFormat.height) + .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) + .setStreamOffsetUs(renderer.getOutputStreamOffsetUs()) + .build()); this.inputFormat = inputFormat; if (registeredLastFrame) { diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java index c0615d45ea..a1c1901716 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorFrameReleaseTest.java @@ -341,9 +341,7 @@ public final class GlEffectsFrameProcessorFrameReleaseTest { () -> { blankFrameProducer.configureGlObjects(); checkNotNull(glEffectsFrameProcessor) - .setInputFrameInfo( - new FrameInfo( - WIDTH, HEIGHT, /* pixelWidthHeightRatio= */ 1, /* streamOffsetUs= */ 0)); + .setInputFrameInfo(new FrameInfo.Builder(WIDTH, HEIGHT).build()); // A frame needs to be registered despite not queuing any external input to ensure // that // the frame processor knows about the stream offset. diff --git a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java index c24165dfe7..af37e9f1ed 100644 --- a/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java +++ b/library/effect/src/androidTest/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessorPixelTest.java @@ -739,11 +739,11 @@ public final class GlEffectsFrameProcessorPixelTest { @Override public void onContainerExtracted(MediaFormat mediaFormat) { glEffectsFrameProcessor.setInputFrameInfo( - new FrameInfo( - mediaFormat.getInteger(MediaFormat.KEY_WIDTH), - mediaFormat.getInteger(MediaFormat.KEY_HEIGHT), - pixelWidthHeightRatio, - /* streamOffsetUs= */ 0)); + new FrameInfo.Builder( + mediaFormat.getInteger(MediaFormat.KEY_WIDTH), + mediaFormat.getInteger(MediaFormat.KEY_HEIGHT)) + .setPixelWidthHeightRatio(pixelWidthHeightRatio) + .build()); glEffectsFrameProcessor.registerInputFrame(); } diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java index 377c0b4d08..c4ae3e0efd 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/ExternalTextureManager.java @@ -160,6 +160,7 @@ import java.util.concurrent.atomic.AtomicInteger; surfaceTexture.getTransformMatrix(textureTransformMatrix); externalTextureProcessor.setTextureTransformMatrix(textureTransformMatrix); long frameTimeNs = surfaceTexture.getTimestamp(); + long offsetToAddUs = currentFrame.offsetToAddUs; long streamOffsetUs = currentFrame.streamOffsetUs; if (streamOffsetUs != previousStreamOffsetUs) { if (previousStreamOffsetUs != C.TIME_UNSET) { @@ -167,8 +168,8 @@ import java.util.concurrent.atomic.AtomicInteger; } previousStreamOffsetUs = streamOffsetUs; } - // Correct for the stream offset so processors see original media presentation timestamps. - long presentationTimeUs = (frameTimeNs / 1000) - streamOffsetUs; + // Correct the presentation time so that processors don't see the stream offset. + long presentationTimeUs = (frameTimeNs / 1000) + offsetToAddUs - streamOffsetUs; externalTextureProcessor.queueInputFrame( new TextureInfo( externalTexId, /* fboId= */ C.INDEX_UNSET, currentFrame.width, currentFrame.height), diff --git a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java index 8812f3c378..5d4ea57f41 100644 --- a/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java +++ b/library/effect/src/main/java/com/google/android/exoplayer2/effect/GlEffectsFrameProcessor.java @@ -438,23 +438,21 @@ public final class GlEffectsFrameProcessor implements FrameProcessor { } /** - * Expands or shrinks the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a - * new {@link FrameInfo} instance with scaled dimensions and {@link - * FrameInfo#pixelWidthHeightRatio} of {@code 1}. + * Expands the frame based on the {@link FrameInfo#pixelWidthHeightRatio} and returns a new {@link + * FrameInfo} instance with scaled dimensions and {@link FrameInfo#pixelWidthHeightRatio} of + * {@code 1}. */ private FrameInfo adjustForPixelWidthHeightRatio(FrameInfo frameInfo) { if (frameInfo.pixelWidthHeightRatio > 1f) { - return new FrameInfo( - (int) (frameInfo.width * frameInfo.pixelWidthHeightRatio), - frameInfo.height, - /* pixelWidthHeightRatio= */ 1, - frameInfo.streamOffsetUs); + return new FrameInfo.Builder(frameInfo) + .setWidth((int) (frameInfo.width * frameInfo.pixelWidthHeightRatio)) + .setPixelWidthHeightRatio(1) + .build(); } else if (frameInfo.pixelWidthHeightRatio < 1f) { - return new FrameInfo( - frameInfo.width, - (int) (frameInfo.height / frameInfo.pixelWidthHeightRatio), - /* pixelWidthHeightRatio= */ 1, - frameInfo.streamOffsetUs); + return new FrameInfo.Builder(frameInfo) + .setHeight((int) (frameInfo.height / frameInfo.pixelWidthHeightRatio)) + .setPixelWidthHeightRatio(1) + .build(); } else { return frameInfo; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 34f0c56db8..f7832df739 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -222,8 +222,10 @@ import org.checkerframework.dataflow.qual.Pure; e, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED); } frameProcessor.setInputFrameInfo( - new FrameInfo( - decodedWidth, decodedHeight, inputFormat.pixelWidthHeightRatio, streamOffsetUs)); + new FrameInfo.Builder(decodedWidth, decodedHeight) + .setPixelWidthHeightRatio(inputFormat.pixelWidthHeightRatio) + .setStreamOffsetUs(streamOffsetUs) + .build()); } @Override