diff --git a/libraries/common/src/main/java/androidx/media3/common/SurfaceInfo.java b/libraries/common/src/main/java/androidx/media3/common/SurfaceInfo.java index 02d6462c5b..6dcf2e7f94 100644 --- a/libraries/common/src/main/java/androidx/media3/common/SurfaceInfo.java +++ b/libraries/common/src/main/java/androidx/media3/common/SurfaceInfo.java @@ -41,6 +41,9 @@ public final class SurfaceInfo { */ public final int orientationDegrees; + /** Whether the {@link #surface} is an encoder input surface. */ + public final boolean isEncoderInputSurface; + /** Creates a new instance. */ public SurfaceInfo(Surface surface, int width, int height) { this(surface, width, height, /* orientationDegrees= */ 0); @@ -48,6 +51,16 @@ public final class SurfaceInfo { /** Creates a new instance. */ public SurfaceInfo(Surface surface, int width, int height, int orientationDegrees) { + this(surface, width, height, orientationDegrees, /* isEncoderInputSurface= */ false); + } + + /** Creates a new instance. */ + public SurfaceInfo( + Surface surface, + int width, + int height, + int orientationDegrees, + boolean isEncoderInputSurface) { checkArgument( orientationDegrees == 0 || orientationDegrees == 90 @@ -58,6 +71,7 @@ public final class SurfaceInfo { this.width = width; this.height = height; this.orientationDegrees = orientationDegrees; + this.isEncoderInputSurface = isEncoderInputSurface; } @Override @@ -72,6 +86,7 @@ public final class SurfaceInfo { return width == that.width && height == that.height && orientationDegrees == that.orientationDegrees + && isEncoderInputSurface == that.isEncoderInputSurface && surface.equals(that.surface); } @@ -81,6 +96,7 @@ public final class SurfaceInfo { result = 31 * result + width; result = 31 * result + height; result = 31 * result + orientationDegrees; + result = 31 * result + (isEncoderInputSurface ? 1 : 0); return result; } } diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 8ab45f7d39..2fb1c9a17b 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -186,6 +186,13 @@ public interface VideoFrameProcessor { /** Indicates the frame should be dropped after {@link #renderOutputFrame(long)} is invoked. */ long DROP_OUTPUT_FRAME = -2; + /** + * Indicates the frame should preserve the input presentation time when {@link + * #renderOutputFrame(long)} is invoked. + */ + @SuppressWarnings("GoodTime-ApiWithNumericTimeUnit") // This is a named constant, not a time unit. + long RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME = -3; + /** * Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}. * @@ -333,7 +340,10 @@ public interface VideoFrameProcessor { * * @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can * be before or after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the - * frame, or {@link #RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately. + * frame, or {@link #RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately, or + * {@link #RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME} to render the frame to the {@linkplain + * #setOutputSurfaceInfo output surface} with the presentation timestamp seen in {@link + * Listener#onOutputFrameAvailableForRendering(long)}. */ void renderOutputFrame(long renderTimeNs); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java index cba3ad2d66..c94bfe3981 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java @@ -15,6 +15,8 @@ */ package androidx.media3.effect; +import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY; +import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP; @@ -443,12 +445,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; GlUtil.clearFocusedBuffers(); defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs); - EGLExt.eglPresentationTimeANDROID( - eglDisplay, - outputEglSurface, - renderTimeNs == VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY - ? System.nanoTime() - : renderTimeNs); + long eglPresentationTimeNs; + if (renderTimeNs == RENDER_OUTPUT_FRAME_IMMEDIATELY) { + eglPresentationTimeNs = System.nanoTime(); + } else if (renderTimeNs == RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME) { + checkState(presentationTimeUs != C.TIME_UNSET); + eglPresentationTimeNs = presentationTimeUs * 1000; + } else { + eglPresentationTimeNs = renderTimeNs; + } + + EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, eglPresentationTimeNs); EGL14.eglSwapBuffers(eglDisplay, outputEglSurface); DebugTraceUtil.logEvent(COMPONENT_VFP, EVENT_RENDERED_TO_OUTPUT_SURFACE, presentationTimeUs); } @@ -524,8 +531,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; eglDisplay, outputSurfaceInfo.surface, outputColorInfo.colorTransfer, - // Frames are only rendered automatically when outputting to an encoder. - /* isEncoderInputSurface= */ renderFramesAutomatically); + outputSurfaceInfo.isEncoderInputSurface); } if (textureOutputListener != null) { outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight); diff --git a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java index 3ca008400b..e0d8bed8d0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java @@ -85,6 +85,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { private final SparseArray compositorOutputTextureReleases; private final long initialTimestampOffsetUs; + private final boolean renderFramesAutomatically; @Nullable private VideoFrameProcessor compositionVideoFrameProcessor; @Nullable private VideoCompositor videoCompositor; @@ -106,7 +107,8 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { checkArgument(videoFrameProcessorFactory instanceof DefaultVideoFrameProcessor.Factory); this.context = context; this.outputColorInfo = outputColorInfo; @@ -116,6 +118,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { this.videoCompositorSettings = videoCompositorSettings; this.compositionEffects = new ArrayList<>(compositionEffects); this.initialTimestampOffsetUs = initialTimestampOffsetUs; + this.renderFramesAutomatically = renderFramesAutomatically; lastRenderedPresentationTimeUs = C.TIME_UNSET; preProcessors = new SparseArray<>(); sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME); @@ -150,7 +153,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { context, debugViewProvider, outputColorInfo, - /* renderFramesAutomatically= */ true, + renderFramesAutomatically, /* listenerExecutor= */ MoreExecutors.directExecutor(), new VideoFrameProcessor.Listener() { // All of this listener's methods are called on the sharedExecutorService. @@ -174,6 +177,9 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { hasProducedFrameWithTimestampZero = true; } lastRenderedPresentationTimeUs = presentationTimeUs; + + listenerExecutor.execute( + () -> listener.onOutputFrameAvailableForRendering(presentationTimeUs)); } @Override @@ -312,6 +318,10 @@ public abstract class MultipleInputVideoGraph implements VideoGraph { released = true; } + protected VideoFrameProcessor getCompositionVideoFrameProcessor() { + return checkStateNotNull(compositionVideoFrameProcessor); + } + protected long getInitialTimestampOffsetUs() { return initialTimestampOffsetUs; } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index f08c36ff8c..f101212238 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -276,6 +276,7 @@ public final class AndroidTestUtil { .setCodecs("avc1.64001F") .build()) .setVideoDurationUs(1_024_000L) + .setVideoFrameCount(30) .setVideoTimestampsUs( ImmutableList.of( 0L, 33_366L, 66_733L, 100_100L, 133_466L, 166_833L, 200_200L, 233_566L, 266_933L, @@ -390,6 +391,7 @@ public final class AndroidTestUtil { .setFrameRate(30.00f) .setCodecs("avc1.42C015") .build()) + .setVideoFrameCount(932) .build(); public static final AssetInfo MP4_ASSET_WITH_SHORTER_AUDIO = diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 32dc6e4551..4296a68431 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -422,6 +422,60 @@ public class TransformerEndToEndTest { assertThat(new File(result.filePath).length()).isGreaterThan(0); } + @Test + public void videoEditing_withOneFrameInEncoder_completesWithConsistentFrameCount() + throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat); + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context)) + .experimentalSetMaxFramesInEncoder(1) + .build(); + MediaItem mediaItem = + MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri)); + EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(result.exportResult.videoFrameCount) + .isEqualTo(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFrameCount); + assertThat(new File(result.filePath).length()).isGreaterThan(0); + } + + @Test + public void videoEditing_withMaxFramesInEncoder_completesWithConsistentFrameCount() + throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat); + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context)) + .experimentalSetMaxFramesInEncoder(16) + .build(); + MediaItem mediaItem = + MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri)); + EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(result.exportResult.videoFrameCount) + .isEqualTo(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFrameCount); + assertThat(new File(result.filePath).length()).isGreaterThan(0); + } + // TODO: b/345483531 - Migrate this test to a Parameterized ImageSequence test. @Test public void videoEditing_withShortAlternatingImages_completesWithCorrectFrameCountAndDuration() diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java index fff4805931..4a873dcbca 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java @@ -31,6 +31,8 @@ import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; import android.graphics.Bitmap; +import android.net.Uri; +import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.MediaItem; import androidx.media3.common.util.Size; @@ -71,9 +73,15 @@ public final class TransformerMultiSequenceCompositionTest { private static final int EXPORT_WIDTH = 360; private static final int EXPORT_HEIGHT = 240; - @Parameters(name = "{0}") - public static ImmutableList workingColorSpaceLinear() { - return ImmutableList.of(false, true); + @Parameters(name = "{0},maxFramesInEncoder={1}") + public static ImmutableList parameters() { + ImmutableList.Builder listBuilder = new ImmutableList.Builder<>(); + for (Boolean workingColorSpaceLinear : new boolean[] {false, true}) { + for (Integer maxFramesInEncoder : new int[] {C.INDEX_UNSET, 1, 16}) { + listBuilder.add(new Object[] {workingColorSpaceLinear, maxFramesInEncoder}); + } + } + return listBuilder.build(); } private final Context context = ApplicationProvider.getApplicationContext(); @@ -81,7 +89,11 @@ public final class TransformerMultiSequenceCompositionTest { private String testId; - @Parameter public boolean workingColorSpaceLinear; + @Parameter(0) + public boolean workingColorSpaceLinear; + + @Parameter(1) + public int maxFramesInEncoder; @Before public void setUpTestId() { @@ -218,6 +230,33 @@ public final class TransformerMultiSequenceCompositionTest { extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId); } + @Test + public void export_completesWithConsistentFrameCount() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET.videoFormat, + /* outputFormat= */ MP4_ASSET.videoFormat); + MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET.uri)); + ImmutableList videoEffects = ImmutableList.of(Presentation.createForHeight(480)); + Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem).setEffects(effects).build(); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence(editedMediaItem), + new EditedMediaItemSequence(editedMediaItem)) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, buildTransformer()) + .build() + .run(testId, composition); + + assertThat(result.exportResult.videoFrameCount).isEqualTo(MP4_ASSET.videoFrameCount); + assertThat(new File(result.filePath).length()).isGreaterThan(0); + } + private Transformer buildTransformer() { // Use linear color space for grayscale effects. Transformer.Builder builder = new Transformer.Builder(context); @@ -227,6 +266,7 @@ public final class TransformerMultiSequenceCompositionTest { .setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR) .build()); } + builder.experimentalSetMaxFramesInEncoder(maxFramesInEncoder); return builder.build(); } @@ -278,7 +318,7 @@ public final class TransformerMultiSequenceCompositionTest { Bitmap actualBitmap = actualBitmaps.get(i); maybeSaveTestBitmap( testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null); - String subTestId = testId + "_" + i; + String subTestId = testId.replaceAll(",maxFramesInEncoder=-?\\d+", "") + "_" + i; Bitmap expectedBitmap = readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId)); float averagePixelAbsoluteDifference = diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java index 2936ba5c09..597ac93530 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java @@ -74,6 +74,7 @@ public final class ExperimentalAnalyzerModeFactory { return transformer .buildUpon() .experimentalSetTrimOptimizationEnabled(false) + .experimentalSetMaxFramesInEncoder(C.INDEX_UNSET) .setEncoderFactory(new DroppingEncoder.Factory(context)) .setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET) .setMuxerFactory( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 778cf9dcaa..9467b30454 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -115,6 +115,7 @@ public final class Transformer { private boolean trimOptimizationEnabled; private boolean fileStartsOnVideoFrameEnabled; private long maxDelayBetweenMuxerSamplesMs; + private int maxFramesInEncoder; private ListenerSet listeners; private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory; private AudioMixer.Factory audioMixerFactory; @@ -133,6 +134,7 @@ public final class Transformer { public Builder(Context context) { this.context = context.getApplicationContext(); maxDelayBetweenMuxerSamplesMs = DEFAULT_MAX_DELAY_BETWEEN_MUXER_SAMPLES_MS; + maxFramesInEncoder = C.INDEX_UNSET; audioProcessors = ImmutableList.of(); videoEffects = ImmutableList.of(); audioMixerFactory = new DefaultAudioMixer.Factory(); @@ -158,6 +160,7 @@ public final class Transformer { this.trimOptimizationEnabled = transformer.trimOptimizationEnabled; this.fileStartsOnVideoFrameEnabled = transformer.fileStartsOnVideoFrameEnabled; this.maxDelayBetweenMuxerSamplesMs = transformer.maxDelayBetweenMuxerSamplesMs; + this.maxFramesInEncoder = transformer.maxFramesInEncoder; this.listeners = transformer.listeners; this.assetLoaderFactory = transformer.assetLoaderFactory; this.audioMixerFactory = transformer.audioMixerFactory; @@ -333,6 +336,30 @@ public final class Transformer { return this; } + /** + * Limits how many video frames can be processed at any time by the {@linkplain Codec encoder}. + * + *

A video frame starts encoding when it enters the {@linkplain Codec#getInputSurface() + * encoder input surface}, and finishes encoding when the corresponding {@linkplain + * Codec#releaseOutputBuffer encoder output buffer is released}. + * + *

The default value is {@link C#INDEX_UNSET}, which means no limit is enforced. + * + *

This method is experimental and will be renamed or removed in a future release. + * + * @param maxFramesInEncoder The maximum number of frames that the video encoder is allowed to + * process at a time, or {@link C#INDEX_UNSET} if no limit is enforced. + * @return This builder. + * @throws IllegalArgumentException If {@code maxFramesInEncoder} is not equal to {@link + * C#INDEX_UNSET} and is non-positive. + */ + @CanIgnoreReturnValue + public Builder experimentalSetMaxFramesInEncoder(int maxFramesInEncoder) { + checkArgument(maxFramesInEncoder > 0 || maxFramesInEncoder == C.INDEX_UNSET); + this.maxFramesInEncoder = maxFramesInEncoder; + return this; + } + /** * Sets whether to ensure that the output file starts on a video frame. * @@ -592,6 +619,7 @@ public final class Transformer { trimOptimizationEnabled, fileStartsOnVideoFrameEnabled, maxDelayBetweenMuxerSamplesMs, + maxFramesInEncoder, listeners, assetLoaderFactory, audioMixerFactory, @@ -844,6 +872,7 @@ public final class Transformer { private final boolean trimOptimizationEnabled; private final boolean fileStartsOnVideoFrameEnabled; private final long maxDelayBetweenMuxerSamplesMs; + private final int maxFramesInEncoder; private final ListenerSet listeners; @Nullable private final AssetLoader.Factory assetLoaderFactory; @@ -881,6 +910,7 @@ public final class Transformer { boolean trimOptimizationEnabled, boolean fileStartsOnVideoFrameEnabled, long maxDelayBetweenMuxerSamplesMs, + int maxFramesInEncoder, ListenerSet listeners, @Nullable AssetLoader.Factory assetLoaderFactory, AudioMixer.Factory audioMixerFactory, @@ -901,6 +931,7 @@ public final class Transformer { this.trimOptimizationEnabled = trimOptimizationEnabled; this.fileStartsOnVideoFrameEnabled = fileStartsOnVideoFrameEnabled; this.maxDelayBetweenMuxerSamplesMs = maxDelayBetweenMuxerSamplesMs; + this.maxFramesInEncoder = maxFramesInEncoder; this.listeners = listeners; this.assetLoaderFactory = assetLoaderFactory; this.audioMixerFactory = audioMixerFactory; @@ -1611,6 +1642,7 @@ public final class Transformer { audioMixerFactory, videoFrameProcessorFactory, encoderFactory, + maxFramesInEncoder, muxerWrapper, componentListener, fallbackListener, diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java index 253d7a057b..244c3bc3e7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java @@ -148,6 +148,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final Object setMaxSequenceDurationUsLock; private final Object progressLock; private final ProgressHolder internalProgressHolder; + private final int maxFramesInEncoder; private boolean isDrainingExporters; @@ -192,6 +193,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; AudioMixer.Factory audioMixerFactory, VideoFrameProcessor.Factory videoFrameProcessorFactory, Codec.EncoderFactory encoderFactory, + int maxFramesInEncoder, MuxerWrapper muxerWrapper, Listener listener, FallbackListener fallbackListener, @@ -202,6 +204,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; this.context = context; this.composition = composition; this.encoderFactory = new CapturingEncoderFactory(encoderFactory); + this.maxFramesInEncoder = maxFramesInEncoder; this.listener = listener; this.applicationHandler = applicationHandler; this.clock = clock; @@ -738,8 +741,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; fallbackListener, debugViewProvider, videoSampleTimestampOffsetUs, - /* hasMultipleInputs= */ assetLoaderInputTracker - .hasMultipleConcurrentVideoTracks())); + /* hasMultipleInputs= */ assetLoaderInputTracker.hasMultipleConcurrentVideoTracks(), + maxFramesInEncoder)); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index 087105538d..5cbb73dd9f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -16,6 +16,8 @@ package androidx.media3.transformer; +import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; + import android.content.Context; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; @@ -53,7 +55,8 @@ import java.util.concurrent.Executor; Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { return new TransformerMultipleInputVideoGraph( context, videoFrameProcessorFactory, @@ -63,7 +66,8 @@ import java.util.concurrent.Executor; listenerExecutor, videoCompositorSettings, compositionEffects, - initialTimestampOffsetUs); + initialTimestampOffsetUs, + renderFramesAutomatically); } } @@ -76,7 +80,8 @@ import java.util.concurrent.Executor; Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { super( context, videoFrameProcessorFactory, @@ -86,7 +91,8 @@ import java.util.concurrent.Executor; listenerExecutor, videoCompositorSettings, compositionEffects, - initialTimestampOffsetUs); + initialTimestampOffsetUs, + renderFramesAutomatically); } @Override @@ -95,4 +101,10 @@ import java.util.concurrent.Executor; return new VideoFrameProcessingWrapper( getProcessor(inputIndex), /* presentation= */ null, getInitialTimestampOffsetUs()); } + + @Override + public void renderOutputFrameWithMediaPresentationTime() { + getCompositionVideoFrameProcessor() + .renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 571dd81ae6..0d2cbea29a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -16,6 +16,7 @@ package androidx.media3.transformer; +import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME; import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; @@ -57,7 +58,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) { + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) { @Nullable Presentation presentation = null; for (int i = 0; i < compositionEffects.size(); i++) { Effect effect = compositionEffects.get(i); @@ -73,7 +75,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; debugViewProvider, listenerExecutor, videoCompositorSettings, - /* renderFramesAutomatically= */ true, + renderFramesAutomatically, presentation, initialTimestampOffsetUs); } @@ -114,4 +116,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; getProcessor(inputIndex), getPresentation(), getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } + + @Override + public void renderOutputFrameWithMediaPresentationTime() { + getProcessor(getInputIndex()).renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java index 016da4c01a..311595eda6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java @@ -20,6 +20,7 @@ import android.content.Context; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; +import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; @@ -44,6 +45,10 @@ import java.util.concurrent.Executor; * composition. * @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition. * @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds. + * @param renderFramesAutomatically If {@code true}, the instance will render output frames to + * the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as the + * instance is done processing them. If {@code false}, the instance will block until {@link + * #renderOutputFrameWithMediaPresentationTime()} is called, to render the frame. * @return A new instance. * @throws VideoFrameProcessingException If a problem occurs while creating the {@link * VideoFrameProcessor}. @@ -56,7 +61,8 @@ import java.util.concurrent.Executor; Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, List compositionEffects, - long initialTimestampOffsetUs) + long initialTimestampOffsetUs, + boolean renderFramesAutomatically) throws VideoFrameProcessingException; } @@ -73,4 +79,18 @@ import java.util.concurrent.Executor; * @param inputIndex The index of the input, which could be used to order the inputs. */ GraphInput createInput(int inputIndex) throws VideoFrameProcessingException; + + /** + * Renders the oldest unrendered output frame that has become {@linkplain + * Listener#onOutputFrameAvailableForRendering(long) available for rendering} to the output + * surface. + * + *

This method must only be called if {@code renderFramesAutomatically} was set to {@code + * false} using the {@link Factory} and should be called exactly once for each frame that becomes + * {@linkplain Listener#onOutputFrameAvailableForRendering(long) available for rendering}. + * + *

This will render the output frame to the {@linkplain #setOutputSurfaceInfo output surface} + * with the presentation seen in {@link Listener#onOutputFrameAvailableForRendering(long)}. + */ + void renderOutputFrameWithMediaPresentationTime(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java index 63ce8adb40..acd51eff2f 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java @@ -24,6 +24,7 @@ import static androidx.media3.common.ColorInfo.SRGB_BT709_FULL; import static androidx.media3.common.ColorInfo.isTransferHdr; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR; import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL; import static androidx.media3.transformer.TransformerUtil.getOutputMimeTypeAndHdrModeAfterFallback; @@ -54,13 +55,14 @@ import java.nio.ByteBuffer; import java.util.List; import java.util.Objects; import org.checkerframework.checker.initialization.qual.Initialized; +import org.checkerframework.checker.lock.qual.GuardedBy; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.dataflow.qual.Pure; /** Processes, encodes and muxes raw video frames. */ /* package */ final class VideoSampleExporter extends SampleExporter { - private final TransformerVideoGraph videoGraph; + private final VideoGraphWrapper videoGraph; private final EncoderWrapper encoderWrapper; private final DecoderInputBuffer encoderOutputBuffer; private final long initialTimestampOffsetUs; @@ -86,7 +88,8 @@ import org.checkerframework.dataflow.qual.Pure; FallbackListener fallbackListener, DebugViewProvider debugViewProvider, long initialTimestampOffsetUs, - boolean hasMultipleInputs) + boolean hasMultipleInputs, + int maxFramesInEncoder) throws ExportException { // TODO(b/278259383) Consider delaying configuration of VideoSampleExporter to use the decoder // output format instead of the extractor output format, to match AudioSampleExporter behavior. @@ -142,7 +145,8 @@ import org.checkerframework.dataflow.qual.Pure; errorConsumer, debugViewProvider, videoCompositorSettings, - compositionEffects); + compositionEffects, + maxFramesInEncoder); videoGraph.initialize(); } catch (VideoFrameProcessingException e) { throw ExportException.createForVideoFrameProcessingException(e); @@ -199,6 +203,7 @@ import org.checkerframework.dataflow.qual.Pure; @Override protected void releaseMuxerInputBuffer() throws ExportException { encoderWrapper.releaseOutputBuffer(/* render= */ false); + videoGraph.onEncoderBufferReleased(); } @Override @@ -340,7 +345,8 @@ import org.checkerframework.dataflow.qual.Pure; encoder.getInputSurface(), actualEncoderFormat.width, actualEncoderFormat.height, - outputRotationDegrees); + outputRotationDegrees, + /* isEncoderInputSurface= */ true); if (releaseEncoder) { encoder.release(); @@ -454,6 +460,12 @@ import org.checkerframework.dataflow.qual.Pure; private final TransformerVideoGraph videoGraph; private final Consumer errorConsumer; + private final int maxFramesInEncoder; + private final boolean renderFramesAutomatically; + private final Object lock; + + private @GuardedBy("lock") int framesInEncoder; + private @GuardedBy("lock") int framesAvailableToRender; public VideoGraphWrapper( Context context, @@ -462,7 +474,8 @@ import org.checkerframework.dataflow.qual.Pure; Consumer errorConsumer, DebugViewProvider debugViewProvider, VideoCompositorSettings videoCompositorSettings, - List compositionEffects) + List compositionEffects, + int maxFramesInEncoder) throws VideoFrameProcessingException { this.errorConsumer = errorConsumer; // To satisfy the nullness checker by declaring an initialized this reference used in the @@ -470,6 +483,11 @@ import org.checkerframework.dataflow.qual.Pure; @SuppressWarnings("nullness:assignment") @Initialized VideoGraphWrapper thisRef = this; + this.maxFramesInEncoder = maxFramesInEncoder; + // Automatically render frames if the sample exporter does not limit the number of frames in + // the encoder. + renderFramesAutomatically = maxFramesInEncoder < 1; + lock = new Object(); videoGraph = videoGraphFactory.create( context, @@ -479,7 +497,8 @@ import org.checkerframework.dataflow.qual.Pure; /* listenerExecutor= */ MoreExecutors.directExecutor(), videoCompositorSettings, compositionEffects, - initialTimestampOffsetUs); + initialTimestampOffsetUs, + renderFramesAutomatically); } @Override @@ -495,7 +514,12 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) { - // Do nothing. + if (!renderFramesAutomatically) { + synchronized (lock) { + framesAvailableToRender += 1; + } + maybeRenderEarliestOutputFrame(); + } } @Override @@ -534,6 +558,11 @@ import org.checkerframework.dataflow.qual.Pure; return videoGraph.createInput(inputIndex); } + @Override + public void renderOutputFrameWithMediaPresentationTime() { + videoGraph.renderOutputFrameWithMediaPresentationTime(); + } + @Override public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) { videoGraph.setOutputSurfaceInfo(outputSurfaceInfo); @@ -548,5 +577,29 @@ import org.checkerframework.dataflow.qual.Pure; public void release() { videoGraph.release(); } + + public void onEncoderBufferReleased() { + if (!renderFramesAutomatically) { + synchronized (lock) { + checkState(framesInEncoder > 0); + framesInEncoder -= 1; + } + maybeRenderEarliestOutputFrame(); + } + } + + private void maybeRenderEarliestOutputFrame() { + boolean shouldRender = false; + synchronized (lock) { + if (framesAvailableToRender > 0 && framesInEncoder < maxFramesInEncoder) { + framesInEncoder += 1; + framesAvailableToRender -= 1; + shouldRender = true; + } + } + if (shouldRender) { + renderOutputFrameWithMediaPresentationTime(); + } + } } }