From a98a37aa75b17deb80b16d3782a9cf42a48e06b3 Mon Sep 17 00:00:00 2001 From: dancho Date: Thu, 1 Aug 2024 10:18:02 -0700 Subject: [PATCH] Initial Frame Extractor Transformer Factory Package-private until API is more useable. Similar to frame analyzer mode: uses ImageReader instead of an encoder, and no muxer. PiperOrigin-RevId: 658446675 --- .../transformer/TransformerEndToEndTest.java | 35 +++ .../ExperimentalAnalyzerModeFactory.java | 56 ----- .../ExperimentalFrameExtractorFactory.java | 217 ++++++++++++++++++ .../media3/transformer/NoWriteMuxer.java | 79 +++++++ 4 files changed, 331 insertions(+), 56 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractorFactory.java create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/NoWriteMuxer.java diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 4296a68431..fd8a25b44c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -1567,6 +1567,41 @@ public class TransformerEndToEndTest { assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); } + @Test + public void extractFrames_completesSuccessfully() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat, + /* outputFormat= */ null); + AtomicInteger imagesOutput = new AtomicInteger(/* initialValue= */ 0); + Transformer transformer = + ExperimentalFrameExtractorFactory.buildFrameExtractorTransformer( + context, image -> imagesOutput.incrementAndGet()); + AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri( + Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri))) + .setRemoveAudio(true) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + ImmutableList.of(createFrameCountingEffect(videoFramesSeen)))) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(videoFramesSeen.get()).isEqualTo(932); + assertThat(imagesOutput.get()).isEqualTo(932); + assertThat(result.exportResult.videoFrameCount).isEqualTo(932); + // Confirm no data was written to file. + assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + @Test public void transcode_withOutputVideoMimeTypeAv1_completesSuccessfully() throws Exception { assumeFormatsSupported( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java index 597ac93530..f412754dc5 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java @@ -19,18 +19,15 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; -import android.media.MediaCodec; import android.media.MediaCodec.BufferInfo; import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; -import androidx.media3.common.Metadata; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.decoder.DecoderInputBuffer; import androidx.media3.exoplayer.video.PlaceholderSurface; -import androidx.media3.muxer.Muxer; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -194,57 +191,4 @@ public final class ExperimentalAnalyzerModeFactory { @Override public void release() {} } - - /** A {@link Muxer} implementation that does nothing. */ - private static final class NoWriteMuxer implements Muxer { - public static final class Factory implements Muxer.Factory { - - private final ImmutableList audioMimeTypes; - private final ImmutableList videoMimeTypes; - - /** - * Creates an instance. - * - * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link - * #getSupportedSampleMimeTypes(int)}. - * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link - * #getSupportedSampleMimeTypes(int)}. - */ - public Factory(ImmutableList audioMimeTypes, ImmutableList videoMimeTypes) { - this.audioMimeTypes = audioMimeTypes; - this.videoMimeTypes = videoMimeTypes; - } - - @Override - public Muxer create(String path) { - return new NoWriteMuxer(); - } - - @Override - public ImmutableList getSupportedSampleMimeTypes(@C.TrackType int trackType) { - if (trackType == C.TRACK_TYPE_AUDIO) { - return audioMimeTypes; - } - if (trackType == C.TRACK_TYPE_VIDEO) { - return videoMimeTypes; - } - return ImmutableList.of(); - } - } - - @Override - public TrackToken addTrack(Format format) { - return new TrackToken() {}; - } - - @Override - public void writeSampleData( - TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {} - - @Override - public void addMetadataEntry(Metadata.Entry metadataEntry) {} - - @Override - public void close() {} - } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractorFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractorFactory.java new file mode 100644 index 0000000000..21ae28d7bb --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractorFactory.java @@ -0,0 +1,217 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.content.Context; +import android.graphics.PixelFormat; +import android.media.Image; +import android.media.ImageReader; +import android.media.MediaCodec.BufferInfo; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Util; +import androidx.media3.decoder.DecoderInputBuffer; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; + +/** + * Factory for creating instances of {@link Transformer} that can be used to extract frames. + * + *

This class is experimental and will be renamed or removed in a future release. + */ +/* package */ final class ExperimentalFrameExtractorFactory { + + private ExperimentalFrameExtractorFactory() {} + + /** A callback to be notified when a new image is available. */ + public interface Listener { + + // TODO: b/350498258 - Make this more user-friendly before making it a public API. + /** + * Called when a new {@link Image} is available. When this method returns, the {@link Image} + * will be closed and can no longer be used. + */ + void onImageAvailable(Image image); + } + + /** + * Builds a {@link Transformer} that runs as an analyzer. + * + *

No encoding or muxing is performed, therefore no data is written to any output files. + * + * @param context The {@link Context}. + * @param listener The {@link Listener} to be used for generated images. + * @return The fame extracting {@link Transformer}. + */ + public static Transformer buildFrameExtractorTransformer(Context context, Listener listener) { + return new Transformer.Builder(context) + .experimentalSetTrimOptimizationEnabled(false) + .setEncoderFactory(new ImageReaderEncoder.Factory(listener)) + .setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET) + .setMuxerFactory( + new NoWriteMuxer.Factory( + /* audioMimeTypes= */ ImmutableList.of(MimeTypes.AUDIO_AAC), + /* videoMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264))) + .setAudioMimeType(MimeTypes.AUDIO_AAC) + .setVideoMimeType(MimeTypes.VIDEO_H264) + .experimentalSetMaxFramesInEncoder(1) // Work around ImageReader frame dropping. + .build(); + } + + /** A {@linkplain Codec encoder} implementation that outputs frames via {@link ImageReader}. */ + private static final class ImageReaderEncoder implements Codec { + public static final class Factory implements Codec.EncoderFactory { + + private final Listener listener; + + public Factory(Listener listener) { + this.listener = listener; + } + + @Override + public Codec createForAudioEncoding(Format format) { + throw new UnsupportedOperationException(); + } + + @Override + public Codec createForVideoEncoding(Format format) { + return new ImageReaderEncoder(format, listener); + } + } + + private static final String TAG = "ImageReaderEncoder"; + private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocateDirect(0); + + private final Format configurationFormat; + private final ImageReader imageReader; + private final Queue processedImageTimestampsNs; + private final BufferInfo outputBufferInfo; + + private boolean hasOutputBuffer; + private boolean inputStreamEnded; + + public ImageReaderEncoder(Format format, Listener listener) { + this.configurationFormat = format; + imageReader = + ImageReader.newInstance( + format.width, format.height, PixelFormat.RGBA_8888, /* maxImages= */ 1); + + processedImageTimestampsNs = new ConcurrentLinkedQueue<>(); + + imageReader.setOnImageAvailableListener( + reader -> { + try (Image image = reader.acquireNextImage()) { + processedImageTimestampsNs.add(image.getTimestamp()); + listener.onImageAvailable(image); + } + }, + Util.createHandlerForCurrentOrMainLooper()); + + outputBufferInfo = new BufferInfo(); + } + + @Override + public String getName() { + return TAG; + } + + @Override + public Format getConfigurationFormat() { + return configurationFormat; + } + + @Override + public Surface getInputSurface() { + return imageReader.getSurface(); + } + + @Override + @EnsuresNonNullIf(expression = "#1.data", result = true) + public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) { + throw new UnsupportedOperationException(); + } + + @Override + public void queueInputBuffer(DecoderInputBuffer inputBuffer) { + throw new UnsupportedOperationException(); + } + + @Override + public void signalEndOfInputStream() { + inputStreamEnded = true; + } + + @Override + public Format getOutputFormat() { + return configurationFormat; + } + + @Override + @Nullable + public ByteBuffer getOutputBuffer() { + return maybeGenerateOutputBuffer() ? EMPTY_BUFFER : null; + } + + @Override + @Nullable + public BufferInfo getOutputBufferInfo() { + return maybeGenerateOutputBuffer() ? outputBufferInfo : null; + } + + @Override + public boolean isEnded() { + return inputStreamEnded && processedImageTimestampsNs.isEmpty(); + } + + @Override + public void releaseOutputBuffer(boolean render) { + releaseOutputBuffer(); + } + + @Override + public void releaseOutputBuffer(long renderPresentationTimeUs) { + releaseOutputBuffer(); + } + + private void releaseOutputBuffer() { + hasOutputBuffer = false; + } + + @Override + public void release() {} + + private boolean maybeGenerateOutputBuffer() { + if (hasOutputBuffer) { + return true; + } + Long timeNs = processedImageTimestampsNs.poll(); + if (timeNs == null) { + return false; + } + + hasOutputBuffer = true; + outputBufferInfo.presentationTimeUs = timeNs / 1000; + return true; + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/NoWriteMuxer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/NoWriteMuxer.java new file mode 100644 index 0000000000..6f25d63ecb --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/NoWriteMuxer.java @@ -0,0 +1,79 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import android.media.MediaCodec; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.muxer.Muxer; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; + +/** A {@link Muxer} implementation that does nothing. */ +/* package */ final class NoWriteMuxer implements Muxer { + public static final class Factory implements Muxer.Factory { + + private final ImmutableList audioMimeTypes; + private final ImmutableList videoMimeTypes; + + /** + * Creates an instance. + * + * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link + * #getSupportedSampleMimeTypes(int)}. + * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link + * #getSupportedSampleMimeTypes(int)}. + */ + public Factory(ImmutableList audioMimeTypes, ImmutableList videoMimeTypes) { + this.audioMimeTypes = audioMimeTypes; + this.videoMimeTypes = videoMimeTypes; + } + + @Override + public Muxer create(String path) { + return new NoWriteMuxer(); + } + + @Override + public ImmutableList getSupportedSampleMimeTypes(@C.TrackType int trackType) { + if (trackType == C.TRACK_TYPE_AUDIO) { + return audioMimeTypes; + } + if (trackType == C.TRACK_TYPE_VIDEO) { + return videoMimeTypes; + } + return ImmutableList.of(); + } + } + + @Override + public TrackToken addTrack(Format format) { + return new TrackToken() {}; + } + + @Override + public void writeSampleData( + TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {} + + @Override + public void addMetadataEntry(Metadata.Entry metadataEntry) {} + + @Override + public void close() {} +}