Initial Frame Extractor Transformer Factory

Package-private until API is more useable.

Similar to frame analyzer mode: uses ImageReader instead of an encoder,
and no muxer.

PiperOrigin-RevId: 658446675

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -1567,6 +1567,41 @@ public class TransformerEndToEndTest {
     assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
   }
 
+  @Test
+  public void extractFrames_completesSuccessfully() throws Exception {
+    assumeFormatsSupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat,
+        /* outputFormat= */ null);
+    AtomicInteger imagesOutput = new AtomicInteger(/* initialValue= */ 0);
+    Transformer transformer =
+        ExperimentalFrameExtractorFactory.buildFrameExtractorTransformer(
+            context, image -> imagesOutput.incrementAndGet());
+    AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0);
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(
+                MediaItem.fromUri(
+                    Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri)))
+            .setRemoveAudio(true)
+            .setEffects(
+                new Effects(
+                    /* audioProcessors= */ ImmutableList.of(),
+                    ImmutableList.of(createFrameCountingEffect(videoFramesSeen))))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(videoFramesSeen.get()).isEqualTo(932);
+    assertThat(imagesOutput.get()).isEqualTo(932);
+    assertThat(result.exportResult.videoFrameCount).isEqualTo(932);
+    // Confirm no data was written to file.
+    assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
   @Test
   public void transcode_withOutputVideoMimeTypeAv1_completesSuccessfully() throws Exception {
     assumeFormatsSupported(

libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java

@@ -19,18 +19,15 @@ package androidx.media3.transformer;
 import static androidx.media3.common.util.Assertions.checkState;
 
 import android.content.Context;
-import android.media.MediaCodec;
 import android.media.MediaCodec.BufferInfo;
 import android.view.Surface;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
-import androidx.media3.common.Metadata;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.decoder.DecoderInputBuffer;
 import androidx.media3.exoplayer.video.PlaceholderSurface;
-import androidx.media3.muxer.Muxer;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
@@ -194,57 +191,4 @@ public final class ExperimentalAnalyzerModeFactory {
     @Override
     public void release() {}
   }
-
-  /** A {@link Muxer} implementation that does nothing. */
-  private static final class NoWriteMuxer implements Muxer {
-    public static final class Factory implements Muxer.Factory {
-
-      private final ImmutableList<String> audioMimeTypes;
-      private final ImmutableList<String> videoMimeTypes;
-
-      /**
-       * Creates an instance.
-       *
-       * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link
-       *     #getSupportedSampleMimeTypes(int)}.
-       * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link
-       *     #getSupportedSampleMimeTypes(int)}.
-       */
-      public Factory(ImmutableList<String> audioMimeTypes, ImmutableList<String> videoMimeTypes) {
-        this.audioMimeTypes = audioMimeTypes;
-        this.videoMimeTypes = videoMimeTypes;
-      }
-
-      @Override
-      public Muxer create(String path) {
-        return new NoWriteMuxer();
-      }
-
-      @Override
-      public ImmutableList<String> getSupportedSampleMimeTypes(@C.TrackType int trackType) {
-        if (trackType == C.TRACK_TYPE_AUDIO) {
-          return audioMimeTypes;
-        }
-        if (trackType == C.TRACK_TYPE_VIDEO) {
-          return videoMimeTypes;
-        }
-        return ImmutableList.of();
-      }
-    }
-
-    @Override
-    public TrackToken addTrack(Format format) {
-      return new TrackToken() {};
-    }
-
-    @Override
-    public void writeSampleData(
-        TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {}
-
-    @Override
-    public void addMetadataEntry(Metadata.Entry metadataEntry) {}
-
-    @Override
-    public void close() {}
-  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalFrameExtractorFactory.java

@@ -0,0 +1,217 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import android.content.Context;
+import android.graphics.PixelFormat;
+import android.media.Image;
+import android.media.ImageReader;
+import android.media.MediaCodec.BufferInfo;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.Util;
+import androidx.media3.decoder.DecoderInputBuffer;
+import com.google.common.collect.ImmutableList;
+import java.nio.ByteBuffer;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
+
+/**
+ * Factory for creating instances of {@link Transformer} that can be used to extract frames.
+ *
+ * <p>This class is experimental and will be renamed or removed in a future release.
+ */
+/* package */ final class ExperimentalFrameExtractorFactory {
+
+  private ExperimentalFrameExtractorFactory() {}
+
+  /** A callback to be notified when a new image is available. */
+  public interface Listener {
+
+    // TODO: b/350498258 - Make this more user-friendly before making it a public API.
+    /**
+     * Called when a new {@link Image} is available. When this method returns, the {@link Image}
+     * will be closed and can no longer be used.
+     */
+    void onImageAvailable(Image image);
+  }
+
+  /**
+   * Builds a {@link Transformer} that runs as an analyzer.
+   *
+   * <p>No encoding or muxing is performed, therefore no data is written to any output files.
+   *
+   * @param context The {@link Context}.
+   * @param listener The {@link Listener} to be used for generated images.
+   * @return The fame extracting {@link Transformer}.
+   */
+  public static Transformer buildFrameExtractorTransformer(Context context, Listener listener) {
+    return new Transformer.Builder(context)
+        .experimentalSetTrimOptimizationEnabled(false)
+        .setEncoderFactory(new ImageReaderEncoder.Factory(listener))
+        .setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET)
+        .setMuxerFactory(
+            new NoWriteMuxer.Factory(
+                /* audioMimeTypes= */ ImmutableList.of(MimeTypes.AUDIO_AAC),
+                /* videoMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264)))
+        .setAudioMimeType(MimeTypes.AUDIO_AAC)
+        .setVideoMimeType(MimeTypes.VIDEO_H264)
+        .experimentalSetMaxFramesInEncoder(1) // Work around ImageReader frame dropping.
+        .build();
+  }
+
+  /** A {@linkplain Codec encoder} implementation that outputs frames via {@link ImageReader}. */
+  private static final class ImageReaderEncoder implements Codec {
+    public static final class Factory implements Codec.EncoderFactory {
+
+      private final Listener listener;
+
+      public Factory(Listener listener) {
+        this.listener = listener;
+      }
+
+      @Override
+      public Codec createForAudioEncoding(Format format) {
+        throw new UnsupportedOperationException();
+      }
+
+      @Override
+      public Codec createForVideoEncoding(Format format) {
+        return new ImageReaderEncoder(format, listener);
+      }
+    }
+
+    private static final String TAG = "ImageReaderEncoder";
+    private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.allocateDirect(0);
+
+    private final Format configurationFormat;
+    private final ImageReader imageReader;
+    private final Queue<Long> processedImageTimestampsNs;
+    private final BufferInfo outputBufferInfo;
+
+    private boolean hasOutputBuffer;
+    private boolean inputStreamEnded;
+
+    public ImageReaderEncoder(Format format, Listener listener) {
+      this.configurationFormat = format;
+      imageReader =
+          ImageReader.newInstance(
+              format.width, format.height, PixelFormat.RGBA_8888, /* maxImages= */ 1);
+
+      processedImageTimestampsNs = new ConcurrentLinkedQueue<>();
+
+      imageReader.setOnImageAvailableListener(
+          reader -> {
+            try (Image image = reader.acquireNextImage()) {
+              processedImageTimestampsNs.add(image.getTimestamp());
+              listener.onImageAvailable(image);
+            }
+          },
+          Util.createHandlerForCurrentOrMainLooper());
+
+      outputBufferInfo = new BufferInfo();
+    }
+
+    @Override
+    public String getName() {
+      return TAG;
+    }
+
+    @Override
+    public Format getConfigurationFormat() {
+      return configurationFormat;
+    }
+
+    @Override
+    public Surface getInputSurface() {
+      return imageReader.getSurface();
+    }
+
+    @Override
+    @EnsuresNonNullIf(expression = "#1.data", result = true)
+    public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void queueInputBuffer(DecoderInputBuffer inputBuffer) {
+      throw new UnsupportedOperationException();
+    }
+
+    @Override
+    public void signalEndOfInputStream() {
+      inputStreamEnded = true;
+    }
+
+    @Override
+    public Format getOutputFormat() {
+      return configurationFormat;
+    }
+
+    @Override
+    @Nullable
+    public ByteBuffer getOutputBuffer() {
+      return maybeGenerateOutputBuffer() ? EMPTY_BUFFER : null;
+    }
+
+    @Override
+    @Nullable
+    public BufferInfo getOutputBufferInfo() {
+      return maybeGenerateOutputBuffer() ? outputBufferInfo : null;
+    }
+
+    @Override
+    public boolean isEnded() {
+      return inputStreamEnded && processedImageTimestampsNs.isEmpty();
+    }
+
+    @Override
+    public void releaseOutputBuffer(boolean render) {
+      releaseOutputBuffer();
+    }
+
+    @Override
+    public void releaseOutputBuffer(long renderPresentationTimeUs) {
+      releaseOutputBuffer();
+    }
+
+    private void releaseOutputBuffer() {
+      hasOutputBuffer = false;
+    }
+
+    @Override
+    public void release() {}
+
+    private boolean maybeGenerateOutputBuffer() {
+      if (hasOutputBuffer) {
+        return true;
+      }
+      Long timeNs = processedImageTimestampsNs.poll();
+      if (timeNs == null) {
+        return false;
+      }
+
+      hasOutputBuffer = true;
+      outputBufferInfo.presentationTimeUs = timeNs / 1000;
+      return true;
+    }
+  }
+}

libraries/transformer/src/main/java/androidx/media3/transformer/NoWriteMuxer.java

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import android.media.MediaCodec;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Metadata;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.muxer.Muxer;
+import com.google.common.collect.ImmutableList;
+import java.nio.ByteBuffer;
+
+/** A {@link Muxer} implementation that does nothing. */
+/* package */ final class NoWriteMuxer implements Muxer {
+  public static final class Factory implements Muxer.Factory {
+
+    private final ImmutableList<String> audioMimeTypes;
+    private final ImmutableList<String> videoMimeTypes;
+
+    /**
+     * Creates an instance.
+     *
+     * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link
+     *     #getSupportedSampleMimeTypes(int)}.
+     * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link
+     *     #getSupportedSampleMimeTypes(int)}.
+     */
+    public Factory(ImmutableList<String> audioMimeTypes, ImmutableList<String> videoMimeTypes) {
+      this.audioMimeTypes = audioMimeTypes;
+      this.videoMimeTypes = videoMimeTypes;
+    }
+
+    @Override
+    public Muxer create(String path) {
+      return new NoWriteMuxer();
+    }
+
+    @Override
+    public ImmutableList<String> getSupportedSampleMimeTypes(@C.TrackType int trackType) {
+      if (trackType == C.TRACK_TYPE_AUDIO) {
+        return audioMimeTypes;
+      }
+      if (trackType == C.TRACK_TYPE_VIDEO) {
+        return videoMimeTypes;
+      }
+      return ImmutableList.of();
+    }
+  }
+
+  @Override
+  public TrackToken addTrack(Format format) {
+    return new TrackToken() {};
+  }
+
+  @Override
+  public void writeSampleData(
+      TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {}
+
+  @Override
+  public void addMetadataEntry(Metadata.Entry metadataEntry) {}
+
+  @Override
+  public void close() {}
+}