From 47b59f98e195f2dfa6faa297a0062882c4218629 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Thu, 2 Feb 2023 15:53:18 +0000 Subject: [PATCH] Add ImageAssetLoader and ImageConfiguration. PiperOrigin-RevId: 506619637 --- .../DefaultAssetLoaderFactory.java | 53 ++++++- .../media3/transformer/EditedMediaItem.java | 58 +++++++- .../media3/transformer/ImageAssetLoader.java | 139 ++++++++++++++++++ .../media3/transformer/SampleConsumer.java | 15 ++ 4 files changed, 259 insertions(+), 6 deletions(-) create mode 100644 libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java index 1db4e794ad..d0e6630385 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java @@ -16,18 +16,29 @@ package androidx.media3.transformer; +import static androidx.media3.common.util.Assertions.checkNotNull; + import android.content.Context; import android.os.Looper; +import androidx.annotation.Nullable; +import androidx.media3.common.MediaItem; import androidx.media3.common.util.Clock; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.source.MediaSource; +import com.google.common.collect.ImmutableList; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** The default {@link AssetLoader.Factory} implementation. */ @UnstableApi public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { - private final AssetLoader.Factory assetLoaderFactory; + private final Context context; + private final Codec.DecoderFactory decoderFactory; + private final Clock clock; + private final MediaSource.@MonotonicNonNull Factory mediaSourceFactory; + private AssetLoader.@MonotonicNonNull Factory imageAssetLoaderFactory; + private AssetLoader.@MonotonicNonNull Factory exoPlayerAssetLoaderFactory; /** * Creates an instance. * @@ -39,7 +50,10 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { */ public DefaultAssetLoaderFactory( Context context, Codec.DecoderFactory decoderFactory, Clock clock) { - assetLoaderFactory = new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock); + this.context = context; + this.decoderFactory = decoderFactory; + this.clock = clock; + this.mediaSourceFactory = null; } /** @@ -58,13 +72,42 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { Codec.DecoderFactory decoderFactory, Clock clock, MediaSource.Factory mediaSourceFactory) { - assetLoaderFactory = - new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock, mediaSourceFactory); + this.context = context; + this.decoderFactory = decoderFactory; + this.clock = clock; + this.mediaSourceFactory = mediaSourceFactory; } @Override public AssetLoader createAssetLoader( EditedMediaItem editedMediaItem, Looper looper, AssetLoader.Listener listener) { - return assetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener); + MediaItem mediaItem = editedMediaItem.mediaItem; + if (isImage(mediaItem.localConfiguration)) { + if (imageAssetLoaderFactory == null) { + imageAssetLoaderFactory = new ImageAssetLoader.Factory(); + } + return imageAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener); + } + if (exoPlayerAssetLoaderFactory == null) { + exoPlayerAssetLoaderFactory = + mediaSourceFactory != null + ? new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock, mediaSourceFactory) + : new ExoPlayerAssetLoader.Factory(context, decoderFactory, clock); + } + return exoPlayerAssetLoaderFactory.createAssetLoader(editedMediaItem, looper, listener); + } + + private static boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) { + if (localConfiguration == null) { + return false; + } + ImmutableList supportedImageTypes = ImmutableList.of(".png", ".webp", ".jpg", ".jpeg"); + String uriPath = checkNotNull(localConfiguration.uri.getPath()); + int fileExtensionStart = uriPath.lastIndexOf("."); + if (fileExtensionStart < 0) { + return false; + } + String extension = uriPath.substring(fileExtensionStart); + return supportedImageTypes.contains(extension); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java index d45d863992..872fb91818 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/EditedMediaItem.java @@ -18,6 +18,8 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkState; +import androidx.annotation.IntRange; +import androidx.media3.common.C; import androidx.media3.common.MediaItem; import androidx.media3.common.util.UnstableApi; import androidx.media3.exoplayer.source.MediaSource; @@ -36,15 +38,22 @@ public final class EditedMediaItem { private boolean removeAudio; private boolean removeVideo; private boolean flattenForSlowMotion; + private long durationUs; + private int frameRate; private Effects effects; /** * Creates an instance. * + *

For image inputs, the values passed into {@link #setRemoveAudio}, {@link #setRemoveVideo} + * and {@link #setFlattenForSlowMotion} will be ignored. + * * @param mediaItem The {@link MediaItem} on which transformations are applied. */ public Builder(MediaItem mediaItem) { this.mediaItem = mediaItem; + durationUs = C.TIME_UNSET; + frameRate = C.RATE_UNSET_INT; effects = Effects.EMPTY; } @@ -111,6 +120,38 @@ public final class EditedMediaItem { return this; } + /** + * Sets the duration of the output video in microseconds. + * + *

This should be set for inputs that don't have an implicit duration (e.g. images). It will + * be ignored for inputs that do have an implicit duration (e.g. video). + * + *

The default value is {@link C#TIME_UNSET}. + */ + @CanIgnoreReturnValue + public Builder setDurationUs(long durationUs) { + checkArgument(durationUs > 0); + this.durationUs = durationUs; + return this; + } + + /** + * Sets the frame rate of the output video in frames per second. + * + *

This should be set for inputs that don't have an implicit frame rate (e.g. images). It + * will be ignored for inputs that do have an implicit frame rate (e.g. video). + * + *

The default value is {@link C#RATE_UNSET_INT}. + */ + // TODO(b/210593170): Remove/deprecate frameRate parameter when frameRate parameter is added to + // transformer. + @CanIgnoreReturnValue + public Builder setFrameRate(@IntRange(from = 0) int frameRate) { + checkArgument(frameRate > 0); + this.frameRate = frameRate; + return this; + } + /** * Sets the {@link Effects} to apply to the {@link MediaItem}. * @@ -128,7 +169,13 @@ public final class EditedMediaItem { /** Builds an {@link EditedMediaItem} instance. */ public EditedMediaItem build() { return new EditedMediaItem( - mediaItem, removeAudio, removeVideo, flattenForSlowMotion, effects); + mediaItem, + removeAudio, + removeVideo, + flattenForSlowMotion, + durationUs, + frameRate, + effects); } } @@ -156,6 +203,11 @@ public final class EditedMediaItem { * */ public final boolean flattenForSlowMotion; + /** The duration of the image in the output video, in microseconds. */ + public final long durationUs; + /** The frame rate of the image in the output video, in frames per second. */ + @IntRange(from = 0) + public final int frameRate; /** The {@link Effects} to apply to the {@link #mediaItem}. */ public final Effects effects; @@ -164,12 +216,16 @@ public final class EditedMediaItem { boolean removeAudio, boolean removeVideo, boolean flattenForSlowMotion, + long durationUs, + int frameRate, Effects effects) { checkState(!removeAudio || !removeVideo, "Audio and video cannot both be removed"); this.mediaItem = mediaItem; this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.flattenForSlowMotion = flattenForSlowMotion; + this.durationUs = durationUs; + this.frameRate = frameRate; this.effects = effects; } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java new file mode 100644 index 0000000000..8f870bd90a --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java @@ -0,0 +1,139 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.transformer.TransformationException.ERROR_CODE_IO_UNSPECIFIED; +import static androidx.media3.transformer.TransformationException.ERROR_CODE_UNSPECIFIED; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; + +import android.graphics.Bitmap; +import android.os.Looper; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.BitmapLoader; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.effect.SimpleBitmapLoader; +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; + +/** An {@link AssetLoader} implementation that loads images into {@link Bitmap} instances. */ +@UnstableApi +public final class ImageAssetLoader implements AssetLoader { + + /** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */ + public static final class Factory implements AssetLoader.Factory { + + @Override + public AssetLoader createAssetLoader( + EditedMediaItem editedMediaItem, Looper looper, Listener listener) { + return new ImageAssetLoader(editedMediaItem, listener); + } + } + + public static final String MIME_TYPE_IMAGE_ALL = MimeTypes.BASE_TYPE_IMAGE + "/*"; + + private final EditedMediaItem editedMediaItem; + private final Listener listener; + + private @Transformer.ProgressState int progressState; + private int progress; + + private ImageAssetLoader(EditedMediaItem editedMediaItem, Listener listener) { + this.editedMediaItem = editedMediaItem; + this.listener = listener; + + progressState = PROGRESS_STATE_NOT_STARTED; + } + + @Override + public void start() { + progressState = PROGRESS_STATE_AVAILABLE; + listener.onTrackCount(1); + BitmapLoader bitmapLoader = new SimpleBitmapLoader(); + MediaItem.LocalConfiguration localConfiguration = + checkNotNull(editedMediaItem.mediaItem.localConfiguration); + ListenableFuture future = bitmapLoader.loadBitmap(localConfiguration.uri); + Futures.addCallback( + future, + new FutureCallback() { + @Override + public void onSuccess(Bitmap bitmap) { + progress = 50; + try { + Format format = + new Format.Builder() + .setHeight(bitmap.getHeight()) + .setWidth(bitmap.getWidth()) + .setSampleMimeType(MIME_TYPE_IMAGE_ALL) + .build(); + SampleConsumer sampleConsumer = + listener.onTrackAdded( + format, + SUPPORTED_OUTPUT_TYPE_DECODED, + /* streamStartPositionUs= */ 0, + /* streamOffsetUs= */ 0); + checkState(editedMediaItem.durationUs != C.TIME_UNSET); + checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT); + // TODO(b/262693274): consider using listener.onDurationUs() or the MediaItem change + // callback (when it's added) rather than setting duration here. + sampleConsumer.queueInputBitmap( + bitmap, editedMediaItem.durationUs, editedMediaItem.frameRate); + sampleConsumer.signalEndOfVideoInput(); + } catch (TransformationException e) { + listener.onTransformationError(e); + } catch (RuntimeException e) { + listener.onTransformationError( + TransformationException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED)); + } + progress = 100; + } + + @Override + public void onFailure(Throwable t) { + listener.onTransformationError( + TransformationException.createForAssetLoader(t, ERROR_CODE_IO_UNSPECIFIED)); + } + }, + MoreExecutors.directExecutor()); + } + + @Override + public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { + if (progressState == PROGRESS_STATE_AVAILABLE) { + progressHolder.progress = progress; + } + return progressState; + } + + @Override + public ImmutableMap getDecoderNames() { + return ImmutableMap.of(); + } + + @Override + public void release() { + progressState = PROGRESS_STATE_NOT_STARTED; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java index 9aa404af1a..3242f55123 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java @@ -15,6 +15,7 @@ */ package androidx.media3.transformer; +import android.graphics.Bitmap; import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; @@ -65,6 +66,20 @@ public interface SampleConsumer { throw new UnsupportedOperationException(); } + /** + * Provides an input {@link Bitmap} to the consumer. + * + *

Should only be used for image data. + * + * @param inputBitmap The {@link Bitmap} queued to the consumer. + * @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds. + * @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per + * second. + */ + default void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) { + throw new UnsupportedOperationException(); + } + // Methods to pass raw video input. /**