diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java index 815b4c369e..f3b4a68212 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecRenderer.java @@ -1099,6 +1099,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { if (codecOperatingRate <= assumedMinimumCodecOperatingRate) { codecOperatingRate = CODEC_OPERATING_RATE_UNSET; } + onReadyToInitializeCodec(inputFormat); codecInitializingTimestamp = SystemClock.elapsedRealtime(); MediaCodecAdapter.Configuration configuration = getMediaCodecConfiguration(codecInfo, inputFormat, crypto, codecOperatingRate); @@ -1383,6 +1384,22 @@ public abstract class MediaCodecRenderer extends BaseRenderer { return true; } + /** + * Called when ready to initialize the {@link MediaCodecAdapter}. + * + *

This method is called just before the renderer obtains the {@linkplain + * #getMediaCodecConfiguration configuration} for the {@link MediaCodecAdapter} and creates the + * adapter via the passed in {@link MediaCodecAdapter.Factory}. + * + *

The default implementation is a no-op. + * + * @param format The {@link Format} for which the codec is being configured. + * @throws ExoPlaybackException If an error occurs preparing for initializing the codec. + */ + protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { + // Do nothing. + } + /** * Called when a {@link MediaCodec} has been created and configured. * diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java index 2041003542..70ec7c1170 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java @@ -46,12 +46,21 @@ import androidx.annotation.DoNotInline; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.DebugViewProvider; import androidx.media3.common.DrmInitData; +import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.FrameInfo; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.FrameProcessor; import androidx.media3.common.MimeTypes; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.SurfaceInfo; import androidx.media3.common.VideoSize; import androidx.media3.common.util.Log; import androidx.media3.common.util.MediaFormatUtil; +import androidx.media3.common.util.Size; import androidx.media3.common.util.TraceUtil; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; @@ -73,8 +82,13 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecUtil; import androidx.media3.exoplayer.mediacodec.MediaCodecUtil.DecoderQueryException; import androidx.media3.exoplayer.video.VideoRendererEventListener.EventDispatcher; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.nio.ByteBuffer; +import java.util.ArrayDeque; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import org.checkerframework.checker.initialization.qual.UnderInitialization; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * Decodes and renders video using {@link MediaCodec}. @@ -86,6 +100,8 @@ import java.util.List; *

  • Message with type {@link #MSG_SET_VIDEO_OUTPUT} to set the output. The message payload * should be the target {@link Surface}, or null to clear the output. Other non-null payloads * have the effect of clearing the output. + *
  • Message with type {@link #MSG_SET_VIDEO_OUTPUT_RESOLUTION} to set the output resolution. + * The message payload should be the output resolution in {@link Size}. *
  • Message with type {@link #MSG_SET_SCALING_MODE} to set the video scaling mode. The message * payload should be one of the integer scaling modes in {@link C.VideoScalingMode}. Note that * the scaling mode only applies if the {@link Surface} targeted by this renderer is owned by @@ -128,6 +144,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { private final Context context; private final VideoFrameReleaseHelper frameReleaseHelper; private final EventDispatcher eventDispatcher; + private final FrameProcessorManager frameProcessorManager; private final long allowedJoiningTimeMs; private final int maxDroppedFramesToNotify; private final boolean deviceNeedsNoPostProcessWorkaround; @@ -331,6 +348,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { this.context = context.getApplicationContext(); frameReleaseHelper = new VideoFrameReleaseHelper(this.context); eventDispatcher = new EventDispatcher(eventHandler, eventListener); + frameProcessorManager = new FrameProcessorManager(this); deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround(); joiningDeadlineMs = C.TIME_UNSET; scalingMode = C.VIDEO_SCALING_MODE_DEFAULT; @@ -615,6 +633,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { try { super.onReset(); } finally { + if (frameProcessorManager.isEnabled()) { + frameProcessorManager.reset(); + } if (placeholderSurface != null) { releasePlaceholderSurface(); } @@ -650,6 +671,12 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } } break; + case MSG_SET_VIDEO_OUTPUT_RESOLUTION: + Size outputResolution = (Size) checkNotNull(message); + if (displaySurface != null && frameProcessorManager.isEnabled()) { + frameProcessorManager.setOutputSurfaceInfo(displaySurface, outputResolution); + } + break; case MSG_SET_AUDIO_ATTRIBUTES: case MSG_SET_AUX_EFFECT_INFO: case MSG_SET_CAMERA_MOTION_LISTENER: @@ -662,6 +689,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } private void setOutput(@Nullable Object output) throws ExoPlaybackException { + // TODO(b/238302341) Handle output surface change in previewing. // Handle unsupported (i.e., non-Surface) outputs by clearing the display surface. @Nullable Surface displaySurface = output instanceof Surface ? (Surface) output : null; @@ -756,8 +784,19 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { } displaySurface = placeholderSurface; } + + if (frameProcessorManager.isEnabled()) { + mediaFormat = frameProcessorManager.amendMediaFormatKeys(mediaFormat); + } + return MediaCodecAdapter.Configuration.createForVideoDecoding( - codecInfo, mediaFormat, format, displaySurface, crypto); + codecInfo, + mediaFormat, + format, + frameProcessorManager.isEnabled() + ? frameProcessorManager.getInputSurface() + : displaySurface, + crypto); } @Override @@ -881,6 +920,14 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return maxFrameRate == -1 ? CODEC_OPERATING_RATE_UNSET : (maxFrameRate * targetPlaybackSpeed); } + @CallSuper + @Override + protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException { + if (!frameProcessorManager.isEnabled()) { + frameProcessorManager.maybeEnable(format); + } + } + @Override protected void onCodecInitialized( String name, @@ -988,6 +1035,10 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { decodedVideoSize = new VideoSize(width, height, unappliedRotationDegrees, pixelWidthHeightRatio); frameReleaseHelper.onFormatChanged(format.frameRate); + + if (frameProcessorManager.isEnabled()) { + frameProcessorManager.setInputFrameInfo(width, height, pixelWidthHeightRatio); + } } @Override @@ -1625,6 +1676,185 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { return new MediaCodecVideoDecoderException(cause, codecInfo, displaySurface); } + /** Manages {@link FrameProcessor} interactions. */ + private static final class FrameProcessorManager { + + private static final String FRAME_PROCESSOR_FACTORY_CLASS = + "androidx.media3.effect.GlEffectsFrameProcessor$Factory"; + + // TODO(b/238302341) Consider removing the reference to the containing class and make this class + // non-static. + private final MediaCodecVideoRenderer renderer; + private final ArrayDeque processedFrames; + + private @MonotonicNonNull Handler handler; + @Nullable private FrameProcessor frameProcessor; + @Nullable private CopyOnWriteArrayList videoEffects; + private boolean canEnableFrameProcessing; + + /** Creates a new instance. */ + public FrameProcessorManager(@UnderInitialization MediaCodecVideoRenderer renderer) { + this.renderer = renderer; + processedFrames = new ArrayDeque<>(); + canEnableFrameProcessing = true; + } + + /** Sets the {@linkplain Effect video effects}. */ + public void setVideoEffects(List videoEffects) { + if (this.videoEffects == null) { + this.videoEffects = new CopyOnWriteArrayList<>(videoEffects); + return; + } + this.videoEffects.clear(); + this.videoEffects.addAll(videoEffects); + } + + /** Returns whether frame processing is enabled. */ + public boolean isEnabled() { + return frameProcessor != null; + } + + /** + * Tries to enable frame processing. + * + *

    Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before + * calling this method. + * + * @param inputFormat The {@link Format} that is input into the {@link FrameProcessor}. + * @return Whether frame processing is enabled. + * @throws ExoPlaybackException When enabling the {@link FrameProcessor} failed. + */ + @CanIgnoreReturnValue + public boolean maybeEnable(Format inputFormat) throws ExoPlaybackException { + checkState(!isEnabled()); + if (!canEnableFrameProcessing) { + return false; + } + if (videoEffects == null) { + canEnableFrameProcessing = false; + return false; + } + + // Playback thread handler. + handler = Util.createHandlerForCurrentLooper(); + try { + frameProcessor = + ((FrameProcessor.Factory) + Class.forName(FRAME_PROCESSOR_FACTORY_CLASS).getConstructor().newInstance()) + .create( + renderer.context, + checkNotNull(videoEffects), + DebugViewProvider.NONE, + inputFormat.colorInfo != null + ? inputFormat.colorInfo + : ColorInfo.SDR_BT709_LIMITED, + /* outputColorInfo= */ ColorInfo.SDR_BT709_LIMITED, + /* releaseFramesAutomatically= */ false, + /* executor= */ handler::post, + new FrameProcessor.Listener() { + @Override + public void onOutputSizeChanged(int width, int height) { + // TODO(b/238302341) Handle output size change. + } + + @Override + public void onOutputFrameAvailable(long presentationTimeUs) { + processedFrames.add(presentationTimeUs); + } + + @Override + public void onFrameProcessingError(FrameProcessingException exception) { + renderer.setPendingPlaybackException( + renderer.createRendererException( + exception, + inputFormat, + // TODO(b/238302341) Add relevant error codes for frame processing. + PlaybackException.ERROR_CODE_UNSPECIFIED)); + } + + @Override + public void onFrameProcessingEnded() { + throw new IllegalStateException(); + } + }); + } catch (Exception e) { + throw renderer.createRendererException( + e, inputFormat, PlaybackException.ERROR_CODE_UNSPECIFIED); + } + setInputFrameInfo(inputFormat.width, inputFormat.height, inputFormat.pixelWidthHeightRatio); + return true; + } + + /** + * Returns the {@linkplain FrameProcessor#getInputSurface input surface} of the {@link + * FrameProcessor}. + * + *

    Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling + * this method. + */ + public Surface getInputSurface() { + return checkNotNull(frameProcessor).getInputSurface(); + } + + /** + * Sets the output surface info. + * + *

    Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling + * this method. + * + * @param outputSurface The {@link Surface} to which {@link FrameProcessor} outputs. + * @param outputResolution The {@link Size} of the output resolution. + */ + public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) { + checkNotNull(frameProcessor) + .setOutputSurfaceInfo( + new SurfaceInfo( + outputSurface, outputResolution.getWidth(), outputResolution.getHeight())); + } + + /** + * Sets the input surface info. + * + *

    Caller must ensure the {@code FrameProcessorManager} {@link #isEnabled()} before calling + * this method. + */ + public void setInputFrameInfo(int width, int height, float pixelWidthHeightRatio) { + checkNotNull(frameProcessor) + .setInputFrameInfo( + new FrameInfo( + width, height, pixelWidthHeightRatio, renderer.getOutputStreamOffsetUs())); + } + + /** Sets the necessary {@link MediaFormat} keys for frame processing. */ + @SuppressWarnings("InlinedApi") + public MediaFormat amendMediaFormatKeys(MediaFormat mediaFormat) { + if (Util.SDK_INT >= 29 + && renderer.context.getApplicationContext().getApplicationInfo().targetSdkVersion >= 29) { + mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); + } + return mediaFormat; + } + + /** + * Releases the resources. + * + *

    Caller must ensure frame processing {@linkplain #isEnabled() is not enabled} before + * calling this method. + */ + public void reset() { + checkNotNull(frameProcessor).release(); + frameProcessor = null; + if (handler != null) { + handler.removeCallbacksAndMessages(/* token= */ null); + } + if (videoEffects != null) { + videoEffects.clear(); + } + processedFrames.clear(); + canEnableFrameProcessing = true; + } + } + /** * Returns a maximum video size to use when configuring a codec for {@code format} in a way that * will allow possible adaptation to other compatible formats that are expected to have the same