This method blocks until the frame has been rendered to the {@code surface}. * * @param assetFilePath The path to the file in the asset directory. * @param listener A {@link Listener} implementation. - * @param surface The {@link Surface} to render the decoded frame to, {@code null} if the decoded - * frame is not needed. - * @throws IOException If the {@link MediaExtractor} or {@link MediaCodec} cannot be created. - */ - public static void decodeOneAssetFileFrame( - String assetFilePath, Listener listener, @Nullable Surface surface) throws IOException { - MediaExtractor mediaExtractor = new MediaExtractor(); - Context context = getApplicationContext(); - - try (AssetFileDescriptor afd = context.getAssets().openFd(assetFilePath)) { - mediaExtractor.setDataSource(afd.getFileDescriptor(), afd.getStartOffset(), afd.getLength()); - if (surface == null) { - decodeOneVideoFrame(mediaExtractor, listener); - } else { - decodeOneVideoFrame(mediaExtractor, listener, surface); - } - } finally { - mediaExtractor.release(); - } - } - - /** - * Reads and decodes one video frame from the {@code mediaExtractor} and renders it to the {@code - * surface}. - * - *
A placeholder surface is used. - * - * @param mediaExtractor The {@link MediaExtractor} with a {@link - * MediaExtractor#setDataSource(String) data source set}. - * @param listener A {@link Listener} implementation. - * @throws UnsupportedOperationException If there is no supported {@linkplain MediaCodec decoders} - * available. - * @throws IOException If the {@link MediaExtractor} or {@link MediaCodec} cannot be created. - */ - private static void decodeOneVideoFrame(MediaExtractor mediaExtractor, Listener listener) - throws IOException { - @Nullable SurfaceTexture placeholderSurfaceTexture = null; - @Nullable Surface placeholderSurface = null; - try { - placeholderSurfaceTexture = new SurfaceTexture(/* texName= */ 0); - placeholderSurface = new Surface(placeholderSurfaceTexture); - decodeOneVideoFrame(mediaExtractor, listener, placeholderSurface); - } finally { - if (placeholderSurfaceTexture != null) { - placeholderSurfaceTexture.release(); - } - if (placeholderSurface != null) { - placeholderSurface.release(); - } - } - } - - /** - * Reads and decodes one video frame from the {@code mediaExtractor} and renders it to the {@code - * surface}. - * - * @param mediaExtractor The {@link MediaExtractor} with a {@link - * MediaExtractor#setDataSource(String) data source set}. - * @param listener A {@link Listener} implementation. * @param surface The {@link Surface} to render the decoded frame to. - * @throws IOException If the {@link MediaCodec} cannot be created. - * @throws UnsupportedOperationException If there is no supported {@linkplain MediaCodec decoders} - * available. */ - private static void decodeOneVideoFrame( - MediaExtractor mediaExtractor, Listener listener, Surface surface) throws IOException { - @Nullable MediaFormat mediaFormat = null; - @Nullable MediaCodec mediaCodec = null; + @SuppressWarnings("CatchingUnchecked") + public static void decodeOneAssetFileFrame( + String assetFilePath, Listener listener, Surface surface) throws Exception { + Context context = getApplicationContext(); + AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>(); + AtomicReference<@NullableType PlaybackException> playbackExceptionReference = + new AtomicReference<>(); + ConditionVariable firstFrameRenderedOrError = new ConditionVariable(); - try { - for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { - if (MimeTypes.isVideo(mediaExtractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME))) { - mediaFormat = mediaExtractor.getTrackFormat(i); - listener.onContainerExtracted(checkNotNull(mediaFormat)); - mediaExtractor.selectTrack(i); - break; - } - } + ExoPlayer exoPlayer = new ExoPlayer.Builder(context).build(); + Handler handler = new Handler(exoPlayer.getApplicationLooper()); + AnalyticsListener analyticsListener = + new AnalyticsListener() { + @Override + public void onVideoInputFormatChanged( + EventTime eventTime, + Format format, + @Nullable DecoderReuseEvaluation decoderReuseEvaluation) { + listener.onContainerExtracted(createMediaFormatFromFormat(format)); + } - @Nullable String decoderName = getSupportedDecoderName(checkStateNotNull(mediaFormat)); - if (decoderName == null) { - throw new UnsupportedOperationException(NO_DECODER_SUPPORT_ERROR_STRING); - } - mediaCodec = MediaCodec.createByCodecName(decoderName); + @Override + public void onRenderedFirstFrame(EventTime eventTime, Object output, long renderTimeMs) { + listener.onFrameDecoded( + createMediaFormatFromFormat(checkNotNull(exoPlayer.getVideoFormat()))); + firstFrameRenderedOrError.open(); + } - // Queue the first video frame from the extractor. - mediaCodec.configure(mediaFormat, surface, /* crypto= */ null, /* flags= */ 0); - mediaCodec.start(); - int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); - assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - ByteBuffer inputBuffer = checkNotNull(mediaCodec.getInputBuffers()[inputBufferIndex]); - int sampleSize = mediaExtractor.readSampleData(inputBuffer, /* offset= */ 0); - mediaCodec.queueInputBuffer( - inputBufferIndex, - /* offset= */ 0, - sampleSize, - mediaExtractor.getSampleTime(), - mediaExtractor.getSampleFlags()); + @Override + public void onEvents(Player player, Events events) { + if (events.contains(EVENT_PLAYER_ERROR)) { + playbackExceptionReference.set(checkNotNull(player.getPlayerError())); + firstFrameRenderedOrError.open(); + } + } + }; - // Queue an end-of-stream buffer to force the codec to produce output. - inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US); - assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - mediaCodec.queueInputBuffer( - inputBufferIndex, - /* offset= */ 0, - /* size= */ 0, - /* presentationTimeUs= */ 0, - MediaCodec.BUFFER_FLAG_END_OF_STREAM); + handler.post( + () -> { + try { + exoPlayer.setVideoSurface(surface); + exoPlayer.addAnalyticsListener(analyticsListener); + exoPlayer.setMediaItem(MediaItem.fromUri(buildAssetUri(assetFilePath))); + exoPlayer.setPlayWhenReady(false); + exoPlayer.prepare(); + // Catch all exceptions to report. Exceptions thrown here and not caught will not + // propagate. + } catch (Exception e) { + unexpectedExceptionReference.set(e); + firstFrameRenderedOrError.open(); + } + }); - // Dequeue and render the output video frame. - MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); - int outputBufferIndex; - boolean decoderFormatRead = false; - do { - outputBufferIndex = mediaCodec.dequeueOutputBuffer(bufferInfo, DEQUEUE_TIMEOUT_US); - if (!decoderFormatRead && outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - listener.onFrameDecoded(mediaCodec.getOutputFormat()); - decoderFormatRead = true; - } - assertThat(outputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER); - } while (outputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED - || outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED); - mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ true); - } finally { - if (mediaCodec != null) { - mediaCodec.release(); - } + if (!firstFrameRenderedOrError.block(TIMEOUT_MS)) { + throw new TimeoutException( + "DecodeOneFrameUtil timed out after " + TIMEOUT_MS + " milliseconds."); } - } - - /** - * Returns the name of a decoder that supports this {@link MediaFormat}. - * - *
Capability check is similar to - * androidx.media3.transformer.EncoderUtil.java#findCodecForFormat(). - */ - @Nullable - private static String getSupportedDecoderName(MediaFormat format) { - // TODO(b/266923205): De-duplicate logic from EncoderUtil.java#findCodecForFormat(). - MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.REGULAR_CODECS); - // Format must not include KEY_FRAME_RATE on API21. - // https://developer.android.com/reference/android/media/MediaCodecList#findDecoderForFormat(android.media.MediaFormat) - float frameRate = Format.NO_VALUE; - if (Util.SDK_INT == 21 && format.containsKey(MediaFormat.KEY_FRAME_RATE)) { - try { - frameRate = format.getFloat(MediaFormat.KEY_FRAME_RATE); - } catch (ClassCastException e) { - frameRate = format.getInteger(MediaFormat.KEY_FRAME_RATE); - } - // Clears the frame rate field. - format.setString(MediaFormat.KEY_FRAME_RATE, null); + handler.post(exoPlayer::release); + @Nullable PlaybackException playbackException = playbackExceptionReference.get(); + if (playbackException != null) { + throw playbackException; } - - @Nullable String mediaCodecName = mediaCodecList.findDecoderForFormat(format); - - if (Util.SDK_INT == 21) { - MediaFormatUtil.maybeSetInteger(format, MediaFormat.KEY_FRAME_RATE, round(frameRate)); + @Nullable Exception unexpectedException = unexpectedExceptionReference.get(); + if (unexpectedException != null) { + throw new IllegalStateException( + "Unexpected exception starting the player.", unexpectedException); } - return mediaCodecName; } private DecodeOneFrameUtil() {} diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java index dfb1ef9c1e..5b52e74cba 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java @@ -19,10 +19,8 @@ import static androidx.media3.common.MimeTypes.VIDEO_H265; import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL; import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888; import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap; -import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import static androidx.media3.transformer.mh.HdrCapabilitiesUtil.assumeDeviceSupportsOpenGlToneMapping; import static androidx.media3.transformer.mh.UnoptimizedGlEffect.NO_OP_EFFECT; -import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import android.graphics.Bitmap; @@ -32,7 +30,6 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.test.utils.DecodeOneFrameUtil; import androidx.media3.test.utils.VideoFrameProcessorTestRunner; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.base.Ascii; @@ -146,22 +143,8 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .build(); Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); - Bitmap actualBitmap; - try { - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); - } catch (UnsupportedOperationException e) { - if (e.getMessage() != null - && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { - recordTestSkipped( - getApplicationContext(), - testId, - /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); - return; - } else { - throw e; - } - } + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); Log.i(TAG, "Successfully tone mapped."); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -182,22 +165,8 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .build(); Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); - Bitmap actualBitmap; - try { - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); - } catch (UnsupportedOperationException e) { - if (e.getMessage() != null - && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { - recordTestSkipped( - getApplicationContext(), - testId, - /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); - return; - } else { - throw e; - } - } + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); Log.i(TAG, "Successfully tone mapped."); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -218,22 +187,8 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .build(); Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH); - Bitmap actualBitmap; - try { - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); - } catch (UnsupportedOperationException e) { - if (e.getMessage() != null - && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { - recordTestSkipped( - getApplicationContext(), - testId, - /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); - return; - } else { - throw e; - } - } + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); Log.i(TAG, "Successfully tone mapped."); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -255,22 +210,8 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .build(); Bitmap expectedBitmap = readBitmap(TONE_MAP_PQ_TO_SDR_PNG_ASSET_PATH); - Bitmap actualBitmap; - try { - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); - } catch (UnsupportedOperationException e) { - if (e.getMessage() != null - && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { - recordTestSkipped( - getApplicationContext(), - testId, - /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); - return; - } else { - throw e; - } - } + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); Log.i(TAG, "Successfully tone mapped."); // TODO(b/207848601): Switch to using proper tooling for testing against golden data. @@ -295,22 +236,8 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest { .build(); Bitmap expectedBitmap = readBitmap(TONE_MAP_HLG_TO_SDR_PNG_ASSET_PATH); - Bitmap actualBitmap; - try { - videoFrameProcessorTestRunner.processFirstFrameAndEnd(); - actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); - } catch (UnsupportedOperationException e) { - if (e.getMessage() != null - && e.getMessage().equals(DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING)) { - recordTestSkipped( - getApplicationContext(), - testId, - /* reason= */ DecodeOneFrameUtil.NO_DECODER_SUPPORT_ERROR_STRING); - return; - } else { - throw e; - } - } + videoFrameProcessorTestRunner.processFirstFrameAndEnd(); + Bitmap actualBitmap = videoFrameProcessorTestRunner.getOutputBitmap(); Log.i(TAG, "Successfully tone mapped."); // TODO(b/207848601): Switch to using proper tooling for testing against golden data.