From 4b7cc80593a1b918f7955bf57c897de8913581cf Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 15 Jul 2024 03:27:55 -0700 Subject: [PATCH] Handle muxing with timestamps offset from zero in wrapper Sources (for example media projection) can populate the `Surface` from `SurfaceAssetLoader` with timestamps that don't start from zero. But `MuxerWrapper` assumes the latest sample timestamp can be used as the duration when it calculates average bitrate and notifies its listener. This can cause a crash because the calculated average bitrate can be zero if the denominator duration is large enough. Use the max minus first sample timestamp across tracks instead to get the duration. Side note: the large timestamps from the surface texture when using media projection arrive unchanged (apart from conversion from ns to us) in effect implementations and in the muxer wrapper (and are passed to the underlying muxer). The outputs of media3 muxer and the framework muxer are similar. PiperOrigin-RevId: 652422674 --- .../transformer/SurfaceAssetLoaderTest.java | 73 +++++++++++++++++++ .../media3/transformer/MuxerWrapper.java | 15 +++- 2 files changed, 85 insertions(+), 3 deletions(-) diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java index 3755d0b6d7..c786e31227 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java @@ -135,4 +135,77 @@ public class SurfaceAssetLoaderTest { assertThat(exportResult.height).isEqualTo(bitmap.getHeight()); assertThat(exportResult.durationMs).isEqualTo(300); } + + @Test + public void encodingFromSurface_withLargeTimestamps_succeeds() throws Exception { + assumeTrue("ImageWriter with pixel format set requires API 29", Util.SDK_INT >= 29); + + SettableFuture surfaceAssetLoaderSettableFuture = SettableFuture.create(); + SettableFuture surfaceSettableFuture = SettableFuture.create(); + Transformer transformer = + new Transformer.Builder(context) + .setAssetLoaderFactory( + new SurfaceAssetLoader.Factory( + new SurfaceAssetLoader.Callback() { + @Override + public void onSurfaceAssetLoaderCreated( + SurfaceAssetLoader surfaceAssetLoader) { + surfaceAssetLoaderSettableFuture.set(surfaceAssetLoader); + } + + @Override + public void onSurfaceReady(Surface surface, EditedMediaItem editedMediaItem) { + surfaceSettableFuture.set(surface); + } + })) + .build(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME + ":")) + .build(); + ListenableFuture exportCompletionFuture = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .runAsync(testId, editedMediaItem); + SurfaceAssetLoader surfaceAssetLoader = + surfaceAssetLoaderSettableFuture.get(TIMEOUT_MS, MILLISECONDS); + Bitmap bitmap = BitmapPixelTestUtil.readBitmap(TEST_BITMAP_PATH); + surfaceAssetLoader.setContentFormat( + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_RAW) + .setWidth(bitmap.getWidth()) + .setHeight(bitmap.getHeight()) + .setColorInfo(ColorInfo.SRGB_BT709_FULL) + .build()); + Surface surface = surfaceSettableFuture.get(TIMEOUT_MS, MILLISECONDS); + + int inputFrameCount = 10; + try (ImageWriter imageWriter = + ImageWriter.newInstance(surface, /* maxImages= */ inputFrameCount, PixelFormat.RGBA_8888)) { + ConditionVariable readyForInputCondition = new ConditionVariable(); + imageWriter.setOnImageReleasedListener( + unusedImageWriter -> readyForInputCondition.open(), new Handler(Looper.getMainLooper())); + for (int i = 0; i < inputFrameCount; i++) { + Image image = imageWriter.dequeueInputImage(); + + // Add a large base offset in nanoseconds. + image.setTimestamp(3_020_642_044_930_642L + i * C.NANOS_PER_SECOND / 30); + BitmapPixelTestUtil.copyRbga8888BitmapToImage(bitmap, image); + readyForInputCondition.close(); + imageWriter.queueInputImage(image); + // When frames are queued as fast as possible some can be dropped, so throttle input by + // blocking until the previous frame has been released by the downstream pipeline. + if (i > 0) { + assertThat(readyForInputCondition.block(TIMEOUT_MS)).isTrue(); + } + } + } + surfaceAssetLoader.signalEndOfInput(); + + ExportResult exportResult = exportCompletionFuture.get(); + assertThat(exportResult.videoFrameCount).isEqualTo(inputFrameCount); + assertThat(exportResult.width).isEqualTo(bitmap.getWidth()); + assertThat(exportResult.height).isEqualTo(bitmap.getHeight()); + assertThat(exportResult.durationMs).isEqualTo(300); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index 465d1e634a..f46fa98db2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -30,6 +30,7 @@ import static androidx.media3.effect.DebugTraceUtil.EVENT_CAN_WRITE_SAMPLE; import static androidx.media3.effect.DebugTraceUtil.EVENT_INPUT_ENDED; import static androidx.media3.effect.DebugTraceUtil.EVENT_OUTPUT_ENDED; import static java.lang.Math.max; +import static java.lang.Math.min; import static java.lang.annotation.ElementType.TYPE_USE; import static java.util.concurrent.TimeUnit.MILLISECONDS; @@ -161,6 +162,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private boolean isEnded; private @C.TrackType int previousTrackType; private long minTrackTimeUs; + private long minEndedTrackTimeUs; private long maxEndedTrackTimeUs; private @MonotonicNonNull ScheduledFuture abortScheduledFuture; private boolean isAborted; @@ -214,6 +216,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; trackTypeToInfo = new SparseArray<>(); previousTrackType = C.TRACK_TYPE_NONE; firstVideoPresentationTimeUs = C.TIME_UNSET; + minEndedTrackTimeUs = Long.MAX_VALUE; abortScheduledExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME); bufferInfo = new BufferInfo(); } @@ -562,6 +565,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return false; } + if (trackInfo.sampleCount == 0) { + trackInfo.startTimeUs = presentationTimeUs; + } trackInfo.sampleCount++; trackInfo.bytesWritten += data.remaining(); trackInfo.timeUs = max(trackInfo.timeUs, presentationTimeUs); @@ -597,6 +603,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } TrackInfo trackInfo = trackTypeToInfo.get(trackType); + minEndedTrackTimeUs = max(0, min(minEndedTrackTimeUs, trackInfo.startTimeUs)); maxEndedTrackTimeUs = max(maxEndedTrackTimeUs, trackInfo.timeUs); listener.onTrackEnded( trackType, trackInfo.format, trackInfo.getAverageBitrate(), trackInfo.sampleCount); @@ -621,10 +628,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + long durationMs = usToMs(maxEndedTrackTimeUs - minEndedTrackTimeUs); if (muxerMode == MUXER_MODE_MUX_PARTIAL && muxedPartialVideo && (muxedPartialAudio || trackCount == 1)) { - listener.onEnded(usToMs(maxEndedTrackTimeUs), getCurrentOutputSizeBytes()); + listener.onEnded(durationMs, getCurrentOutputSizeBytes()); if (abortScheduledFuture != null) { abortScheduledFuture.cancel(/* mayInterruptIfRunning= */ false); } @@ -632,7 +640,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } if (isEnded) { - listener.onEnded(usToMs(maxEndedTrackTimeUs), getCurrentOutputSizeBytes()); + listener.onEnded(durationMs, getCurrentOutputSizeBytes()); abortScheduledExecutorService.shutdownNow(); } } @@ -775,6 +783,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public final Format format; public final TrackToken trackToken; + public long startTimeUs; public long bytesWritten; public int sampleCount; public long timeUs; @@ -799,7 +808,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Util.scaleLargeTimestamp( /* timestamp= */ bytesWritten, /* multiplier= */ C.BITS_PER_BYTE * C.MICROS_PER_SECOND, - /* divisor= */ timeUs); + /* divisor= */ timeUs - startTimeUs); } } }