From 541460a01debb852d37b8b3a4b9ec57e47944dc6 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Fri, 27 May 2022 10:22:04 +0000 Subject: [PATCH] Fix handling clipping in transformer renderers. Decode-only video frames (needed when the frame at / first frame after the clipping start is not a key frame) need to be decoded but not passed to the frame processor chain or encoder. The clipping start offset needs to be removed from the frame timestamps in the passthrough and video pipelines. There are no changes needed for this in the audio pipeline, as it doesn't use the input timestamps -- it uses its own timestamps derived from the buffer sizes instead. Also add demo option to try this out. #minor-release PiperOrigin-RevId: 451353609 --- .../ConfigurationActivity.java | 37 ++++++++++++++ .../transformerdemo/TransformerActivity.java | 21 +++++++- .../res/layout/configuration_activity.xml | 10 ++++ .../src/main/res/layout/trim_options.xml | 48 +++++++++++++++++++ .../src/main/res/values/strings.xml | 2 + .../TransformerAndroidTestRunner.java | 25 ++++++---- .../transformer/TransformerEndToEndTest.java | 33 ++++++++++++- .../RepeatedTranscodeTransformationTest.java | 8 ++-- .../mh/SetFrameEditTransformationTest.java | 5 +- .../transformer/mh/TranscodeQualityTest.java | 17 +++++-- .../transformer/mh/TransformationTest.java | 16 ++++--- .../mh/analysis/BitrateAnalysisTest.java | 4 +- .../EncoderPerformanceAnalysisTest.java | 3 +- .../PassthroughSamplePipeline.java | 4 ++ .../transformer/TransformationRequest.java | 3 ++ .../exoplayer2/transformer/Transformer.java | 7 +++ .../transformer/TransformerAudioRenderer.java | 3 +- .../transformer/TransformerBaseRenderer.java | 2 + .../transformer/TransformerVideoRenderer.java | 8 +++- .../VideoTranscodingSamplePipeline.java | 35 ++++++++++++-- 20 files changed, 257 insertions(+), 34 deletions(-) create mode 100644 demos/transformer/src/main/res/layout/trim_options.xml diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java index af3412dd1c..d659738ebf 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/ConfigurationActivity.java @@ -32,6 +32,7 @@ import android.widget.TextView; import androidx.annotation.Nullable; import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.android.material.slider.RangeSlider; @@ -55,6 +56,8 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; + public static final String TRIM_START_MS = "trim_start_ms"; + public static final String TRIM_END_MS = "trim_end_ms"; public static final String ENABLE_FALLBACK = "enable_fallback"; public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; @@ -115,12 +118,15 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner resolutionHeightSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; + private @MonotonicNonNull CheckBox trimCheckBox; private @MonotonicNonNull CheckBox enableFallbackCheckBox; private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; private @MonotonicNonNull Button selectDemoEffectsButton; private boolean @MonotonicNonNull [] demoEffectsSelections; private int inputUriPosition; + private long trimStartMs; + private long trimEndMs; private float periodicVignetteCenterX; private float periodicVignetteCenterY; private float periodicVignetteInnerRadius; @@ -188,6 +194,11 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateSpinner.setAdapter(rotateAdapter); rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); + trimCheckBox = findViewById(R.id.trim_checkbox); + trimCheckBox.setOnCheckedChangeListener(this::selectTrimBounds); + trimStartMs = C.TIME_UNSET; + trimEndMs = C.TIME_UNSET; + enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox); enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox); enableRequestSdrToneMappingCheckBox.setEnabled(isRequestSdrToneMappingSupported()); @@ -224,6 +235,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "trimCheckBox", "enableFallbackCheckBox", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", @@ -258,6 +270,10 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedRotate)) { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } + if (trimCheckBox.isChecked()) { + bundle.putLong(TRIM_START_MS, trimStartMs); + bundle.putLong(TRIM_END_MS, trimEndMs); + } bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked()); bundle.putBoolean( ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); @@ -295,6 +311,27 @@ public final class ConfigurationActivity extends AppCompatActivity { .show(); } + private void selectTrimBounds(View view, boolean isChecked) { + if (!isChecked) { + return; + } + View dialogView = getLayoutInflater().inflate(R.layout.trim_options, /* root= */ null); + RangeSlider radiusRangeSlider = + checkNotNull(dialogView.findViewById(R.id.trim_bounds_range_slider)); + radiusRangeSlider.setValues(0f, 60f); // seconds + new AlertDialog.Builder(/* context= */ this) + .setView(dialogView) + .setPositiveButton( + android.R.string.ok, + (DialogInterface dialogInterface, int i) -> { + List radiusRange = radiusRangeSlider.getValues(); + trimStartMs = 1000 * radiusRange.get(0).longValue(); + trimEndMs = 1000 * radiusRange.get(1).longValue(); + }) + .create() + .show(); + } + @RequiresNonNull("selectedFileTextView") private void selectFileInDialog(DialogInterface dialog, int which) { inputUriPosition = which; diff --git a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java index 2da0aea3dc..2e4d651201 100644 --- a/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java +++ b/demos/transformer/src/main/java/com/google/android/exoplayer2/transformerdemo/TransformerActivity.java @@ -152,9 +152,10 @@ public final class TransformerActivity extends AppCompatActivity { externalCacheFile = createExternalCacheFile("transformer-output.mp4"); String filePath = externalCacheFile.getAbsolutePath(); @Nullable Bundle bundle = intent.getExtras(); + MediaItem mediaItem = createMediaItem(bundle, uri); Transformer transformer = createTransformer(bundle, filePath); transformationStopwatch.start(); - transformer.startTransformation(MediaItem.fromUri(uri), filePath); + transformer.startTransformation(mediaItem, filePath); this.transformer = transformer; } catch (IOException e) { throw new IllegalStateException(e); @@ -181,6 +182,24 @@ public final class TransformerActivity extends AppCompatActivity { }); } + private MediaItem createMediaItem(@Nullable Bundle bundle, Uri uri) { + MediaItem.Builder mediaItemBuilder = new MediaItem.Builder().setUri(uri); + if (bundle != null) { + long trimStartMs = + bundle.getLong(ConfigurationActivity.TRIM_START_MS, /* defaultValue= */ C.TIME_UNSET); + long trimEndMs = + bundle.getLong(ConfigurationActivity.TRIM_END_MS, /* defaultValue= */ C.TIME_UNSET); + if (trimStartMs != C.TIME_UNSET && trimEndMs != C.TIME_UNSET) { + mediaItemBuilder.setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(trimStartMs) + .setEndPositionMs(trimEndMs) + .build()); + } + } + return mediaItemBuilder.build(); + } + // Create a cache file, resetting it if it already exists. private File createExternalCacheFile(String fileName) throws IOException { File file = new File(getExternalCacheDir(), fileName); diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 3af465719a..2879d6a637 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -159,6 +159,16 @@ android:layout_gravity="right|center_vertical" android:gravity="right" /> + + + + diff --git a/demos/transformer/src/main/res/layout/trim_options.xml b/demos/transformer/src/main/res/layout/trim_options.xml new file mode 100644 index 0000000000..cf2de0f310 --- /dev/null +++ b/demos/transformer/src/main/res/layout/trim_options.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 98dc42ecb8..50ac310080 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -27,6 +27,7 @@ Scale video Rotate video (degrees) Enable fallback + Trim Request SDR tone-mapping (API 31+) [Experimental] HDR editing Add demo effects @@ -42,4 +43,5 @@ Center X Center Y Radius range + Bounds in seconds diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java index 69a043584c..583ec3c3b2 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerAndroidTestRunner.java @@ -19,7 +19,6 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static java.util.concurrent.TimeUnit.SECONDS; import android.content.Context; -import android.net.Uri; import android.view.Surface; import androidx.annotation.Nullable; import androidx.test.platform.app.InstrumentationRegistry; @@ -179,17 +178,17 @@ public class TransformerAndroidTestRunner { * cache. * * @param testId A unique identifier for the transformer test run. - * @param uriString The uri (as a {@link String}) of the file to transform. + * @param mediaItem The {@link MediaItem} to transform. * @return The {@link TransformationTestResult}. * @throws Exception The cause of the transformation not completing. */ - public TransformationTestResult run(String testId, String uriString) throws Exception { + public TransformationTestResult run(String testId, MediaItem mediaItem) throws Exception { JSONObject resultJson = new JSONObject(); if (inputValues != null) { resultJson.put("inputValues", JSONObject.wrap(inputValues)); } try { - TransformationTestResult transformationTestResult = runInternal(testId, uriString); + TransformationTestResult transformationTestResult = runInternal(testId, mediaItem); resultJson.put("transformationResult", transformationTestResult.asJsonObject()); if (!suppressAnalysisExceptions && transformationTestResult.analysisException != null) { throw transformationTestResult.analysisException; @@ -208,7 +207,7 @@ public class TransformerAndroidTestRunner { * Transforms the {@code uriString}. * * @param testId An identifier for the test. - * @param uriString The uri (as a {@link String}) of the file to transform. + * @param mediaItem The {@link MediaItem} to transform. * @return The {@link TransformationTestResult}. * @throws IOException If an error occurs opening the output file for writing * @throws TimeoutException If the transformation takes longer than the {@link #timeoutSeconds}. @@ -218,8 +217,14 @@ public class TransformerAndroidTestRunner { * @throws IllegalArgumentException If the path is invalid. * @throws IllegalStateException If an unexpected exception occurs when starting a transformation. */ - private TransformationTestResult runInternal(String testId, String uriString) + private TransformationTestResult runInternal(String testId, MediaItem mediaItem) throws InterruptedException, IOException, TimeoutException, TransformationException { + if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) + && calculateSsim) { + throw new UnsupportedOperationException( + "SSIM calculation is not supported for clipped inputs."); + } + AtomicReference<@NullableType TransformationException> transformationExceptionReference = new AtomicReference<>(); AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>(); @@ -249,15 +254,13 @@ public class TransformerAndroidTestRunner { }) .build(); - Uri uri = Uri.parse(uriString); File outputVideoFile = AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-output.mp4"); InstrumentationRegistry.getInstrumentation() .runOnMainSync( () -> { try { - testTransformer.startTransformation( - MediaItem.fromUri(uri), outputVideoFile.getAbsolutePath()); + testTransformer.startTransformation(mediaItem, outputVideoFile.getAbsolutePath()); // Catch all exceptions to report. Exceptions thrown here and not caught will NOT // propagate. } catch (Exception e) { @@ -300,7 +303,9 @@ public class TransformerAndroidTestRunner { if (calculateSsim) { double ssim = SsimHelper.calculate( - context, /* referenceVideoPath= */ uriString, outputVideoFile.getPath()); + context, + /* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(), + outputVideoFile.getPath()); resultBuilder.setSsim(ssim); } } catch (InterruptedException interruptedException) { diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index bfd250b542..e9d17cdd8a 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -16,11 +16,14 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; +import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING; import static com.google.common.truth.Truth.assertThat; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; import org.junit.Test; import org.junit.runner.RunWith; @@ -50,7 +53,7 @@ public class TransformerEndToEndTest { .build() .run( /* testId= */ "videoEditing_completesWithConsistentFrameCount", - MP4_ASSET_URI_STRING); + MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); assertThat(result.transformationResult.videoFrameCount).isEqualTo(expectedFrameCount); } @@ -71,8 +74,34 @@ public class TransformerEndToEndTest { TransformationTestResult result = new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(/* testId= */ "videoOnly_completesWithConsistentDuration", MP4_ASSET_URI_STRING); + .run( + /* testId= */ "videoOnly_completesWithConsistentDuration", + MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); assertThat(result.transformationResult.durationMs).isEqualTo(expectedDurationMs); } + + @Test + public void clippedMedia_completesWithClippedDuration() throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + Transformer transformer = new Transformer.Builder(context).build(); + long clippingStartMs = 10_000; + long clippingEndMs = 11_000; + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(clippingStartMs) + .setEndPositionMs(clippingEndMs) + .build()) + .build(); + + TransformationTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(/* testId= */ "clippedMedia_completesWithClippedDuration", mediaItem); + + assertThat(result.transformationResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); + } } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/RepeatedTranscodeTransformationTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/RepeatedTranscodeTransformationTest.java index d248ecc08d..17baef3302 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/RepeatedTranscodeTransformationTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/RepeatedTranscodeTransformationTest.java @@ -19,8 +19,10 @@ import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.AndroidTestUtil; import com.google.android.exoplayer2.transformer.TransformationRequest; import com.google.android.exoplayer2.transformer.TransformationTestResult; @@ -56,7 +58,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscode_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } @@ -86,7 +88,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscodeNoAudio_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } @@ -115,7 +117,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscodeNoVideo_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/SetFrameEditTransformationTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/SetFrameEditTransformationTest.java index cf36c50b68..7fc01415a1 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/SetFrameEditTransformationTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/SetFrameEditTransformationTest.java @@ -18,8 +18,10 @@ package com.google.android.exoplayer2.transformer.mh; import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.TransformationRequest; import com.google.android.exoplayer2.transformer.Transformer; import com.google.android.exoplayer2.transformer.TransformerAndroidTestRunner; @@ -41,6 +43,7 @@ public class SetFrameEditTransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run( - /* testId= */ "SetFrameEditTransform", MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + /* testId= */ "SetFrameEditTransform", + MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TranscodeQualityTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TranscodeQualityTest.java index fe94f59ff8..a671387626 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TranscodeQualityTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TranscodeQualityTest.java @@ -19,8 +19,10 @@ package com.google.android.exoplayer2.transformer.mh; import static com.google.common.truth.Truth.assertThat; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.AndroidTestUtil; import com.google.android.exoplayer2.transformer.TransformationRequest; import com.google.android.exoplayer2.transformer.TransformationTestResult; @@ -58,7 +60,10 @@ public final class TranscodeQualityTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run( + testId, + MediaItem.fromUri( + Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); @@ -92,7 +97,10 @@ public final class TranscodeQualityTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run( + testId, + MediaItem.fromUri( + Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); @@ -121,7 +129,10 @@ public final class TranscodeQualityTest { .build() .run( testId, - AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING); + MediaItem.fromUri( + Uri.parse( + AndroidTestUtil + .MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TransformationTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TransformationTest.java index 74f90902b6..4c5d02e10f 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TransformationTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/TransformationTest.java @@ -24,8 +24,10 @@ import static com.google.android.exoplayer2.transformer.AndroidTestUtil.MP4_REMO import static com.google.android.exoplayer2.transformer.AndroidTestUtil.recordTestSkipped; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.AndroidTestUtil; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; import com.google.android.exoplayer2.transformer.EncoderSelector; @@ -54,7 +56,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -65,7 +67,7 @@ public class TransformationTest { // No need to calculate SSIM because no decode/encoding, so input frames match output frames. new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -84,7 +86,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -106,7 +108,7 @@ public class TransformationTest { .setCalculateSsim(true) .setTimeoutSeconds(180) .build() - .run(testId, MP4_REMOTE_4K60_PORTRAIT_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_REMOTE_4K60_PORTRAIT_URI_STRING))); } @Test @@ -121,7 +123,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -135,7 +137,7 @@ public class TransformationTest { .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); } @Test @@ -159,6 +161,6 @@ public class TransformationTest { .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_SEF_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_URI_STRING))); } } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/BitrateAnalysisTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/BitrateAnalysisTest.java index b3c7927385..63fa4ca1b3 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/BitrateAnalysisTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/BitrateAnalysisTest.java @@ -19,7 +19,9 @@ import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR; import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR; import android.content.Context; +import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.AndroidTestUtil; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; import com.google.android.exoplayer2.transformer.EncoderSelector; @@ -116,6 +118,6 @@ public class BitrateAnalysisTest { .setInputValues(inputValues) .setCalculateSsim(true) .build() - .run(testId, fileUri); + .run(testId, MediaItem.fromUri(Uri.parse(fileUri))); } } diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java index b4d12ef001..142b0340b7 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java @@ -23,6 +23,7 @@ import android.content.Context; import android.media.MediaFormat; import android.net.Uri; import androidx.test.core.app.ApplicationProvider; +import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.transformer.AndroidTestUtil; import com.google.android.exoplayer2.transformer.DefaultEncoderFactory; import com.google.android.exoplayer2.transformer.EncoderSelector; @@ -136,6 +137,6 @@ public class EncoderPerformanceAnalysisTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setInputValues(inputValues) .build() - .run(testId, fileUri); + .run(testId, MediaItem.fromUri(Uri.parse(fileUri))); } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java index 042819b0fb..8ee9c7dcf3 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/PassthroughSamplePipeline.java @@ -25,14 +25,17 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; private final DecoderInputBuffer buffer; private final Format format; + private final long outputPresentationTimeOffsetUs; private boolean hasPendingBuffer; public PassthroughSamplePipeline( Format format, + long outputPresentationTimeOffsetUs, TransformationRequest transformationRequest, FallbackListener fallbackListener) { this.format = format; + this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; fallbackListener.onTransformationRequestFinalized(transformationRequest); @@ -46,6 +49,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; @Override public void queueInputBuffer() { + buffer.timeUs -= outputPresentationTimeOffsetUs; hasPendingBuffer = true; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java index 3e4f1e51dd..b8eb7c1025 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformationRequest.java @@ -88,6 +88,9 @@ public final class TransformationRequest { * Mp4Extractor#FLAG_READ_SEF_DATA} is set on the {@link Mp4Extractor} used. Otherwise, the slow * motion metadata will be ignored and the input won't be flattened. * + *

Using slow motion flattening together with {@link + * com.google.android.exoplayer2.MediaItem.ClippingConfiguration} is not supported yet. + * * @param flattenForSlowMotion Whether to flatten for slow motion. * @return This builder. */ diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 7462b7231a..28fda06e3e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -60,6 +60,7 @@ import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoRendererEventListener; import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -683,6 +684,12 @@ public final class Transformer { * @throws IOException If an error occurs opening the output file for writing. */ public void startTransformation(MediaItem mediaItem, String path) throws IOException { + if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) + && transformationRequest.flattenForSlowMotion) { + // TODO(b/233986762): Support clipping with SEF flattening. + throw new UnsupportedEncodingException( + "Clipping is not supported when slow motion flattening is requested"); + } startTransformation(mediaItem, muxerFactory.create(path, containerMimeType)); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index 651888daec..15a0ef21ce 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -70,7 +70,8 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; Format inputFormat = checkNotNull(formatHolder.format); if (shouldPassthrough(inputFormat)) { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener); } else { samplePipeline = new AudioTranscodingSamplePipeline( diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java index 52e2fe3931..964d15eb6e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java @@ -45,6 +45,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected boolean muxerWrapperTrackAdded; protected boolean muxerWrapperTrackEnded; protected long streamOffsetUs; + protected long startPositionOffsetUs; protected @MonotonicNonNull SamplePipeline samplePipeline; public TransformerBaseRenderer( @@ -109,6 +110,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { this.streamOffsetUs = offsetUs; + this.startPositionOffsetUs = startPositionUs - offsetUs; } @Override diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java index 6a0b4b6538..c7aee466dc 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java @@ -86,12 +86,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Format inputFormat = checkNotNull(formatHolder.format); if (shouldPassthrough(inputFormat)) { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener); } else { samplePipeline = new VideoTranscodingSamplePipeline( context, inputFormat, + startPositionOffsetUs, transformationRequest, effects, decoderFactory, @@ -108,6 +110,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } private boolean shouldPassthrough(Format inputFormat) { + // TODO(b/233988291): Use passthrough pipeline if the clipping start is a key-frame. + if (startPositionOffsetUs != 0) { + return false; + } if (encoderFactory.videoNeedsEncoding()) { return false; } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 46aa4d1b3a..a0191e75e1 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -27,6 +27,7 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import org.checkerframework.dataflow.qual.Pure; @@ -35,9 +36,12 @@ import org.checkerframework.dataflow.qual.Pure; */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { private final int outputRotationDegrees; + private final long outputPresentationTimeOffsetUs; + private final int maxPendingFrameCount; + private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; - private final int maxPendingFrameCount; + private final ArrayList decodeOnlyPresentationTimestamps; private final FrameProcessorChain frameProcessorChain; @@ -49,6 +53,7 @@ import org.checkerframework.dataflow.qual.Pure; public VideoTranscodingSamplePipeline( Context context, Format inputFormat, + long outputPresentationTimeOffsetUs, TransformationRequest transformationRequest, ImmutableList effects, Codec.DecoderFactory decoderFactory, @@ -58,10 +63,12 @@ import org.checkerframework.dataflow.qual.Pure; FrameProcessorChain.Listener frameProcessorChainListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { + this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + decodeOnlyPresentationTimestamps = new ArrayList<>(); // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. int decodedWidth = @@ -148,6 +155,9 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void queueInputBuffer() throws TransformationException { + if (decoderInputBuffer.isDecodeOnly()) { + decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); + } decoder.queueInputBuffer(decoderInputBuffer); } @@ -192,7 +202,7 @@ import org.checkerframework.dataflow.qual.Pure; return null; } MediaCodec.BufferInfo bufferInfo = checkNotNull(encoder.getOutputBufferInfo()); - encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs; + encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs - outputPresentationTimeOffsetUs; encoderOutputBuffer.setFlags(bufferInfo.flags); return encoderOutputBuffer; } @@ -251,10 +261,16 @@ import org.checkerframework.dataflow.qual.Pure; * @throws TransformationException If a problem occurs while processing the frame. */ private boolean maybeProcessDecoderOutput() throws TransformationException { - if (decoder.getOutputBufferInfo() == null) { + @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); + if (decoderOutputBufferInfo == null) { return false; } + if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { + decoder.releaseOutputBuffer(/* render= */ false); + return true; + } + if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { return false; @@ -264,4 +280,17 @@ import org.checkerframework.dataflow.qual.Pure; decoder.releaseOutputBuffer(/* render= */ true); return true; } + + private boolean isDecodeOnlyBuffer(long presentationTimeUs) { + // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would + // box presentationTimeUs, creating a Long object that would need to be garbage collected. + int size = decodeOnlyPresentationTimestamps.size(); + for (int i = 0; i < size; i++) { + if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { + decodeOnlyPresentationTimestamps.remove(i); + return true; + } + } + return false; + } }