diff --git a/libraries/test_data/src/test/assets/media/mp4/long_1080p_lowbitrate.mp4 b/libraries/test_data/src/test/assets/media/mp4/long_1080p_lowbitrate.mp4 new file mode 100644 index 0000000000..1eaf214dec Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/long_1080p_lowbitrate.mp4 differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 1ef723df22..24792bd37c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -377,6 +377,18 @@ public final class AndroidTestUtil { .build()) .build(); + public static final AssetInfo MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS = + new AssetInfo.Builder("asset:///media/mp4/long_1080p_lowbitrate.mp4") + .setVideoFormat( + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setWidth(1920) + .setHeight(1080) + .setFrameRate(30.00f) + .setCodecs("avc1.42C028") + .build()) + .build(); + /** Baseline profile level 3.0 H.264 stream, which should be supported on all devices. */ public static final AssetInfo MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S = new AssetInfo.Builder("asset:///media/mp4/sample_with_increasing_timestamps_320w_240h.mp4") diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java new file mode 100644 index 0000000000..6486abc4ab --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeForegroundSpeedTest.java @@ -0,0 +1,187 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer.mh; + +import static androidx.media3.transformer.AndroidTestUtil.MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS; +import static androidx.media3.transformer.AndroidTestUtil.assumeFormatsSupported; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assume.assumeTrue; + +import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.audio.ChannelMixingAudioProcessor; +import androidx.media3.common.audio.ChannelMixingMatrix; +import androidx.media3.common.audio.SonicAudioProcessor; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.Util; +import androidx.media3.effect.Presentation; +import androidx.media3.transformer.AndroidTestUtil; +import androidx.media3.transformer.AssetLoader; +import androidx.media3.transformer.Codec; +import androidx.media3.transformer.DefaultAssetLoaderFactory; +import androidx.media3.transformer.DefaultDecoderFactory; +import androidx.media3.transformer.EditedMediaItem; +import androidx.media3.transformer.Effects; +import androidx.media3.transformer.ExportTestResult; +import androidx.media3.transformer.SurfaceTestActivity; +import androidx.media3.transformer.Transformer; +import androidx.media3.transformer.TransformerAndroidTestRunner; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.rules.ActivityScenarioRule; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** Checks transcoding speed when running in foreground. */ +@RunWith(AndroidJUnit4.class) +public class TranscodeForegroundSpeedTest { + private final Context context = ApplicationProvider.getApplicationContext(); + @Rule public final TestName testName = new TestName(); + + // Creating a SurfaceTestActivity rule turns the screen on and puts the test app in foreground. + // This affects transcoding performance as foreground apps are more likely to schedule on the + // faster CPU cores. + @Rule + public ActivityScenarioRule rule = + new ActivityScenarioRule<>(SurfaceTestActivity.class); + + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @Test + public void + export1080pWithAudioTo720p_onMediumPerformanceDeviceWithDynamicScheduling_completesWithAtLeast140Fps() + throws Exception { + assumeTrue( + Ascii.toLowerCase(Util.MODEL).contains("pixel 2") + || Ascii.toLowerCase(Util.MODEL).contains("dn2103") + || Ascii.toLowerCase(Util.MODEL).contains("sm-g960f") + || Ascii.toLowerCase(Util.MODEL).contains("g8441")); + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat, + /* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat); + + ExportTestResult exportTestResult = + exportVideoAndAudioTo720pWithDynamicScheduling( + testId, + Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri), + /* durationMs= */ 30_000); + + // Running this without dynamic scheduling runs at under 80 fps. + assertThat(exportTestResult.throughputFps).isAtLeast(140); + } + + @Test + public void + export1080pWithAudioTo720p_onLowerPerformanceDevicesWithDynamicScheduling_completesWithAtLeast60Fps() + throws Exception { + assumeTrue( + (Ascii.toLowerCase(Util.MODEL).contains("f-01l") + || Ascii.toLowerCase(Util.MODEL).contains("asus_x00td") + || Ascii.toLowerCase(Util.MODEL).contains("redmi note 5") + || Ascii.toLowerCase(Util.MODEL).contains("mha-l29") + || Ascii.toLowerCase(Util.MODEL).contains("oneplus a6013") + || Ascii.toLowerCase(Util.MODEL).contains("cph1803") + || Ascii.toLowerCase(Util.MODEL).contains("mi a2 lite"))); + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat, + /* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat); + + ExportTestResult exportTestResult = + exportVideoAndAudioTo720pWithDynamicScheduling( + testId, + Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri), + /* durationMs= */ 15_000); + + // Running this without dynamic scheduling runs at under 40 fps. + assertThat(exportTestResult.throughputFps).isAtLeast(60); + } + + @Test + public void export1080pWithAudioTo720p_withDynamicScheduling_completesWithCorrectNumberOfFrames() + throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat, + /* outputFormat= */ MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.videoFormat); + + ExportTestResult exportTestResult = + exportVideoAndAudioTo720pWithDynamicScheduling( + testId, + Uri.parse(MP4_LONG_ASSET_WITH_AUDIO_AND_INCREASING_TIMESTAMPS.uri), + /* durationMs= */ 5_000); + + assertThat(exportTestResult.exportResult.videoFrameCount).isEqualTo(150); + } + + private static ExportTestResult exportVideoAndAudioTo720pWithDynamicScheduling( + String testId, Uri mediaUri, long durationMs) throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + Codec.DecoderFactory decoderFactory = + new DefaultDecoderFactory.Builder(context) + .experimentalSetDynamicSchedulingEnabled(true) + .setShouldConfigureOperatingRate(true) + .build(); + AssetLoader.Factory assetLoaderFactory = + new DefaultAssetLoaderFactory(context, decoderFactory, Clock.DEFAULT); + Transformer transformer = + new Transformer.Builder(context) + .setVideoMimeType(MimeTypes.VIDEO_H264) + .setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context)) + .setAssetLoaderFactory(assetLoaderFactory) + .build(); + MediaItem mediaItem = + MediaItem.fromUri(mediaUri) + .buildUpon() + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder().setEndPositionMs(durationMs).build()) + .build(); + SonicAudioProcessor sonicAudioProcessor = new SonicAudioProcessor(); + sonicAudioProcessor.setOutputSampleRateHz(44_100); + ChannelMixingAudioProcessor mixingAudioProcessor = new ChannelMixingAudioProcessor(); + mixingAudioProcessor.putChannelMixingMatrix( + ChannelMixingMatrix.create(/* inputChannelCount= */ 2, /* outputChannelCount= */ 1)); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder(mediaItem) + .setEffects( + new Effects( + ImmutableList.of(sonicAudioProcessor, mixingAudioProcessor), + ImmutableList.of( + Presentation.createForWidthAndHeight( + 1280, 720, Presentation.LAYOUT_SCALE_TO_FIT)))) + .build(); + + return new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java index 06a8c75ba3..bd76ed662e 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultDecoderFactory.java @@ -78,6 +78,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { private @C.Priority int codecPriority; private boolean shouldConfigureOperatingRate; private MediaCodecSelector mediaCodecSelector; + private boolean dynamicSchedulingEnabled; /** Creates a new {@link Builder}. */ public Builder(Context context) { @@ -165,6 +166,28 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { return this; } + /** + * Sets whether decoder dynamic scheduling is enabled. + * + *

If enabled, the {@link ExoPlayerAssetLoader} can change how often the rendering loop for + * {@linkplain DefaultCodec decoders} created by this factory is run. + * + *

On some devices, setting this to {@code true} will {@linkplain + * DefaultCodec#queueInputBuffer feed} and {@linkplain DefaultCodec#releaseOutputBuffer drain} + * decoders more frequently, and will lead to improved performance. + * + *

The default value is {@code false}. + * + *

This method is experimental, and will be renamed or removed in a future release. + * + * @param dynamicSchedulingEnabled Whether to enable dynamic scheduling. + */ + @CanIgnoreReturnValue + public Builder experimentalSetDynamicSchedulingEnabled(boolean dynamicSchedulingEnabled) { + this.dynamicSchedulingEnabled = dynamicSchedulingEnabled; + return this; + } + /** Creates an instance of {@link DefaultDecoderFactory}, using defaults if values are unset. */ public DefaultDecoderFactory build() { return new DefaultDecoderFactory(this); @@ -177,6 +200,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { private final @C.Priority int codecPriority; private final boolean shouldConfigureOperatingRate; private final MediaCodecSelector mediaCodecSelector; + private final boolean dynamicSchedulingEnabled; /** * @deprecated Use {@link Builder} instead. @@ -210,6 +234,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { this.codecPriority = builder.codecPriority; this.shouldConfigureOperatingRate = builder.shouldConfigureOperatingRate; this.mediaCodecSelector = builder.mediaCodecSelector; + this.dynamicSchedulingEnabled = builder.dynamicSchedulingEnabled; } @Override @@ -327,6 +352,15 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { return codec; } + /** + * Returns whether decoder dynamic scheduling is enabled. + * + *

See {@link Builder#experimentalSetDynamicSchedulingEnabled}. + */ + public boolean isDynamicSchedulingEnabled() { + return dynamicSchedulingEnabled; + } + private static DefaultCodec createCodecFromDecoderInfos( Context context, List decoderInfos, @@ -354,7 +388,7 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { } private static void configureOperatingRate(MediaFormat mediaFormat) { - if (Util.SDK_INT < 25) { + if (SDK_INT < 25) { // Not setting priority and operating rate achieves better decoding performance. return; } @@ -371,7 +405,8 @@ public final class DefaultDecoderFactory implements Codec.DecoderFactory { private static boolean deviceNeedsPriorityWorkaround() { // On these chipsets, decoder configuration fails if KEY_OPERATING_RATE is set but not // KEY_PRIORITY. See b/358519863. - return Util.SDK_INT >= 31 && Build.SOC_MODEL.equals("s5e8835"); + return SDK_INT >= 31 + && (Build.SOC_MODEL.equals("s5e8835") || Build.SOC_MODEL.equals("SA8155P")); } private static boolean deviceNeedsDisable8kWorkaround(Format format) { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java index 85f601e73b..c7cf809ef3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java @@ -54,6 +54,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; this.decoderFactory = decoderFactory; this.hdrMode = hdrMode; decodeOnlyPresentationTimestamps = new ArrayList<>(); + maxDecoderPendingFrameCount = C.INDEX_UNSET; } @Override @@ -61,6 +62,22 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return TAG; } + /** + * {@inheritDoc} + * + *

The duration is calculated based on the number of {@linkplain #maxDecoderPendingFrameCount + * allowed pending frames}. + */ + @Override + public long getDurationToProgressUs(long positionUs, long elapsedRealtimeUs) { + if (maxDecoderPendingFrameCount == C.INDEX_UNSET) { + return DEFAULT_DURATION_TO_PROGRESS_US; + } + // TODO: b/258809496 - Consider using async API and dynamic scheduling when decoder input + // slots are available. + return maxDecoderPendingFrameCount * 2_000L; + } + @Override protected Format overrideInputFormat(Format format) { if (hdrMode == Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java index 8c52968f45..cc23b27062 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExoPlayerAssetLoader.java @@ -189,6 +189,10 @@ public final class ExoPlayerAssetLoader implements AssetLoader { .setLooper(looper) .setUsePlatformDiagnostics(false) .setReleaseTimeoutMs(getReleaseTimeoutMs()); + if (decoderFactory instanceof DefaultDecoderFactory) { + playerBuilder.experimentalSetDynamicSchedulingEnabled( + ((DefaultDecoderFactory) decoderFactory).isDynamicSchedulingEnabled()); + } if (clock != Clock.DEFAULT) { // Transformer.Builder#setClock is also @VisibleForTesting, so if we're using a non-default // clock we must be in a test context.