From 7387284c1c1079306daecf9bbd24eca1f2a29def Mon Sep 17 00:00:00 2001 From: lpribanic Date: Thu, 2 Nov 2023 04:22:16 -0700 Subject: [PATCH] Add image track selection to DefaultTrackSelector DefaultTrackSelector now has all logic necessary for selecting an image track. If isPrioritizeImageOverVideoEnabled is set to true, image track will try to be selected first and a video track will only be selected if no image track is available. If isPrioritizeImageOverVideoEnabled is set to false, image track will be selected only if video track wasn't selected. PiperOrigin-RevId: 578806006 --- RELEASENOTES.md | 6 + .../trackselection/DefaultTrackSelector.java | 100 ++++++++++++++++- .../DefaultTrackSelectorTest.java | 106 ++++++++++++++++++ 3 files changed, 210 insertions(+), 2 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 29872f8bb7..2f1ff63ecf 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -17,6 +17,12 @@ * Transformer: * Add support for flattening H.265/HEVC SEF slow motion videos. * Track Selection: + * Add `DefaultTrackSelector.selectImageTrack` to enable image track + selection. + * Add `TrackSelectionParameters.isPrioritizeImageOverVideoEnabled` to + determine whether to select an image track if both an image track and a + video track are available. The default value is `false` which means + selecting a video track is prioritized. * Extractors: * Audio: * Video: diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java index 966995854a..b1523f2d7a 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java @@ -2615,7 +2615,16 @@ public class DefaultTrackSelector extends MappingTrackSelector rendererFormatSupports, rendererMixedMimeTypeAdaptationSupports, params); - if (selectedVideo != null) { + + @Nullable + Pair selectedImage = + params.isPrioritizeImageOverVideoEnabled || selectedVideo == null + ? selectImageTrack(mappedTrackInfo, rendererFormatSupports, params) + : null; + + if (selectedImage != null) { + definitions[selectedImage.second] = selectedImage.first; + } else if (selectedVideo != null) { definitions[selectedVideo.second] = selectedVideo.first; } @@ -2646,7 +2655,8 @@ public class DefaultTrackSelector extends MappingTrackSelector int trackType = mappedTrackInfo.getRendererType(i); if (trackType != C.TRACK_TYPE_VIDEO && trackType != C.TRACK_TYPE_AUDIO - && trackType != C.TRACK_TYPE_TEXT) { + && trackType != C.TRACK_TYPE_TEXT + && trackType != C.TRACK_TYPE_IMAGE) { definitions[i] = selectOtherTrack( trackType, mappedTrackInfo.getTrackGroups(i), rendererFormatSupports[i], params); @@ -2810,6 +2820,38 @@ public class DefaultTrackSelector extends MappingTrackSelector TextTrackInfo::compareSelections); } + // Image track selection implementation. + + /** + * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a + * {@link ExoTrackSelection.Definition} for an image track selection. + * + * @param mappedTrackInfo Mapped track information. + * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by + * renderer, track group and track (in that order). + * @param params The selector's current constraint parameters. + * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding + * renderer index, or null if no selection was made. + * @throws ExoPlaybackException If an error occurs while selecting the tracks. + */ + @Nullable + protected Pair selectImageTrack( + MappedTrackInfo mappedTrackInfo, + @Capabilities int[][][] rendererFormatSupports, + Parameters params) + throws ExoPlaybackException { + if (params.audioOffloadPreferences.audioOffloadMode == AUDIO_OFFLOAD_MODE_REQUIRED) { + return null; + } + return selectTracksForType( + C.TRACK_TYPE_IMAGE, + mappedTrackInfo, + rendererFormatSupports, + (int rendererIndex, TrackGroup group, @Capabilities int[] support) -> + ImageTrackInfo.createForTrackGroup(rendererIndex, group, params, support), + ImageTrackInfo::compareSelections); + } + // Generic track selection methods. /** @@ -3975,6 +4017,60 @@ public class DefaultTrackSelector extends MappingTrackSelector } } + private static final class ImageTrackInfo extends TrackInfo + implements Comparable { + + public static ImmutableList createForTrackGroup( + int rendererIndex, + TrackGroup trackGroup, + Parameters params, + @Capabilities int[] formatSupport) { + ImmutableList.Builder imageTracks = ImmutableList.builder(); + for (int i = 0; i < trackGroup.length; i++) { + imageTracks.add( + new ImageTrackInfo( + rendererIndex, trackGroup, /* trackIndex= */ i, params, formatSupport[i])); + } + return imageTracks.build(); + } + + private final @SelectionEligibility int selectionEligibility; + private final int pixelCount; + + public ImageTrackInfo( + int rendererIndex, + TrackGroup trackGroup, + int trackIndex, + Parameters parameters, + @Capabilities int trackFormatSupport) { + super(rendererIndex, trackGroup, trackIndex); + selectionEligibility = + isSupported(trackFormatSupport, parameters.exceedRendererCapabilitiesIfNecessary) + ? SELECTION_ELIGIBILITY_FIXED + : SELECTION_ELIGIBILITY_NO; + pixelCount = format.getPixelCount(); + } + + @Override + public @SelectionEligibility int getSelectionEligibility() { + return selectionEligibility; + } + + @Override + public boolean isCompatibleForAdaptationWith(ImageTrackInfo otherTrack) { + return false; + } + + @Override + public int compareTo(ImageTrackInfo other) { + return Integer.compare(this.pixelCount, other.pixelCount); + } + + public static int compareSelections(List infos1, List infos2) { + return infos1.get(0).compareTo(infos2.get(0)); + } + } + private static final class OtherTrackScore implements Comparable { private final boolean isDefault; diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java index 2ea08255bc..d4b8ad02d3 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java @@ -103,11 +103,16 @@ public final class DefaultTrackSelectorTest { private static final RendererCapabilities ALL_AUDIO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES = new FakeRendererCapabilities( C.TRACK_TYPE_AUDIO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES)); + private static final RendererCapabilities ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES = + new FakeRendererCapabilities( + C.TRACK_TYPE_VIDEO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES)); private static final RendererCapabilities VIDEO_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_VIDEO); private static final RendererCapabilities AUDIO_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_AUDIO); + private static final RendererCapabilities IMAGE_CAPABILITIES = + new FakeRendererCapabilities(C.TRACK_TYPE_IMAGE); private static final RendererCapabilities NO_SAMPLE_CAPABILITIES = new FakeRendererCapabilities(C.TRACK_TYPE_NONE); private static final RendererCapabilities[] RENDERER_CAPABILITIES = @@ -131,6 +136,8 @@ public final class DefaultTrackSelectorTest { .build(); private static final Format TEXT_FORMAT = new Format.Builder().setSampleMimeType(MimeTypes.TEXT_VTT).build(); + private static final Format IMAGE_FORMAT = + new Format.Builder().setSampleMimeType(MimeTypes.IMAGE_PNG).build(); private static final TrackGroup VIDEO_TRACK_GROUP = new TrackGroup(VIDEO_FORMAT); private static final TrackGroup AUDIO_TRACK_GROUP = new TrackGroup(AUDIO_FORMAT); @@ -2906,6 +2913,105 @@ public final class DefaultTrackSelectorTest { verify(invalidationListener).onRendererCapabilitiesChanged(renderer); } + @Test + public void + selectTracks_withImageAndVideoAndPrioritizeImageOverVideoEnabled_selectsOnlyImageTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build()); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertThat(result.selections[ /* video renderer index */0]).isNull(); + assertFixedSelection( + result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT); + } + + @Test + public void selectTracks_withImageAndVideoTracksBothSupported_selectsOnlyVideoTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertFixedSelection( + result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT); + assertThat(result.selections[ /* image renderer index */1]).isNull(); + } + + @Test + public void selectTracks_withVideoAndImageAndOnlyImageSupported_selectsImageTrack() + throws Exception { + TrackGroupArray trackGroups = + new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setExceedRendererCapabilitiesIfNecessary(false)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] { + ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES, IMAGE_CAPABILITIES + }, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertThat(result.selections[ /* video renderer index */0]).isNull(); + assertFixedSelection( + result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT); + } + + @Test + public void selectTracks_withVideoTrackOnlyAndPrioritizeImageOverVideoEnabled_selectsVideoTrack() + throws Exception { + TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(VIDEO_FORMAT)); + trackSelector.setParameters( + defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build()); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES}, + trackGroups, + periodId, + TIMELINE); + + assertThat(result.length).isEqualTo(2); + assertFixedSelection( + result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT); + assertThat(result.selections[ /* image renderer index */1]).isNull(); + } + + @Test + public void selectTracks_withMultipleImageTracks_selectsHighestResolutionTrack() + throws Exception { + Format image1 = IMAGE_FORMAT.buildUpon().setWidth(320).setHeight(320).build(); + Format image2 = IMAGE_FORMAT.buildUpon().setWidth(480).setHeight(480).build(); + TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(image1, image2)); + + TrackSelectorResult result = + trackSelector.selectTracks( + new RendererCapabilities[] {IMAGE_CAPABILITIES}, trackGroups, periodId, TIMELINE); + + assertThat(result.length).isEqualTo(1); + assertFixedSelection(result.selections[0], trackGroups, image2); + } + private static void assertSelections(TrackSelectorResult result, TrackSelection[] expected) { assertThat(result.length).isEqualTo(expected.length); for (int i = 0; i < expected.length; i++) {