Add image track selection to DefaultTrackSelector

DefaultTrackSelector now has all logic necessary for selecting an
image track.

If isPrioritizeImageOverVideoEnabled is set to true, image track will
try to be selected first and a video track will only be selected if no
image track is available. If isPrioritizeImageOverVideoEnabled is set
to false, image track will be selected only if video track wasn't
selected.

PiperOrigin-RevId: 578806006

RELEASENOTES.md

@@ -17,6 +17,12 @@
 *   Transformer:
     *   Add support for flattening H.265/HEVC SEF slow motion videos.
 *   Track Selection:
+    *   Add `DefaultTrackSelector.selectImageTrack` to enable image track
+        selection.
+    *   Add `TrackSelectionParameters.isPrioritizeImageOverVideoEnabled` to
+        determine whether to select an image track if both an image track and a
+        video track are available. The default value is `false` which means
+        selecting a video track is prioritized.
 *   Extractors:
 *   Audio:
 *   Video:

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelector.java

@@ -2615,7 +2615,16 @@ public class DefaultTrackSelector extends MappingTrackSelector
             rendererFormatSupports,
             rendererMixedMimeTypeAdaptationSupports,
             params);
-    if (selectedVideo != null) {
+
+    @Nullable
+    Pair<ExoTrackSelection.Definition, Integer> selectedImage =
+        params.isPrioritizeImageOverVideoEnabled || selectedVideo == null
+            ? selectImageTrack(mappedTrackInfo, rendererFormatSupports, params)
+            : null;
+
+    if (selectedImage != null) {
+      definitions[selectedImage.second] = selectedImage.first;
+    } else if (selectedVideo != null) {
       definitions[selectedVideo.second] = selectedVideo.first;
     }
 
@@ -2646,7 +2655,8 @@ public class DefaultTrackSelector extends MappingTrackSelector
       int trackType = mappedTrackInfo.getRendererType(i);
       if (trackType != C.TRACK_TYPE_VIDEO
           && trackType != C.TRACK_TYPE_AUDIO
-          && trackType != C.TRACK_TYPE_TEXT) {
+          && trackType != C.TRACK_TYPE_TEXT
+          && trackType != C.TRACK_TYPE_IMAGE) {
         definitions[i] =
             selectOtherTrack(
                 trackType, mappedTrackInfo.getTrackGroups(i), rendererFormatSupports[i], params);
@@ -2810,6 +2820,38 @@ public class DefaultTrackSelector extends MappingTrackSelector
         TextTrackInfo::compareSelections);
   }
 
+  // Image track selection implementation.
+
+  /**
+   * Called by {@link #selectAllTracks(MappedTrackInfo, int[][][], int[], Parameters)} to create a
+   * {@link ExoTrackSelection.Definition} for an image track selection.
+   *
+   * @param mappedTrackInfo Mapped track information.
+   * @param rendererFormatSupports The {@link Capabilities} for each mapped track, indexed by
+   *     renderer, track group and track (in that order).
+   * @param params The selector's current constraint parameters.
+   * @return A pair of the selected {@link ExoTrackSelection.Definition} and the corresponding
+   *     renderer index, or null if no selection was made.
+   * @throws ExoPlaybackException If an error occurs while selecting the tracks.
+   */
+  @Nullable
+  protected Pair<ExoTrackSelection.Definition, Integer> selectImageTrack(
+      MappedTrackInfo mappedTrackInfo,
+      @Capabilities int[][][] rendererFormatSupports,
+      Parameters params)
+      throws ExoPlaybackException {
+    if (params.audioOffloadPreferences.audioOffloadMode == AUDIO_OFFLOAD_MODE_REQUIRED) {
+      return null;
+    }
+    return selectTracksForType(
+        C.TRACK_TYPE_IMAGE,
+        mappedTrackInfo,
+        rendererFormatSupports,
+        (int rendererIndex, TrackGroup group, @Capabilities int[] support) ->
+            ImageTrackInfo.createForTrackGroup(rendererIndex, group, params, support),
+        ImageTrackInfo::compareSelections);
+  }
+
   // Generic track selection methods.
 
   /**
@@ -3975,6 +4017,60 @@ public class DefaultTrackSelector extends MappingTrackSelector
     }
   }
 
+  private static final class ImageTrackInfo extends TrackInfo<ImageTrackInfo>
+      implements Comparable<ImageTrackInfo> {
+
+    public static ImmutableList<ImageTrackInfo> createForTrackGroup(
+        int rendererIndex,
+        TrackGroup trackGroup,
+        Parameters params,
+        @Capabilities int[] formatSupport) {
+      ImmutableList.Builder<ImageTrackInfo> imageTracks = ImmutableList.builder();
+      for (int i = 0; i < trackGroup.length; i++) {
+        imageTracks.add(
+            new ImageTrackInfo(
+                rendererIndex, trackGroup, /* trackIndex= */ i, params, formatSupport[i]));
+      }
+      return imageTracks.build();
+    }
+
+    private final @SelectionEligibility int selectionEligibility;
+    private final int pixelCount;
+
+    public ImageTrackInfo(
+        int rendererIndex,
+        TrackGroup trackGroup,
+        int trackIndex,
+        Parameters parameters,
+        @Capabilities int trackFormatSupport) {
+      super(rendererIndex, trackGroup, trackIndex);
+      selectionEligibility =
+          isSupported(trackFormatSupport, parameters.exceedRendererCapabilitiesIfNecessary)
+              ? SELECTION_ELIGIBILITY_FIXED
+              : SELECTION_ELIGIBILITY_NO;
+      pixelCount = format.getPixelCount();
+    }
+
+    @Override
+    public @SelectionEligibility int getSelectionEligibility() {
+      return selectionEligibility;
+    }
+
+    @Override
+    public boolean isCompatibleForAdaptationWith(ImageTrackInfo otherTrack) {
+      return false;
+    }
+
+    @Override
+    public int compareTo(ImageTrackInfo other) {
+      return Integer.compare(this.pixelCount, other.pixelCount);
+    }
+
+    public static int compareSelections(List<ImageTrackInfo> infos1, List<ImageTrackInfo> infos2) {
+      return infos1.get(0).compareTo(infos2.get(0));
+    }
+  }
+
   private static final class OtherTrackScore implements Comparable<OtherTrackScore> {
 
     private final boolean isDefault;

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/trackselection/DefaultTrackSelectorTest.java

@@ -103,11 +103,16 @@ public final class DefaultTrackSelectorTest {
   private static final RendererCapabilities ALL_AUDIO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES =
       new FakeRendererCapabilities(
           C.TRACK_TYPE_AUDIO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES));
+  private static final RendererCapabilities ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES =
+      new FakeRendererCapabilities(
+          C.TRACK_TYPE_VIDEO, RendererCapabilities.create(FORMAT_EXCEEDS_CAPABILITIES));
 
   private static final RendererCapabilities VIDEO_CAPABILITIES =
       new FakeRendererCapabilities(C.TRACK_TYPE_VIDEO);
   private static final RendererCapabilities AUDIO_CAPABILITIES =
       new FakeRendererCapabilities(C.TRACK_TYPE_AUDIO);
+  private static final RendererCapabilities IMAGE_CAPABILITIES =
+      new FakeRendererCapabilities(C.TRACK_TYPE_IMAGE);
   private static final RendererCapabilities NO_SAMPLE_CAPABILITIES =
       new FakeRendererCapabilities(C.TRACK_TYPE_NONE);
   private static final RendererCapabilities[] RENDERER_CAPABILITIES =
@@ -131,6 +136,8 @@ public final class DefaultTrackSelectorTest {
           .build();
   private static final Format TEXT_FORMAT =
       new Format.Builder().setSampleMimeType(MimeTypes.TEXT_VTT).build();
+  private static final Format IMAGE_FORMAT =
+      new Format.Builder().setSampleMimeType(MimeTypes.IMAGE_PNG).build();
 
   private static final TrackGroup VIDEO_TRACK_GROUP = new TrackGroup(VIDEO_FORMAT);
   private static final TrackGroup AUDIO_TRACK_GROUP = new TrackGroup(AUDIO_FORMAT);
@@ -2906,6 +2913,105 @@ public final class DefaultTrackSelectorTest {
     verify(invalidationListener).onRendererCapabilitiesChanged(renderer);
   }
 
+  @Test
+  public void
+      selectTracks_withImageAndVideoAndPrioritizeImageOverVideoEnabled_selectsOnlyImageTrack()
+          throws Exception {
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT));
+    trackSelector.setParameters(
+        defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build());
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES},
+            trackGroups,
+            periodId,
+            TIMELINE);
+
+    assertThat(result.length).isEqualTo(2);
+    assertThat(result.selections[ /* video renderer index */0]).isNull();
+    assertFixedSelection(
+        result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT);
+  }
+
+  @Test
+  public void selectTracks_withImageAndVideoTracksBothSupported_selectsOnlyVideoTrack()
+      throws Exception {
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES},
+            trackGroups,
+            periodId,
+            TIMELINE);
+
+    assertThat(result.length).isEqualTo(2);
+    assertFixedSelection(
+        result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT);
+    assertThat(result.selections[ /* image renderer index */1]).isNull();
+  }
+
+  @Test
+  public void selectTracks_withVideoAndImageAndOnlyImageSupported_selectsImageTrack()
+      throws Exception {
+    TrackGroupArray trackGroups =
+        new TrackGroupArray(new TrackGroup(IMAGE_FORMAT), new TrackGroup(VIDEO_FORMAT));
+    trackSelector.setParameters(
+        defaultParameters.buildUpon().setExceedRendererCapabilitiesIfNecessary(false));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {
+              ALL_VIDEO_FORMAT_EXCEEDED_RENDERER_CAPABILITIES, IMAGE_CAPABILITIES
+            },
+            trackGroups,
+            periodId,
+            TIMELINE);
+
+    assertThat(result.length).isEqualTo(2);
+    assertThat(result.selections[ /* video renderer index */0]).isNull();
+    assertFixedSelection(
+        result.selections[ /* image renderer index */1], trackGroups, IMAGE_FORMAT);
+  }
+
+  @Test
+  public void selectTracks_withVideoTrackOnlyAndPrioritizeImageOverVideoEnabled_selectsVideoTrack()
+      throws Exception {
+    TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(VIDEO_FORMAT));
+    trackSelector.setParameters(
+        defaultParameters.buildUpon().setPrioritizeImageOverVideoEnabled(true).build());
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {VIDEO_CAPABILITIES, IMAGE_CAPABILITIES},
+            trackGroups,
+            periodId,
+            TIMELINE);
+
+    assertThat(result.length).isEqualTo(2);
+    assertFixedSelection(
+        result.selections[ /* video renderer index */0], trackGroups, VIDEO_FORMAT);
+    assertThat(result.selections[ /* image renderer index */1]).isNull();
+  }
+
+  @Test
+  public void selectTracks_withMultipleImageTracks_selectsHighestResolutionTrack()
+      throws Exception {
+    Format image1 = IMAGE_FORMAT.buildUpon().setWidth(320).setHeight(320).build();
+    Format image2 = IMAGE_FORMAT.buildUpon().setWidth(480).setHeight(480).build();
+    TrackGroupArray trackGroups = new TrackGroupArray(new TrackGroup(image1, image2));
+
+    TrackSelectorResult result =
+        trackSelector.selectTracks(
+            new RendererCapabilities[] {IMAGE_CAPABILITIES}, trackGroups, periodId, TIMELINE);
+
+    assertThat(result.length).isEqualTo(1);
+    assertFixedSelection(result.selections[0], trackGroups, image2);
+  }
+
   private static void assertSelections(TrackSelectorResult result, TrackSelection[] expected) {
     assertThat(result.length).isEqualTo(expected.length);
     for (int i = 0; i < expected.length; i++) {