Add debug trace logs for input/output events for codecs.

PiperOrigin-RevId: 633244952

libraries/effect/src/main/java/androidx/media3/effect/DebugTraceUtil.java

@@ -60,8 +60,11 @@ public final class DebugTraceUtil {
   @Retention(RetentionPolicy.SOURCE)
   @StringDef({
     EVENT_INPUT_FORMAT,
-    EVENT_DECODED_FRAME,
+    EVENT_OUTPUT_FORMAT,
-    EVENT_ENCODED_FRAME,
+    EVENT_ACCEPTED_INPUT,
+    EVENT_PRODUCED_OUTPUT,
+    EVENT_INPUT_ENDED,
+    EVENT_OUTPUT_ENDED,
     EVENT_REGISTER_NEW_INPUT_STREAM,
     EVENT_SURFACE_TEXTURE_INPUT,
     EVENT_QUEUE_FRAME,
@@ -84,8 +87,11 @@ public final class DebugTraceUtil {
   public @interface Event {}
 
   public static final String EVENT_INPUT_FORMAT = "InputFormat";
-  public static final String EVENT_DECODED_FRAME = "DecodedFrame";
+  public static final String EVENT_OUTPUT_FORMAT = "OutputFormat";
-  public static final String EVENT_ENCODED_FRAME = "EncodedFrame";
+  public static final String EVENT_ACCEPTED_INPUT = "AcceptedInput";
+  public static final String EVENT_PRODUCED_OUTPUT = "ProducedOutput";
+  public static final String EVENT_INPUT_ENDED = "InputEnded";
+  public static final String EVENT_OUTPUT_ENDED = "OutputEnded";
   public static final String EVENT_REGISTER_NEW_INPUT_STREAM = "RegisterNewInputStream";
   public static final String EVENT_SURFACE_TEXTURE_INPUT = "SurfaceTextureInput";
   public static final String EVENT_QUEUE_FRAME = "QueueFrame";
@@ -109,13 +115,15 @@ public final class DebugTraceUtil {
   @Retention(RetentionPolicy.SOURCE)
   @StringDef({
     COMPONENT_VIDEO,
-    COMPONENT_DECODER,
+    COMPONENT_AUDIO_DECODER,
+    COMPONENT_VIDEO_DECODER,
     COMPONENT_VFP,
     COMPONENT_BITMAP_TEXTURE_MANAGER,
     COMPONENT_EXTERNAL_TEXTURE_MANAGER,
     COMPONENT_TEX_ID_TEXTURE_MANAGER,
     COMPONENT_COMPOSITOR,
-    COMPONENT_ENCODER,
+    COMPONENT_AUDIO_ENCODER,
+    COMPONENT_VIDEO_ENCODER,
     COMPONENT_MUXER
   })
   @Target(TYPE_USE)
@@ -124,22 +132,41 @@ public final class DebugTraceUtil {
   // TODO - b/339639306: Migrate COMPONENT_VIDEO usage to COMPONENT_ASSETLOADER.
   public static final String COMPONENT_VIDEO = "Video";
 
-  public static final String COMPONENT_DECODER = "Decoder";
+  public static final String COMPONENT_AUDIO_DECODER = "AudioDecoder";
+  public static final String COMPONENT_VIDEO_DECODER = "VideoDecoder";
   public static final String COMPONENT_VFP = "VFP";
   public static final String COMPONENT_EXTERNAL_TEXTURE_MANAGER = "ExternalTextureManager";
   public static final String COMPONENT_BITMAP_TEXTURE_MANAGER = "BitmapTextureManager";
   public static final String COMPONENT_TEX_ID_TEXTURE_MANAGER = "TexIdTextureManager";
   public static final String COMPONENT_COMPOSITOR = "Compositor";
-  public static final String COMPONENT_ENCODER = "Encoder";
+  public static final String COMPONENT_AUDIO_ENCODER = "AudioEncoder";
+  public static final String COMPONENT_VIDEO_ENCODER = "VideoEncoder";
   public static final String COMPONENT_MUXER = "Muxer";
 
   // For a given component, events are in the rough expected order that they occur.
   private static final ImmutableMap<@Component String, List<@Event String>> COMPONENTS_TO_EVENTS =
-      ImmutableMap.of(
+      ImmutableMap.<String, List<String>>builder()
-          COMPONENT_VIDEO, ImmutableList.of(EVENT_INPUT_FORMAT),
+          .put(COMPONENT_VIDEO, ImmutableList.of(EVENT_INPUT_FORMAT))
-          COMPONENT_DECODER,
+          .put(
-              ImmutableList.of(EVENT_DECODED_FRAME, EVENT_RECEIVE_EOS, EVENT_SIGNAL_EOS),
+              COMPONENT_AUDIO_DECODER,
-          COMPONENT_VFP,
+              ImmutableList.of(
+                  EVENT_INPUT_FORMAT,
+                  EVENT_OUTPUT_FORMAT,
+                  EVENT_ACCEPTED_INPUT,
+                  EVENT_PRODUCED_OUTPUT,
+                  EVENT_INPUT_ENDED,
+                  EVENT_OUTPUT_ENDED))
+          .put(
+              COMPONENT_VIDEO_DECODER,
+              ImmutableList.of(
+                  EVENT_INPUT_FORMAT,
+                  EVENT_OUTPUT_FORMAT,
+                  EVENT_ACCEPTED_INPUT,
+                  EVENT_PRODUCED_OUTPUT,
+                  EVENT_INPUT_ENDED,
+                  EVENT_OUTPUT_ENDED))
+          .put(
+              COMPONENT_VFP,
               ImmutableList.of(
                   EVENT_REGISTER_NEW_INPUT_STREAM,
                   EVENT_SURFACE_TEXTURE_INPUT,
@@ -149,20 +176,39 @@ public final class DebugTraceUtil {
                   EVENT_RENDERED_TO_OUTPUT_SURFACE,
                   EVENT_OUTPUT_TEXTURE_RENDERED,
                   EVENT_RECEIVE_END_OF_ALL_INPUT,
-                  EVENT_SIGNAL_ENDED),
+                  EVENT_SIGNAL_ENDED))
-          COMPONENT_EXTERNAL_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS),
+          .put(COMPONENT_EXTERNAL_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS))
-          COMPONENT_BITMAP_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS),
+          .put(COMPONENT_BITMAP_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS))
-          COMPONENT_TEX_ID_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS),
+          .put(COMPONENT_TEX_ID_TEXTURE_MANAGER, ImmutableList.of(EVENT_SIGNAL_EOS))
-          COMPONENT_COMPOSITOR, ImmutableList.of(EVENT_OUTPUT_TEXTURE_RENDERED),
+          .put(COMPONENT_COMPOSITOR, ImmutableList.of(EVENT_OUTPUT_TEXTURE_RENDERED))
-          COMPONENT_ENCODER, ImmutableList.of(EVENT_ENCODED_FRAME, EVENT_RECEIVE_EOS),
+          .put(
-          COMPONENT_MUXER,
+              COMPONENT_AUDIO_ENCODER,
+              ImmutableList.of(
+                  EVENT_INPUT_FORMAT,
+                  EVENT_OUTPUT_FORMAT,
+                  EVENT_ACCEPTED_INPUT,
+                  EVENT_PRODUCED_OUTPUT,
+                  EVENT_INPUT_ENDED,
+                  EVENT_OUTPUT_ENDED))
+          .put(
+              COMPONENT_VIDEO_ENCODER,
+              ImmutableList.of(
+                  EVENT_INPUT_FORMAT,
+                  EVENT_OUTPUT_FORMAT,
+                  EVENT_ACCEPTED_INPUT,
+                  EVENT_PRODUCED_OUTPUT,
+                  EVENT_INPUT_ENDED,
+                  EVENT_OUTPUT_ENDED))
+          .put(
+              COMPONENT_MUXER,
               ImmutableList.of(
                   EVENT_CAN_WRITE_SAMPLE_VIDEO,
                   EVENT_WRITE_SAMPLE_VIDEO,
                   EVENT_CAN_WRITE_SAMPLE_AUDIO,
                   EVENT_WRITE_SAMPLE_AUDIO,
                   EVENT_TRACK_ENDED_AUDIO,
-                  EVENT_TRACK_ENDED_VIDEO));
+                  EVENT_TRACK_ENDED_VIDEO))
+          .buildOrThrow();
 
   private static final int MAX_FIRST_LAST_LOGS = 10;
 
@@ -226,6 +272,33 @@ public final class DebugTraceUtil {
     logEvent(component, event, presentationTimeUs, /* extraFormat= */ "");
   }
 
+  /**
+   * Logs an {@link Event} for a codec, if debug logging is enabled.
+   *
+   * @param isDecoder Whether the codec is a decoder.
+   * @param isVideo Whether the codec is for video.
+   * @param eventName The {@link Event} to log.
+   * @param presentationTimeUs The current presentation time of the media. Use {@link C#TIME_UNSET}
+   *     if unknown, {@link C#TIME_END_OF_SOURCE} if EOS.
+   * @param extraFormat Format string for optional extra information. See {@link
+   *     Util#formatInvariant(String, Object...)}.
+   * @param extraArgs Arguments for optional extra information.
+   */
+  public static synchronized void logCodecEvent(
+      boolean isDecoder,
+      boolean isVideo,
+      @Event String eventName,
+      long presentationTimeUs,
+      String extraFormat,
+      Object... extraArgs) {
+    logEvent(
+        getCodecComponent(isDecoder, isVideo),
+        eventName,
+        presentationTimeUs,
+        extraFormat,
+        extraArgs);
+  }
+
   /**
    * Generate a summary of the logged events, containing the total number of times an event happened
    * and the detailed log of a window of the oldest and newest events.
@@ -304,6 +377,22 @@ public final class DebugTraceUtil {
     }
   }
 
+  private static @Component String getCodecComponent(boolean isDecoder, boolean isVideo) {
+    if (isDecoder) {
+      if (isVideo) {
+        return COMPONENT_VIDEO_DECODER;
+      } else {
+        return COMPONENT_AUDIO_DECODER;
+      }
+    } else {
+      if (isVideo) {
+        return COMPONENT_VIDEO_ENCODER;
+      } else {
+        return COMPONENT_AUDIO_ENCODER;
+      }
+    }
+  }
+
   private static final class EventLog {
     public final long presentationTimeUs;
     public final long eventTimeMs;

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultCodec.java

@@ -21,9 +21,12 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
 import static androidx.media3.common.util.Util.SDK_INT;
-import static androidx.media3.effect.DebugTraceUtil.COMPONENT_DECODER;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_ACCEPTED_INPUT;
-import static androidx.media3.effect.DebugTraceUtil.COMPONENT_ENCODER;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_INPUT_ENDED;
-import static androidx.media3.effect.DebugTraceUtil.EVENT_RECEIVE_EOS;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_INPUT_FORMAT;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_OUTPUT_ENDED;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_OUTPUT_FORMAT;
+import static androidx.media3.effect.DebugTraceUtil.EVENT_PRODUCED_OUTPUT;
 
 import android.content.Context;
 import android.media.MediaCodec;
@@ -109,6 +112,8 @@ public final class DefaultCodec implements Codec {
     outputBufferInfo = new BufferInfo();
     inputBufferIndex = C.INDEX_UNSET;
     outputBufferIndex = C.INDEX_UNSET;
+    DebugTraceUtil.logCodecEvent(
+        isDecoder, isVideo, EVENT_INPUT_FORMAT, C.TIME_UNSET, "%s", configurationFormat);
 
     @Nullable MediaCodec mediaCodec = null;
     @Nullable Surface inputSurface = null;
@@ -221,10 +226,8 @@ public final class DefaultCodec implements Codec {
       inputStreamEnded = true;
       flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
 
+      debugTraceLogEvent(EVENT_INPUT_ENDED, C.TIME_END_OF_SOURCE);
       if (isDecoder) {
-        if (isVideo) {
-          DebugTraceUtil.logEvent(COMPONENT_DECODER, EVENT_RECEIVE_EOS, C.TIME_END_OF_SOURCE);
-        }
         // EOS buffer on the decoder input should never carry data.
         checkState(inputBuffer.data == null || !inputBuffer.data.hasRemaining());
         offset = 0;
@@ -238,13 +241,14 @@ public final class DefaultCodec implements Codec {
       Log.d(TAG, "MediaCodec error", e);
       throw createExportException(e);
     }
+    debugTraceLogEvent(EVENT_ACCEPTED_INPUT, timestampUs, "bytes=%s", size);
     inputBufferIndex = C.INDEX_UNSET;
     inputBuffer.data = null;
   }
 
   @Override
   public void signalEndOfInputStream() throws ExportException {
-    DebugTraceUtil.logEvent(COMPONENT_ENCODER, EVENT_RECEIVE_EOS, C.TIME_END_OF_SOURCE);
+    debugTraceLogEvent(EVENT_INPUT_ENDED, C.TIME_END_OF_SOURCE);
     try {
       mediaCodec.signalEndOfInputStream();
     } catch (RuntimeException e) {
@@ -264,7 +268,16 @@ public final class DefaultCodec implements Codec {
   @Override
   @Nullable
   public ByteBuffer getOutputBuffer() throws ExportException {
-    return maybeDequeueOutputBuffer(/* setOutputBuffer= */ true) ? outputBuffer : null;
+    boolean outputAvailable = maybeDequeueOutputBuffer(/* setOutputBuffer= */ true);
+    if (!outputAvailable) {
+      return null;
+    }
+    debugTraceLogEvent(
+        EVENT_PRODUCED_OUTPUT,
+        outputBufferInfo.presentationTimeUs,
+        "bytesOutput=%s",
+        outputBufferInfo.size);
+    return outputBuffer;
   }
 
   @Override
@@ -295,6 +308,7 @@ public final class DefaultCodec implements Codec {
       if (render) {
         mediaCodec.releaseOutputBuffer(
             outputBufferIndex, /* renderTimestampNs= */ renderPresentationTimeUs * 1000);
+        debugTraceLogEvent(EVENT_PRODUCED_OUTPUT, renderPresentationTimeUs);
       } else {
         mediaCodec.releaseOutputBuffer(outputBufferIndex, /* render= */ false);
       }
@@ -373,11 +387,15 @@ public final class DefaultCodec implements Codec {
                   .setPcmEncoding(configurationFormat.pcmEncoding)
                   .build();
         }
+        debugTraceLogEvent(
+            EVENT_OUTPUT_FORMAT, outputBufferInfo.presentationTimeUs, "%s", outputFormat);
       }
       return false;
     }
     if ((outputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
       outputStreamEnded = true;
+      debugTraceLogEvent(EVENT_OUTPUT_ENDED, C.TIME_END_OF_SOURCE);
+
       if (outputBufferInfo.size == 0) {
         releaseOutputBuffer(/* render= */ false);
         return false;
@@ -466,6 +484,19 @@ public final class DefaultCodec implements Codec {
             == MediaFormat.COLOR_TRANSFER_SDR_VIDEO;
   }
 
+  private void debugTraceLogEvent(@DebugTraceUtil.Event String event, long presentationTimeUs) {
+    debugTraceLogEvent(event, presentationTimeUs, /* extraFormat= */ "");
+  }
+
+  private void debugTraceLogEvent(
+      @DebugTraceUtil.Event String event,
+      long presentationTimeUs,
+      String extraFormat,
+      Object... extraArgs) {
+    DebugTraceUtil.logCodecEvent(
+        isDecoder, isVideo, event, presentationTimeUs, extraFormat, extraArgs);
+  }
+
   @RequiresApi(29)
   private static final class Api29 {
     @DoNotInline

libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderVideoRenderer.java

@@ -17,11 +17,8 @@ package androidx.media3.transformer;
 
 import static androidx.media3.common.util.Assertions.checkNotNull;
 import static androidx.media3.common.util.Assertions.checkStateNotNull;
-import static androidx.media3.effect.DebugTraceUtil.COMPONENT_DECODER;
 import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VIDEO;
-import static androidx.media3.effect.DebugTraceUtil.EVENT_DECODED_FRAME;
 import static androidx.media3.effect.DebugTraceUtil.EVENT_INPUT_FORMAT;
-import static androidx.media3.effect.DebugTraceUtil.EVENT_SIGNAL_EOS;
 import static androidx.media3.transformer.TransformerUtil.getDecoderOutputColor;
 
 import android.media.MediaCodec;
@@ -157,7 +154,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
   @RequiresNonNull({"sampleConsumer", "decoder"})
   protected boolean feedConsumerFromDecoder() throws ExportException {
     if (decoder.isEnded()) {
-      DebugTraceUtil.logEvent(COMPONENT_DECODER, EVENT_SIGNAL_EOS, C.TIME_END_OF_SOURCE);
       sampleConsumer.signalEndOfVideoInput();
       isEnded = true;
       return false;
@@ -184,7 +180,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
     }
 
     decoder.releaseOutputBuffer(presentationTimeUs);
-    DebugTraceUtil.logEvent(COMPONENT_DECODER, EVENT_DECODED_FRAME, presentationTimeUs);
     return true;
   }
 

libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java

@@ -24,8 +24,6 @@ import static androidx.media3.common.ColorInfo.SRGB_BT709_FULL;
 import static androidx.media3.common.ColorInfo.isTransferHdr;
 import static androidx.media3.common.util.Assertions.checkArgument;
 import static androidx.media3.common.util.Assertions.checkNotNull;
-import static androidx.media3.effect.DebugTraceUtil.COMPONENT_ENCODER;
-import static androidx.media3.effect.DebugTraceUtil.EVENT_ENCODED_FRAME;
 import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR;
 import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL;
 import static androidx.media3.transformer.TransformerUtil.getOutputMimeTypeAndHdrModeAfterFallback;
@@ -50,7 +48,6 @@ import androidx.media3.common.VideoGraph;
 import androidx.media3.common.util.Consumer;
 import androidx.media3.common.util.Util;
 import androidx.media3.decoder.DecoderInputBuffer;
-import androidx.media3.effect.DebugTraceUtil;
 import androidx.media3.effect.VideoCompositorSettings;
 import com.google.common.util.concurrent.MoreExecutors;
 import java.nio.ByteBuffer;
@@ -195,7 +192,6 @@ import org.checkerframework.dataflow.qual.Pure;
         hasMuxedTimestampZero = true;
       }
     }
-    DebugTraceUtil.logEvent(COMPONENT_ENCODER, EVENT_ENCODED_FRAME, bufferInfo.presentationTimeUs);
     encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs;
     encoderOutputBuffer.setFlags(bufferInfo.flags);
     return encoderOutputBuffer;