mirror of
https://github.com/samsonjs/media.git
synced 2026-04-27 15:07:40 +00:00
Fix last sample detection issue for ClippingMediaSource
If stream is not considered final and `ClippingMediaSource` is read to `endUs`, then `BaseRenderer.readSource` returns `C.RESULT_NOTHING_READ`. In that case, the `lastBufferInStreamPresentationTimeUs` is not set and the last frame is not rendered. PiperOrigin-RevId: 554418971
This commit is contained in:
michaelkatz
Tianyi Feng
parent
82387ccfe6
commit
aed21e464e
2 changed files with 230 additions and 0 deletions
|
|
@ -1306,6 +1306,10 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
|
||||||
}
|
}
|
||||||
|
|
||||||
if (result == C.RESULT_NOTHING_READ) {
|
if (result == C.RESULT_NOTHING_READ) {
|
||||||
|
if (hasReadStreamToEnd()) {
|
||||||
|
// Notify output queue of the last buffer's timestamp.
|
||||||
|
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
|
||||||
|
}
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
if (result == C.RESULT_FORMAT_READ) {
|
if (result == C.RESULT_FORMAT_READ) {
|
||||||
|
|
@ -1321,6 +1325,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer {
|
||||||
|
|
||||||
// We've read a buffer.
|
// We've read a buffer.
|
||||||
if (buffer.isEndOfStream()) {
|
if (buffer.isEndOfStream()) {
|
||||||
|
lastBufferInStreamPresentationTimeUs = largestQueuedPresentationTimeUs;
|
||||||
if (codecReconfigurationState == RECONFIGURATION_STATE_QUEUE_PENDING) {
|
if (codecReconfigurationState == RECONFIGURATION_STATE_QUEUE_PENDING) {
|
||||||
// We received a new format immediately before the end of the stream. We need to clear
|
// We received a new format immediately before the end of the stream. We need to clear
|
||||||
// the corresponding reconfiguration data from the current buffer, but re-write it into
|
// the corresponding reconfiguration data from the current buffer, but re-write it into
|
||||||
|
|
|
||||||
|
|
@ -46,6 +46,7 @@ import androidx.annotation.Nullable;
|
||||||
import androidx.media3.common.C;
|
import androidx.media3.common.C;
|
||||||
import androidx.media3.common.Format;
|
import androidx.media3.common.Format;
|
||||||
import androidx.media3.common.MimeTypes;
|
import androidx.media3.common.MimeTypes;
|
||||||
|
import androidx.media3.common.TrackGroup;
|
||||||
import androidx.media3.common.VideoSize;
|
import androidx.media3.common.VideoSize;
|
||||||
import androidx.media3.common.util.Clock;
|
import androidx.media3.common.util.Clock;
|
||||||
import androidx.media3.decoder.CryptoInfo;
|
import androidx.media3.decoder.CryptoInfo;
|
||||||
|
|
@ -61,8 +62,19 @@ import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
|
||||||
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
|
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
|
||||||
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
|
import androidx.media3.exoplayer.mediacodec.MediaCodecSelector;
|
||||||
import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter;
|
import androidx.media3.exoplayer.mediacodec.SynchronousMediaCodecAdapter;
|
||||||
|
import androidx.media3.exoplayer.source.ClippingMediaPeriod;
|
||||||
|
import androidx.media3.exoplayer.source.MediaPeriod;
|
||||||
|
import androidx.media3.exoplayer.source.MediaSource;
|
||||||
|
import androidx.media3.exoplayer.source.MediaSourceEventListener;
|
||||||
|
import androidx.media3.exoplayer.source.SampleStream;
|
||||||
|
import androidx.media3.exoplayer.source.TrackGroupArray;
|
||||||
|
import androidx.media3.exoplayer.trackselection.ExoTrackSelection;
|
||||||
|
import androidx.media3.exoplayer.trackselection.FixedTrackSelection;
|
||||||
|
import androidx.media3.exoplayer.upstream.Allocator;
|
||||||
import androidx.media3.exoplayer.upstream.DefaultAllocator;
|
import androidx.media3.exoplayer.upstream.DefaultAllocator;
|
||||||
|
import androidx.media3.test.utils.FakeMediaPeriod;
|
||||||
import androidx.media3.test.utils.FakeSampleStream;
|
import androidx.media3.test.utils.FakeSampleStream;
|
||||||
|
import androidx.media3.test.utils.robolectric.RobolectricUtil;
|
||||||
import androidx.test.core.app.ApplicationProvider;
|
import androidx.test.core.app.ApplicationProvider;
|
||||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||||
import com.google.common.collect.ImmutableList;
|
import com.google.common.collect.ImmutableList;
|
||||||
|
|
@ -71,6 +83,7 @@ import java.nio.ByteBuffer;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
import java.util.concurrent.TimeUnit;
|
import java.util.concurrent.TimeUnit;
|
||||||
|
import java.util.concurrent.atomic.AtomicBoolean;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
import org.junit.After;
|
import org.junit.After;
|
||||||
import org.junit.Before;
|
import org.junit.Before;
|
||||||
|
|
@ -98,6 +111,8 @@ public class MediaCodecVideoRendererTest {
|
||||||
.setHeight(1080)
|
.setHeight(1080)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
private static final TrackGroup TRACK_GROUP_H264 = new TrackGroup(VIDEO_H264);
|
||||||
|
|
||||||
private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO =
|
private static final MediaCodecInfo H264_PROFILE8_LEVEL4_HW_MEDIA_CODEC_INFO =
|
||||||
MediaCodecInfo.newInstance(
|
MediaCodecInfo.newInstance(
|
||||||
/* name= */ "h264-codec-hw",
|
/* name= */ "h264-codec-hw",
|
||||||
|
|
@ -279,6 +294,216 @@ public class MediaCodecVideoRendererTest {
|
||||||
assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
|
assertThat(argumentDecoderCounters.getValue().skippedOutputBufferCount).isEqualTo(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void
|
||||||
|
render_withClippingMediaPeriodAndBufferContainingLastAndClippingSamples_rendersLastFrame()
|
||||||
|
throws Exception {
|
||||||
|
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
|
||||||
|
ArgumentCaptor.forClass(DecoderCounters.class);
|
||||||
|
// Set up MediaPeriod with samples.
|
||||||
|
FakeMediaPeriod mediaPeriod =
|
||||||
|
new FakeMediaPeriod(
|
||||||
|
new TrackGroupArray(TRACK_GROUP_H264),
|
||||||
|
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
|
||||||
|
/* trackDataFactory= */ (format, mediaPeriodId) -> ImmutableList.of(),
|
||||||
|
new MediaSourceEventListener.EventDispatcher()
|
||||||
|
.withParameters(
|
||||||
|
/* windowIndex= */ 0,
|
||||||
|
new MediaSource.MediaPeriodId(/* periodUid= */ new Object())),
|
||||||
|
DrmSessionManager.DRM_UNSUPPORTED,
|
||||||
|
new DrmSessionEventListener.EventDispatcher(),
|
||||||
|
/* deferOnPrepared= */ false) {
|
||||||
|
@Override
|
||||||
|
protected FakeSampleStream createSampleStream(
|
||||||
|
Allocator allocator,
|
||||||
|
@Nullable MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher,
|
||||||
|
DrmSessionManager drmSessionManager,
|
||||||
|
DrmSessionEventListener.EventDispatcher drmEventDispatcher,
|
||||||
|
Format initialFormat,
|
||||||
|
List<FakeSampleStream.FakeSampleStreamItem> fakeSampleStreamItems) {
|
||||||
|
FakeSampleStream fakeSampleStream =
|
||||||
|
new FakeSampleStream(
|
||||||
|
allocator,
|
||||||
|
mediaSourceEventDispatcher,
|
||||||
|
drmSessionManager,
|
||||||
|
drmEventDispatcher,
|
||||||
|
initialFormat,
|
||||||
|
/* fakeSampleStreamItems= */ ImmutableList.of(
|
||||||
|
oneByteSample(/* timeUs= */ 90, C.BUFFER_FLAG_KEY_FRAME),
|
||||||
|
oneByteSample(/* timeUs= */ 200, C.BUFFER_FLAG_KEY_FRAME),
|
||||||
|
END_OF_STREAM_ITEM));
|
||||||
|
fakeSampleStream.writeData(0);
|
||||||
|
return fakeSampleStream;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ClippingMediaPeriod clippingMediaPeriod =
|
||||||
|
new ClippingMediaPeriod(
|
||||||
|
mediaPeriod,
|
||||||
|
/* enableInitialDiscontinuity= */ true,
|
||||||
|
/* startUs= */ 0,
|
||||||
|
/* endUs= */ 100);
|
||||||
|
AtomicBoolean periodPrepared = new AtomicBoolean();
|
||||||
|
clippingMediaPeriod.prepare(
|
||||||
|
new MediaPeriod.Callback() {
|
||||||
|
@Override
|
||||||
|
public void onPrepared(MediaPeriod mediaPeriod) {
|
||||||
|
periodPrepared.set(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onContinueLoadingRequested(MediaPeriod source) {
|
||||||
|
clippingMediaPeriod.continueLoading(/* positionUs= */ 0);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
/* positionUs= */ 100);
|
||||||
|
RobolectricUtil.runMainLooperUntil(periodPrepared::get);
|
||||||
|
SampleStream[] sampleStreams = new SampleStream[1];
|
||||||
|
clippingMediaPeriod.selectTracks(
|
||||||
|
new ExoTrackSelection[] {new FixedTrackSelection(TRACK_GROUP_H264, /* track= */ 0)},
|
||||||
|
/* mayRetainStreamFlags= */ new boolean[] {false},
|
||||||
|
sampleStreams,
|
||||||
|
/* streamResetFlags= */ new boolean[] {true},
|
||||||
|
/* positionUs= */ 100);
|
||||||
|
mediaCodecVideoRenderer =
|
||||||
|
new MediaCodecVideoRenderer(
|
||||||
|
ApplicationProvider.getApplicationContext(),
|
||||||
|
new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
|
||||||
|
mediaCodecSelector,
|
||||||
|
/* allowedJoiningTimeMs= */ 0,
|
||||||
|
/* enableDecoderFallback= */ false,
|
||||||
|
/* eventHandler= */ new Handler(testMainLooper),
|
||||||
|
/* eventListener= */ eventListener,
|
||||||
|
/* maxDroppedFramesToNotify= */ 1);
|
||||||
|
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
|
||||||
|
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
|
||||||
|
mediaCodecVideoRenderer.enable(
|
||||||
|
RendererConfiguration.DEFAULT,
|
||||||
|
new Format[] {VIDEO_H264},
|
||||||
|
sampleStreams[0],
|
||||||
|
/* positionUs= */ 0,
|
||||||
|
/* joining= */ false,
|
||||||
|
/* mayRenderStartOfStream= */ true,
|
||||||
|
/* startPositionUs= */ 100,
|
||||||
|
/* offsetUs= */ 0);
|
||||||
|
|
||||||
|
mediaCodecVideoRenderer.start();
|
||||||
|
// Call to render should have read all samples up before endUs.
|
||||||
|
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000L);
|
||||||
|
assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue();
|
||||||
|
// Following call to render should force-render last frame.
|
||||||
|
mediaCodecVideoRenderer.render(100, SystemClock.elapsedRealtime() * 1000L);
|
||||||
|
shadowOf(testMainLooper).idle();
|
||||||
|
|
||||||
|
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
|
||||||
|
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
|
||||||
|
assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void render_withClippingMediaPeriodSetCurrentStreamFinal_rendersLastFrame()
|
||||||
|
throws Exception {
|
||||||
|
ArgumentCaptor<DecoderCounters> argumentDecoderCounters =
|
||||||
|
ArgumentCaptor.forClass(DecoderCounters.class);
|
||||||
|
// Set up MediaPeriod with samples.
|
||||||
|
FakeMediaPeriod mediaPeriod =
|
||||||
|
new FakeMediaPeriod(
|
||||||
|
new TrackGroupArray(TRACK_GROUP_H264),
|
||||||
|
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
|
||||||
|
/* trackDataFactory= */ (format, mediaPeriodId) -> ImmutableList.of(),
|
||||||
|
new MediaSourceEventListener.EventDispatcher()
|
||||||
|
.withParameters(
|
||||||
|
/* windowIndex= */ 0,
|
||||||
|
new MediaSource.MediaPeriodId(/* periodUid= */ new Object())),
|
||||||
|
DrmSessionManager.DRM_UNSUPPORTED,
|
||||||
|
new DrmSessionEventListener.EventDispatcher(),
|
||||||
|
/* deferOnPrepared= */ false) {
|
||||||
|
@Override
|
||||||
|
protected FakeSampleStream createSampleStream(
|
||||||
|
Allocator allocator,
|
||||||
|
@Nullable MediaSourceEventListener.EventDispatcher mediaSourceEventDispatcher,
|
||||||
|
DrmSessionManager drmSessionManager,
|
||||||
|
DrmSessionEventListener.EventDispatcher drmEventDispatcher,
|
||||||
|
Format initialFormat,
|
||||||
|
List<FakeSampleStream.FakeSampleStreamItem> fakeSampleStreamItems) {
|
||||||
|
FakeSampleStream fakeSampleStream =
|
||||||
|
new FakeSampleStream(
|
||||||
|
allocator,
|
||||||
|
mediaSourceEventDispatcher,
|
||||||
|
drmSessionManager,
|
||||||
|
drmEventDispatcher,
|
||||||
|
initialFormat,
|
||||||
|
/* fakeSampleStreamItems= */ ImmutableList.of(
|
||||||
|
oneByteSample(/* timeUs= */ 90, C.BUFFER_FLAG_KEY_FRAME),
|
||||||
|
oneByteSample(/* timeUs= */ 200, C.BUFFER_FLAG_KEY_FRAME),
|
||||||
|
END_OF_STREAM_ITEM));
|
||||||
|
fakeSampleStream.writeData(0);
|
||||||
|
return fakeSampleStream;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
ClippingMediaPeriod clippingMediaPeriod =
|
||||||
|
new ClippingMediaPeriod(
|
||||||
|
mediaPeriod,
|
||||||
|
/* enableInitialDiscontinuity= */ true,
|
||||||
|
/* startUs= */ 0,
|
||||||
|
/* endUs= */ 100);
|
||||||
|
AtomicBoolean periodPrepared = new AtomicBoolean();
|
||||||
|
clippingMediaPeriod.prepare(
|
||||||
|
new MediaPeriod.Callback() {
|
||||||
|
@Override
|
||||||
|
public void onPrepared(MediaPeriod mediaPeriod) {
|
||||||
|
periodPrepared.set(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void onContinueLoadingRequested(MediaPeriod source) {
|
||||||
|
clippingMediaPeriod.continueLoading(/* positionUs= */ 0);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
/* positionUs= */ 100);
|
||||||
|
RobolectricUtil.runMainLooperUntil(periodPrepared::get);
|
||||||
|
SampleStream[] sampleStreams = new SampleStream[1];
|
||||||
|
clippingMediaPeriod.selectTracks(
|
||||||
|
new ExoTrackSelection[] {new FixedTrackSelection(TRACK_GROUP_H264, /* track= */ 0)},
|
||||||
|
/* mayRetainStreamFlags= */ new boolean[] {false},
|
||||||
|
sampleStreams,
|
||||||
|
/* streamResetFlags= */ new boolean[] {true},
|
||||||
|
/* positionUs= */ 100);
|
||||||
|
mediaCodecVideoRenderer =
|
||||||
|
new MediaCodecVideoRenderer(
|
||||||
|
ApplicationProvider.getApplicationContext(),
|
||||||
|
new ForwardingSynchronousMediaCodecAdapterWithBufferLimit.Factory(/* bufferLimit= */ 3),
|
||||||
|
mediaCodecSelector,
|
||||||
|
/* allowedJoiningTimeMs= */ 0,
|
||||||
|
/* enableDecoderFallback= */ false,
|
||||||
|
/* eventHandler= */ new Handler(testMainLooper),
|
||||||
|
/* eventListener= */ eventListener,
|
||||||
|
/* maxDroppedFramesToNotify= */ 1);
|
||||||
|
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
|
||||||
|
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
|
||||||
|
mediaCodecVideoRenderer.enable(
|
||||||
|
RendererConfiguration.DEFAULT,
|
||||||
|
new Format[] {VIDEO_H264},
|
||||||
|
sampleStreams[0],
|
||||||
|
/* positionUs= */ 0,
|
||||||
|
/* joining= */ false,
|
||||||
|
/* mayRenderStartOfStream= */ true,
|
||||||
|
/* startPositionUs= */ 100,
|
||||||
|
/* offsetUs= */ 0);
|
||||||
|
|
||||||
|
mediaCodecVideoRenderer.start();
|
||||||
|
mediaCodecVideoRenderer.setCurrentStreamFinal();
|
||||||
|
// Call to render should have read all samples up before endUs.
|
||||||
|
mediaCodecVideoRenderer.render(0, SystemClock.elapsedRealtime() * 1000L);
|
||||||
|
assertThat(mediaCodecVideoRenderer.hasReadStreamToEnd()).isTrue();
|
||||||
|
// Following call to render should force-render last frame.
|
||||||
|
mediaCodecVideoRenderer.render(100, SystemClock.elapsedRealtime() * 1000L);
|
||||||
|
shadowOf(testMainLooper).idle();
|
||||||
|
|
||||||
|
verify(eventListener).onRenderedFirstFrame(eq(surface), /* renderTimeMs= */ anyLong());
|
||||||
|
verify(eventListener).onVideoEnabled(argumentDecoderCounters.capture());
|
||||||
|
assertThat(argumentDecoderCounters.getValue().renderedOutputBufferCount).isEqualTo(1);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception {
|
public void render_sendsVideoSizeChangeWithCurrentFormatValues() throws Exception {
|
||||||
FakeSampleStream fakeSampleStream =
|
FakeSampleStream fakeSampleStream =
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue