From bf1a15652dc1cc5df2fc237381b9ac24bf4ff039 Mon Sep 17 00:00:00 2001 From: Dustin Date: Sun, 30 Jan 2022 21:05:33 -0700 Subject: [PATCH] Added support for AVC picOrderCountType = 2 --- .../extractor/avi/AvcChunkPeeker.java | 21 +++++--- .../extractor/avi/AviExtractor.java | 54 ++++++++++--------- .../exoplayer2/extractor/avi/AviTrack.java | 11 ++-- .../extractor/avi/PicCountClock.java | 13 +++-- .../extractor/avi/PicCountClockTest.java | 6 +-- 5 files changed, 62 insertions(+), 43 deletions(-) diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AvcChunkPeeker.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AvcChunkPeeker.java index 0fa888a266..950823da2a 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AvcChunkPeeker.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AvcChunkPeeker.java @@ -18,6 +18,7 @@ public class AvcChunkPeeker extends NalChunkPeeker { private static final int NAL_TYPE_SEI = 6; private static final int NAL_TYPE_SPS = 7; private static final int NAL_TYPE_PPS = 8; + private static final int NAL_TYPE_AUD = 9; private final PicCountClock picCountClock; private final Format.Builder formatBuilder; @@ -26,15 +27,14 @@ public class AvcChunkPeeker extends NalChunkPeeker { private float pixelWidthHeightRatio = 1f; private NalUnitUtil.SpsData spsData; - public AvcChunkPeeker(Format.Builder formatBuilder, TrackOutput trackOutput, long durationUs, - int length) { + public AvcChunkPeeker(Format.Builder formatBuilder, TrackOutput trackOutput, LinearClock clock) { super(16); this.formatBuilder = formatBuilder; this.trackOutput = trackOutput; - picCountClock = new PicCountClock(durationUs, length); + picCountClock = new PicCountClock(clock.durationUs, clock.length); } - public PicCountClock getPicCountClock() { + public LinearClock getClock() { return picCountClock; } @@ -58,7 +58,7 @@ public class AvcChunkPeeker extends NalChunkPeeker { if (spsData.separateColorPlaneFlag) { in.skipBits(2); //colour_plane_id } - in.readBits(spsData.frameNumLength); //frame_num + final int frameNum = in.readBits(spsData.frameNumLength); //frame_num if (!spsData.frameMbsOnlyFlag) { boolean field_pic_flag = in.readBit(); // field_pic_flag if (field_pic_flag) { @@ -71,6 +71,9 @@ public class AvcChunkPeeker extends NalChunkPeeker { //Log.d("Test", "FrameNum: " + frame + " cnt=" + picOrderCountLsb); picCountClock.setPicCount(picOrderCountLsb); return; + } else if (spsData.picOrderCountType == 2) { + picCountClock.setPicCount(frameNum); + return; } picCountClock.setIndex(picCountClock.getIndex()); } @@ -80,7 +83,12 @@ public class AvcChunkPeeker extends NalChunkPeeker { final int spsStart = nalTypeOffset + 1; nalTypeOffset = seekNextNal(input, spsStart); spsData = NalUnitUtil.parseSpsNalUnitPayload(buffer, spsStart, pos); - picCountClock.setMaxPicCount(1 << (spsData.picOrderCntLsbLength)); + if (spsData.picOrderCountType == 0) { + picCountClock.setMaxPicCount(1 << spsData.picOrderCntLsbLength, 2); + } else if (spsData.picOrderCountType == 2) { + //Plus one because we double the frame number + picCountClock.setMaxPicCount(1 << spsData.frameNumLength, 1); + } if (spsData.pixelWidthHeightRatio != pixelWidthHeightRatio) { pixelWidthHeightRatio = spsData.pixelWidthHeightRatio; formatBuilder.setPixelWidthHeightRatio(pixelWidthHeightRatio); @@ -103,6 +111,7 @@ public class AvcChunkPeeker extends NalChunkPeeker { case NAL_TYPE_IRD: picCountClock.syncIndexes(); return; + case NAL_TYPE_AUD: case NAL_TYPE_SEI: case NAL_TYPE_PPS: { nalTypeOffset = seekNextNal(input, nalTypeOffset); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviExtractor.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviExtractor.java index ec7ec42864..a19a4178df 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviExtractor.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviExtractor.java @@ -255,15 +255,14 @@ public class AviExtractor implements Extractor { builder.setFrameRate(streamHeader.getFrameRate()); builder.setSampleMimeType(mimeType); + final LinearClock clock = new LinearClock(durationUs, length); + aviTrack = new AviTrack(streamId, C.TRACK_TYPE_VIDEO, clock, trackOutput); + if (MimeTypes.VIDEO_H264.equals(mimeType)) { - final AvcChunkPeeker avcChunkPeeker = new AvcChunkPeeker(builder, trackOutput, durationUs, - length); - aviTrack = new AviTrack(streamId, C.TRACK_TYPE_VIDEO, avcChunkPeeker.getPicCountClock(), - trackOutput); + final AvcChunkPeeker avcChunkPeeker = new AvcChunkPeeker(builder, trackOutput, clock); + aviTrack.setClock(avcChunkPeeker.getClock()); aviTrack.setChunkPeeker(avcChunkPeeker); } else { - aviTrack = new AviTrack(streamId, C.TRACK_TYPE_VIDEO, - new LinearClock(durationUs, length), trackOutput); if (MimeTypes.VIDEO_MP4V.equals(mimeType)) { aviTrack.setChunkPeeker(new Mp4vChunkPeeker(builder, trackOutput)); } @@ -361,23 +360,31 @@ public class AviExtractor implements Extractor { return null; } - void updateAudioTiming(final int[] keyFrameCounts, final long videoDuration) { + void fixTimings(final int[] keyFrameCounts, final long videoDuration) { for (final AviTrack aviTrack : aviTracks) { - if (aviTrack != null && aviTrack.isAudio()) { - final long durationUs = aviTrack.getClock().durationUs; - i("Audio #" + aviTrack.id + " chunks: " + aviTrack.chunks + " us=" + durationUs + - " size=" + aviTrack.size); - final LinearClock linearClock = aviTrack.getClock(); - //If the audio track duration is off from the video by >5 % recalc using video - if ((durationUs - videoDuration) / (float)videoDuration > .05f) { - w("Audio #" + aviTrack.id + " duration is off using videoDuration"); - linearClock.setDuration(videoDuration); - } - linearClock.setLength(aviTrack.chunks); - if (aviTrack.chunks != keyFrameCounts[aviTrack.id]) { - w("Audio is not all key frames chunks=" + aviTrack.chunks + " keyFrames=" + - keyFrameCounts[aviTrack.id]); - } + if (aviTrack != null) { + if (aviTrack.isAudio()) { + final long durationUs = aviTrack.getClock().durationUs; + i("Audio #" + aviTrack.id + " chunks: " + aviTrack.chunks + " us=" + durationUs + + " size=" + aviTrack.size); + final LinearClock linearClock = aviTrack.getClock(); + //If the audio track duration is off from the video by >5 % recalc using video + if ((durationUs - videoDuration) / (float)videoDuration > .05f) { + w("Audio #" + aviTrack.id + " duration is off using videoDuration"); + linearClock.setDuration(videoDuration); + } + linearClock.setLength(aviTrack.chunks); + if (aviTrack.chunks != keyFrameCounts[aviTrack.id]) { + w("Audio is not all key frames chunks=" + aviTrack.chunks + " keyFrames=" + + keyFrameCounts[aviTrack.id]); + } + } /* else if (aviTrack.isVideo()) { + final LinearClock clock = aviTrack.getClock(); + if (clock.length != aviTrack.chunks) { + w("Video #" + aviTrack.id + " chunks != length changing FPS"); + clock.setLength(aviTrack.chunks); + } + }*/ } } } @@ -464,8 +471,7 @@ public class AviExtractor implements Extractor { i("Video chunks=" + videoTrack.chunks + " us=" + seekMap.getDurationUs()); - //Needs to be called after the duration is updated - updateAudioTiming(keyFrameCounts, durationUs); + fixTimings(keyFrameCounts, durationUs); setSeekMap(seekMap); } diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviTrack.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviTrack.java index 2a321f8289..c507604164 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviTrack.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/AviTrack.java @@ -20,7 +20,7 @@ public class AviTrack { final @C.TrackType int trackType; @NonNull - final LinearClock clock; + LinearClock clock; @NonNull final TrackOutput trackOutput; @@ -56,7 +56,7 @@ public class AviTrack { return getChunkIdLower(id) | ('w' << 16) | ('b' << 24); } - AviTrack(int id, final @C.TrackType int trackType, @NonNull LinearClock clock, + AviTrack(int id, @C.TrackType int trackType, @NonNull LinearClock clock, @NonNull TrackOutput trackOutput) { this.id = id; this.clock = clock; @@ -77,10 +77,15 @@ public class AviTrack { return this.chunkId == chunkId || chunkIdAlt == chunkId; } + @NonNull public LinearClock getClock() { return clock; } + public void setClock(@NonNull LinearClock clock) { + this.clock = clock; + } + public void setChunkPeeker(ChunkPeeker chunkPeeker) { this.chunkPeeker = chunkPeeker; } @@ -144,7 +149,7 @@ public class AviTrack { trackOutput.sampleMetadata( clock.getUs(), (isKeyFrame() ? C.BUFFER_FLAG_KEY_FRAME : 0), size, 0, null); final LinearClock clock = getClock(); -// Log.d(AviExtractor.TAG, "Frame: " + (isVideo()? 'V' : 'A') + " us=" + clock.getUs() + " size=" + size + " frame=" + clock.getIndex() + " key=" + isKeyFrame()); + Log.d(AviExtractor.TAG, "Frame: " + (isVideo()? 'V' : 'A') + " us=" + clock.getUs() + " size=" + size + " frame=" + clock.getIndex() + " key=" + isKeyFrame()); clock.advance(); } } diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/PicCountClock.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/PicCountClock.java index 7e515f786f..050bdf59f9 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/PicCountClock.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/PicCountClock.java @@ -4,14 +4,13 @@ package com.google.android.exoplayer2.extractor.avi; * Properly calculates the frame time for H264 frames using PicCount */ public class PicCountClock extends LinearClock { - //I believe this is 2 because there is a bottom pic order and a top pic order - private static final int STEP = 2; //The frame as a calculated from the picCount private int picIndex; private int lastPicCount; //Largest picFrame, used when we hit an I frame private int maxPicIndex =-1; private int maxPicCount; + private int step = 2; private int posHalf; private int negHalf; @@ -19,15 +18,15 @@ public class PicCountClock extends LinearClock { super(durationUs, length); } - public void setMaxPicCount(int maxPicCount) { + public void setMaxPicCount(int maxPicCount, int step) { this.maxPicCount = maxPicCount; - posHalf = maxPicCount / STEP; + this.step = step; + posHalf = maxPicCount / step; negHalf = -posHalf; } /** - * Done on seek. May cause sync issues if frame picCount != 0 (I frames are always 0) - * @param index + * Used primarily on seek. May cause issues if not a key frame */ @Override public void setIndex(int index) { @@ -42,7 +41,7 @@ public class PicCountClock extends LinearClock { } else if (delta > posHalf) { delta -= maxPicCount; } - picIndex += delta / STEP; + picIndex += delta / step; lastPicCount = picCount; if (maxPicIndex < picIndex) { maxPicIndex = picIndex; diff --git a/library/extractor/src/test/java/com/google/android/exoplayer2/extractor/avi/PicCountClockTest.java b/library/extractor/src/test/java/com/google/android/exoplayer2/extractor/avi/PicCountClockTest.java index c0bc1fad09..2139f13c1a 100644 --- a/library/extractor/src/test/java/com/google/android/exoplayer2/extractor/avi/PicCountClockTest.java +++ b/library/extractor/src/test/java/com/google/android/exoplayer2/extractor/avi/PicCountClockTest.java @@ -7,7 +7,7 @@ public class PicCountClockTest { @Test public void us_givenTwoStepsForward() { final PicCountClock picCountClock = new PicCountClock(10_000L, 100); - picCountClock.setMaxPicCount(16*2); + picCountClock.setMaxPicCount(16*2, 2); picCountClock.setPicCount(2*2); Assert.assertEquals(2*100, picCountClock.getUs()); } @@ -15,7 +15,7 @@ public class PicCountClockTest { @Test public void us_givenThreeStepsBackwards() { final PicCountClock picCountClock = new PicCountClock(10_000L, 100); - picCountClock.setMaxPicCount(16*2); + picCountClock.setMaxPicCount(16*2, 2); picCountClock.setPicCount(4*2); // 400ms Assert.assertEquals(400, picCountClock.getUs()); picCountClock.setPicCount(1*2); @@ -32,7 +32,7 @@ public class PicCountClockTest { @Test public void us_giveWrapBackwards() { final PicCountClock picCountClock = new PicCountClock(10_000L, 100); - picCountClock.setMaxPicCount(16*2); + picCountClock.setMaxPicCount(16*2, 2); //Need to walk up no faster than maxPicCount / 2 picCountClock.setPicCount(7*2); picCountClock.setPicCount(11*2);