diff --git a/RELEASENOTES.md b/RELEASENOTES.md index ddc5d06fdb..1c60071326 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -11,8 +11,11 @@ ([#2843](https://github.com/google/ExoPlayer/issues/2843)). * Fix crash when switching surface on Moto E(4) ([#4134](https://github.com/google/ExoPlayer/issues/4134)). -* Audio: Fix extraction of PCM in MP4/MOV - ([#4228](https://github.com/google/ExoPlayer/issues/4228)). +* Audio: + * Fix extraction of PCM in MP4/MOV + ([#4228](https://github.com/google/ExoPlayer/issues/4228)). + * FLAC: Supports seeking for FLAC files without SEEKTABLE + ([#1808](https://github.com/google/ExoPlayer/issues/1808)). * HLS: * Fix playback of livestreams with EXT-X-PROGRAM-DATE-TIME tags ([#4239](https://github.com/google/ExoPlayer/issues/4239)). diff --git a/extensions/flac/src/androidTest/java/com/google/android/exoplayer2/ext/flac/FlacExtractorSeekTest.java b/extensions/flac/src/androidTest/java/com/google/android/exoplayer2/ext/flac/FlacExtractorSeekTest.java new file mode 100644 index 0000000000..58ab260277 --- /dev/null +++ b/extensions/flac/src/androidTest/java/com/google/android/exoplayer2/ext/flac/FlacExtractorSeekTest.java @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.flac; + +import static com.google.common.truth.Truth.assertThat; + +import android.content.Context; +import android.net.Uri; +import android.support.annotation.Nullable; +import android.test.InstrumentationTestCase; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.extractor.DefaultExtractorInput; +import com.google.android.exoplayer2.extractor.Extractor; +import com.google.android.exoplayer2.extractor.ExtractorInput; +import com.google.android.exoplayer2.extractor.PositionHolder; +import com.google.android.exoplayer2.extractor.SeekMap; +import com.google.android.exoplayer2.testutil.FakeExtractorInput; +import com.google.android.exoplayer2.testutil.FakeExtractorOutput; +import com.google.android.exoplayer2.testutil.FakeTrackOutput; +import com.google.android.exoplayer2.testutil.TestUtil; +import com.google.android.exoplayer2.upstream.DataSpec; +import com.google.android.exoplayer2.upstream.DefaultDataSource; +import com.google.android.exoplayer2.upstream.DefaultDataSourceFactory; +import com.google.android.exoplayer2.util.Util; +import java.io.IOException; +import java.util.List; +import java.util.Random; + +/** Seeking tests for {@link FlacExtractor} when the FLAC stream does not have a SEEKTABLE. */ +public final class FlacExtractorSeekTest extends InstrumentationTestCase { + + private static final String NO_SEEKTABLE_FLAC = "bear_no_seek.flac"; + private static final int DURATION_US = 2_741_000; + private static final Uri FILE_URI = Uri.parse("file:///android_asset/" + NO_SEEKTABLE_FLAC); + private static final Random RANDOM = new Random(1234L); + + private FakeExtractorOutput expectedOutput; + private FakeTrackOutput expectedTrackOutput; + + private DefaultDataSource dataSource; + private PositionHolder positionHolder; + private long totalInputLength; + + @Override + protected void setUp() throws Exception { + super.setUp(); + if (!FlacLibrary.isAvailable()) { + fail("Flac library not available."); + } + expectedOutput = new FakeExtractorOutput(); + extractAllSamplesFromFileToExpectedOutput(getInstrumentation().getContext(), NO_SEEKTABLE_FLAC); + expectedTrackOutput = expectedOutput.trackOutputs.get(0); + + dataSource = + new DefaultDataSourceFactory(getInstrumentation().getContext(), "UserAgent") + .createDataSource(); + totalInputLength = readInputLength(); + positionHolder = new PositionHolder(); + } + + public void testFlacExtractorReads_nonSeekTableFile_returnSeekableSeekMap() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + SeekMap seekMap = extractSeekMap(extractor, new FakeExtractorOutput()); + + assertThat(seekMap).isNotNull(); + assertThat(seekMap.getDurationUs()).isEqualTo(DURATION_US); + assertThat(seekMap.isSeekable()).isTrue(); + } + + public void testHandlePendingSeek_handlesSeekingToPositionInFile_extractsCorrectFrame() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + FakeExtractorOutput extractorOutput = new FakeExtractorOutput(); + SeekMap seekMap = extractSeekMap(extractor, extractorOutput); + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + + long targetSeekTimeUs = 987_000; + int extractedFrameIndex = seekToTimeUs(extractor, seekMap, targetSeekTimeUs, trackOutput); + + assertThat(extractedFrameIndex).isNotEqualTo(-1); + assertFirstFrameAfterSeekContainTargetSeekTime( + trackOutput, targetSeekTimeUs, extractedFrameIndex); + } + + public void testHandlePendingSeek_handlesSeekToEoF_extractsLastFrame() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + FakeExtractorOutput extractorOutput = new FakeExtractorOutput(); + SeekMap seekMap = extractSeekMap(extractor, extractorOutput); + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + + long targetSeekTimeUs = seekMap.getDurationUs(); + + int extractedFrameIndex = seekToTimeUs(extractor, seekMap, targetSeekTimeUs, trackOutput); + + assertThat(extractedFrameIndex).isNotEqualTo(-1); + assertFirstFrameAfterSeekContainTargetSeekTime( + trackOutput, targetSeekTimeUs, extractedFrameIndex); + } + + public void testHandlePendingSeek_handlesSeekingBackward_extractsCorrectFrame() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + FakeExtractorOutput extractorOutput = new FakeExtractorOutput(); + SeekMap seekMap = extractSeekMap(extractor, extractorOutput); + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + + long firstSeekTimeUs = 987_000; + seekToTimeUs(extractor, seekMap, firstSeekTimeUs, trackOutput); + + long targetSeekTimeUs = 0; + int extractedFrameIndex = seekToTimeUs(extractor, seekMap, targetSeekTimeUs, trackOutput); + + assertThat(extractedFrameIndex).isNotEqualTo(-1); + assertFirstFrameAfterSeekContainTargetSeekTime( + trackOutput, targetSeekTimeUs, extractedFrameIndex); + } + + public void testHandlePendingSeek_handlesSeekingForward_extractsCorrectFrame() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + FakeExtractorOutput extractorOutput = new FakeExtractorOutput(); + SeekMap seekMap = extractSeekMap(extractor, extractorOutput); + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + + long firstSeekTimeUs = 987_000; + seekToTimeUs(extractor, seekMap, firstSeekTimeUs, trackOutput); + + long targetSeekTimeUs = 1_234_000; + int extractedFrameIndex = seekToTimeUs(extractor, seekMap, targetSeekTimeUs, trackOutput); + + assertThat(extractedFrameIndex).isNotEqualTo(-1); + assertFirstFrameAfterSeekContainTargetSeekTime( + trackOutput, targetSeekTimeUs, extractedFrameIndex); + } + + public void testHandlePendingSeek_handlesRandomSeeks_extractsCorrectFrame() + throws IOException, InterruptedException { + FlacExtractor extractor = new FlacExtractor(); + + FakeExtractorOutput extractorOutput = new FakeExtractorOutput(); + SeekMap seekMap = extractSeekMap(extractor, extractorOutput); + FakeTrackOutput trackOutput = extractorOutput.trackOutputs.get(0); + + long numSeek = 100; + for (long i = 0; i < numSeek; i++) { + long targetSeekTimeUs = RANDOM.nextInt(DURATION_US + 1); + int extractedFrameIndex = seekToTimeUs(extractor, seekMap, targetSeekTimeUs, trackOutput); + + assertThat(extractedFrameIndex).isNotEqualTo(-1); + assertFirstFrameAfterSeekContainTargetSeekTime( + trackOutput, targetSeekTimeUs, extractedFrameIndex); + } + } + + // Internal methods + + private long readInputLength() throws IOException { + DataSpec dataSpec = new DataSpec(FILE_URI, 0, C.LENGTH_UNSET, null); + long totalInputLength = dataSource.open(dataSpec); + Util.closeQuietly(dataSource); + return totalInputLength; + } + + /** + * Seeks to the given seek time and keeps reading from input until we can extract at least one + * frame from the seek position, or until end-of-input is reached. + * + * @return The index of the first extracted frame written to the given {@code trackOutput} after + * the seek is completed, or -1 if the seek is completed without any extracted frame. + */ + private int seekToTimeUs( + FlacExtractor flacExtractor, SeekMap seekMap, long seekTimeUs, FakeTrackOutput trackOutput) + throws IOException, InterruptedException { + int numSampleBeforeSeek = trackOutput.getSampleCount(); + SeekMap.SeekPoints seekPoints = seekMap.getSeekPoints(seekTimeUs); + + long initialSeekLoadPosition = seekPoints.first.position; + flacExtractor.seek(initialSeekLoadPosition, seekTimeUs); + + positionHolder.position = C.POSITION_UNSET; + ExtractorInput extractorInput = getExtractorInputFromPosition(initialSeekLoadPosition); + int extractorReadResult = Extractor.RESULT_CONTINUE; + while (true) { + try { + // Keep reading until we can read at least one frame after seek + while (extractorReadResult == Extractor.RESULT_CONTINUE + && trackOutput.getSampleCount() == numSampleBeforeSeek) { + extractorReadResult = flacExtractor.read(extractorInput, positionHolder); + } + } finally { + Util.closeQuietly(dataSource); + } + + if (extractorReadResult == Extractor.RESULT_SEEK) { + extractorInput = getExtractorInputFromPosition(positionHolder.position); + extractorReadResult = Extractor.RESULT_CONTINUE; + } else if (extractorReadResult == Extractor.RESULT_END_OF_INPUT) { + return -1; + } else if (trackOutput.getSampleCount() > numSampleBeforeSeek) { + // First index after seek = num sample before seek. + return numSampleBeforeSeek; + } + } + } + + private @Nullable SeekMap extractSeekMap(FlacExtractor extractor, FakeExtractorOutput output) + throws IOException, InterruptedException { + try { + ExtractorInput input = getExtractorInputFromPosition(0); + extractor.init(output); + while (output.seekMap == null) { + extractor.read(input, positionHolder); + } + } finally { + Util.closeQuietly(dataSource); + } + return output.seekMap; + } + + private void assertFirstFrameAfterSeekContainTargetSeekTime( + FakeTrackOutput trackOutput, long seekTimeUs, int firstFrameIndexAfterSeek) { + int expectedSampleIndex = findTargetFrameInExpectedOutput(seekTimeUs); + // Assert that after seeking, the first sample frame written to output contains the sample + // at seek time. + trackOutput.assertSample( + firstFrameIndexAfterSeek, + expectedTrackOutput.getSampleData(expectedSampleIndex), + expectedTrackOutput.getSampleTimeUs(expectedSampleIndex), + expectedTrackOutput.getSampleFlags(expectedSampleIndex), + expectedTrackOutput.getSampleCryptoData(expectedSampleIndex)); + } + + private int findTargetFrameInExpectedOutput(long seekTimeUs) { + List sampleTimes = expectedTrackOutput.getSampleTimesUs(); + for (int i = 0; i < sampleTimes.size() - 1; i++) { + long currentSampleTime = sampleTimes.get(i); + long nextSampleTime = sampleTimes.get(i + 1); + if (currentSampleTime <= seekTimeUs && nextSampleTime > seekTimeUs) { + return i; + } + } + return sampleTimes.size() - 1; + } + + private ExtractorInput getExtractorInputFromPosition(long position) throws IOException { + DataSpec dataSpec = new DataSpec(FILE_URI, position, totalInputLength, /* key= */ null); + dataSource.open(dataSpec); + return new DefaultExtractorInput(dataSource, position, totalInputLength); + } + + private void extractAllSamplesFromFileToExpectedOutput(Context context, String fileName) + throws IOException, InterruptedException { + byte[] data = TestUtil.getByteArray(context, fileName); + + FlacExtractor extractor = new FlacExtractor(); + extractor.init(expectedOutput); + FakeExtractorInput input = new FakeExtractorInput.Builder().setData(data).build(); + + while (extractor.read(input, new PositionHolder()) != Extractor.RESULT_END_OF_INPUT) {} + } +} diff --git a/extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/FlacExtractor.java b/extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/FlacExtractor.java index 7672f2f8ec..a5efeb69f9 100644 --- a/extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/FlacExtractor.java +++ b/extensions/flac/src/main/java/com/google/android/exoplayer2/ext/flac/FlacExtractor.java @@ -88,10 +88,12 @@ public final class FlacExtractor implements Extractor { private ParsableByteArray outputBuffer; private ByteBuffer outputByteBuffer; + private FlacStreamInfo streamInfo; private Metadata id3Metadata; + private @Nullable FlacBinarySearchSeeker flacBinarySearchSeeker; - private boolean metadataParsed; + private boolean readPastStreamInfo; /** Constructs an instance with flags = 0. */ public FlacExtractor() { @@ -136,47 +138,10 @@ public final class FlacExtractor implements Extractor { } decoderJni.setData(input); + readPastStreamInfo(input); - if (!metadataParsed) { - final FlacStreamInfo streamInfo; - try { - streamInfo = decoderJni.decodeMetadata(); - if (streamInfo == null) { - throw new IOException("Metadata decoding failed"); - } - } catch (IOException e) { - decoderJni.reset(0); - input.setRetryPosition(0, e); - throw e; // never executes - } - metadataParsed = true; - - boolean isSeekable = decoderJni.getSeekPosition(0) != -1; - extractorOutput.seekMap( - isSeekable - ? new FlacSeekMap(streamInfo.durationUs(), decoderJni) - : new SeekMap.Unseekable(streamInfo.durationUs(), 0)); - Format mediaFormat = - Format.createAudioSampleFormat( - /* id= */ null, - MimeTypes.AUDIO_RAW, - /* codecs= */ null, - streamInfo.bitRate(), - streamInfo.maxDecodedFrameSize(), - streamInfo.channels, - streamInfo.sampleRate, - getPcmEncoding(streamInfo.bitsPerSample), - /* encoderDelay= */ 0, - /* encoderPadding= */ 0, - /* initializationData= */ null, - /* drmInitData= */ null, - /* selectionFlags= */ 0, - /* language= */ null, - isId3MetadataDisabled ? null : id3Metadata); - trackOutput.format(mediaFormat); - - outputBuffer = new ParsableByteArray(streamInfo.maxDecodedFrameSize()); - outputByteBuffer = ByteBuffer.wrap(outputBuffer.data); + if (flacBinarySearchSeeker != null && flacBinarySearchSeeker.hasPendingSeek()) { + return handlePendingSeek(input, seekPosition); } long lastDecodePosition = decoderJni.getDecodePosition(); @@ -189,26 +154,27 @@ public final class FlacExtractor implements Extractor { if (outputSize == 0) { return RESULT_END_OF_INPUT; } - outputBuffer.setPosition(0); - trackOutput.sampleData(outputBuffer, outputSize); - trackOutput.sampleMetadata( - decoderJni.getLastFrameTimestamp(), C.BUFFER_FLAG_KEY_FRAME, outputSize, 0, null); + writeLastSampleToOutput(outputSize, decoderJni.getLastFrameTimestamp()); return decoderJni.isEndOfData() ? RESULT_END_OF_INPUT : RESULT_CONTINUE; } @Override public void seek(long position, long timeUs) { if (position == 0) { - metadataParsed = false; + readPastStreamInfo = false; } if (decoderJni != null) { decoderJni.reset(position); } + if (flacBinarySearchSeeker != null) { + flacBinarySearchSeeker.setSeekTargetUs(timeUs); + } } @Override public void release() { + flacBinarySearchSeeker = null; if (decoderJni != null) { decoderJni.release(); decoderJni = null; @@ -240,6 +206,100 @@ public final class FlacExtractor implements Extractor { return Arrays.equals(header, FLAC_SIGNATURE); } + private void readPastStreamInfo(ExtractorInput input) throws InterruptedException, IOException { + if (readPastStreamInfo) { + return; + } + + FlacStreamInfo streamInfo = decodeStreamInfo(input); + readPastStreamInfo = true; + if (this.streamInfo == null) { + updateFlacStreamInfo(input, streamInfo); + } + } + + private void updateFlacStreamInfo(ExtractorInput input, FlacStreamInfo streamInfo) { + this.streamInfo = streamInfo; + outputSeekMap(input, streamInfo); + outputFormat(streamInfo); + outputBuffer = new ParsableByteArray(streamInfo.maxDecodedFrameSize()); + outputByteBuffer = ByteBuffer.wrap(outputBuffer.data); + } + + private FlacStreamInfo decodeStreamInfo(ExtractorInput input) + throws InterruptedException, IOException { + try { + FlacStreamInfo streamInfo = decoderJni.decodeMetadata(); + if (streamInfo == null) { + throw new IOException("Metadata decoding failed"); + } + return streamInfo; + } catch (IOException e) { + decoderJni.reset(0); + input.setRetryPosition(0, e); + throw e; + } + } + + private void outputSeekMap(ExtractorInput input, FlacStreamInfo streamInfo) { + boolean hasSeekTable = decoderJni.getSeekPosition(0) != -1; + SeekMap seekMap = + hasSeekTable + ? new FlacSeekMap(streamInfo.durationUs(), decoderJni) + : getSeekMapForNonSeekTableFlac(input, streamInfo); + extractorOutput.seekMap(seekMap); + } + + private SeekMap getSeekMapForNonSeekTableFlac(ExtractorInput input, FlacStreamInfo streamInfo) { + long inputLength = input.getLength(); + if (inputLength != C.LENGTH_UNSET) { + long firstFramePosition = decoderJni.getDecodePosition(); + flacBinarySearchSeeker = + new FlacBinarySearchSeeker(streamInfo, firstFramePosition, inputLength, decoderJni); + return flacBinarySearchSeeker.getSeekMap(); + } else { // can't seek at all, because there's no SeekTable and the input length is unknown. + return new SeekMap.Unseekable(streamInfo.durationUs()); + } + } + + private void outputFormat(FlacStreamInfo streamInfo) { + Format mediaFormat = + Format.createAudioSampleFormat( + /* id= */ null, + MimeTypes.AUDIO_RAW, + /* codecs= */ null, + streamInfo.bitRate(), + streamInfo.maxDecodedFrameSize(), + streamInfo.channels, + streamInfo.sampleRate, + getPcmEncoding(streamInfo.bitsPerSample), + /* encoderDelay= */ 0, + /* encoderPadding= */ 0, + /* initializationData= */ null, + /* drmInitData= */ null, + /* selectionFlags= */ 0, + /* language= */ null, + isId3MetadataDisabled ? null : id3Metadata); + trackOutput.format(mediaFormat); + } + + private int handlePendingSeek(ExtractorInput input, PositionHolder seekPosition) + throws InterruptedException, IOException { + int seekResult = + flacBinarySearchSeeker.handlePendingSeek(input, seekPosition, outputByteBuffer); + if (seekResult == RESULT_CONTINUE && outputByteBuffer.limit() > 0) { + writeLastSampleToOutput(outputByteBuffer.limit(), decoderJni.getLastFrameTimestamp()); + } + return seekResult; + } + + private void writeLastSampleToOutput(int size, long lastSampleTimestamp) { + outputBuffer.setPosition(0); + trackOutput.sampleData(outputBuffer, size); + trackOutput.sampleMetadata(lastSampleTimestamp, C.BUFFER_FLAG_KEY_FRAME, size, 0, null); + } + + /** A {@link SeekMap} implementation using a SeekTable within the Flac stream. */ private static final class FlacSeekMap implements SeekMap { private final long durationUs; diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeTrackOutput.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeTrackOutput.java index 639cb82c2d..6432842df4 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeTrackOutput.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/FakeTrackOutput.java @@ -26,6 +26,8 @@ import java.io.EOFException; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; /** * A fake {@link TrackOutput}. @@ -114,6 +116,26 @@ public final class FakeTrackOutput implements TrackOutput, Dumper.Dumpable { sampleEndOffsets.get(index)); } + public long getSampleTimeUs(int index) { + return sampleTimesUs.get(index); + } + + public int getSampleFlags(int index) { + return sampleFlags.get(index); + } + + public CryptoData getSampleCryptoData(int index) { + return cryptoDatas.get(index); + } + + public int getSampleCount() { + return sampleTimesUs.size(); + } + + public List getSampleTimesUs() { + return Collections.unmodifiableList(sampleTimesUs); + } + public void assertEquals(FakeTrackOutput expected) { assertThat(format).isEqualTo(expected.format); assertThat(sampleTimesUs).hasSize(expected.sampleTimesUs.size());