1 /*
2  * Copyright (C) 2014 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package android.media.cts;
17 
18 import android.media.AudioTimestamp;
19 import android.media.AudioTrack;
20 import android.media.MediaCodec;
21 import android.media.MediaExtractor;
22 import android.media.MediaFormat;
23 import android.os.Bundle;
24 import android.os.Handler;
25 import android.os.Looper;
26 import android.util.Log;
27 import android.view.Surface;
28 import java.nio.ByteBuffer;
29 import java.util.ArrayList;
30 import java.util.LinkedList;
31 
32 /**
33  * Class for directly managing both audio and video playback by
34  * using {@link MediaCodec} and {@link AudioTrack}.
35  */
36 public class CodecState {
37     private static final String TAG = CodecState.class.getSimpleName();
38 
39     private boolean mSawInputEOS;
40     private volatile boolean mSawOutputEOS;
41     private boolean mLimitQueueDepth;
42     private boolean mTunneled;
43     private boolean mIsAudio;
44     private int mAudioSessionId;
45     private ByteBuffer[] mCodecInputBuffers;
46     private ByteBuffer[] mCodecOutputBuffers;
47     private int mTrackIndex;
48     private int mAvailableInputBufferIndex;
49     private LinkedList<Integer> mAvailableOutputBufferIndices;
50     private LinkedList<MediaCodec.BufferInfo> mAvailableOutputBufferInfos;
51     private volatile long mPresentationTimeUs;
52     private long mFirstSampleTimeUs;
53     private long mPlaybackStartTimeUs;
54     private long mLastPresentTimeUs;
55     private MediaCodec mCodec;
56     private MediaTimeProvider mMediaTimeProvider;
57     private MediaExtractor mExtractor;
58     private MediaFormat mFormat;
59     private MediaFormat mOutputFormat;
60     private NonBlockingAudioTrack mAudioTrack;
61     private volatile OnFrameRenderedListener mOnFrameRenderedListener;
62     /** A list of reported rendered video frames' timestamps. */
63     private ArrayList<Long> mRenderedVideoFrameTimestampList;
64     private boolean mFirstTunnelFrameReady;
65     private volatile OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener;
66 
67 
68     /** If true the video/audio will start from the beginning when it reaches the end. */
69     private boolean mLoopEnabled = false;
70 
71     /**
72      * Manages audio and video playback using MediaCodec and AudioTrack.
73      */
CodecState( MediaTimeProvider mediaTimeProvider, MediaExtractor extractor, int trackIndex, MediaFormat format, MediaCodec codec, boolean limitQueueDepth, boolean tunneled, int audioSessionId)74     public CodecState(
75             MediaTimeProvider mediaTimeProvider,
76             MediaExtractor extractor,
77             int trackIndex,
78             MediaFormat format,
79             MediaCodec codec,
80             boolean limitQueueDepth,
81             boolean tunneled,
82             int audioSessionId) {
83         mMediaTimeProvider = mediaTimeProvider;
84         mExtractor = extractor;
85         mTrackIndex = trackIndex;
86         mFormat = format;
87         mSawInputEOS = mSawOutputEOS = false;
88         mLimitQueueDepth = limitQueueDepth;
89         mTunneled = tunneled;
90         mAudioSessionId = audioSessionId;
91         mFirstSampleTimeUs = -1;
92         mPlaybackStartTimeUs = 0;
93         mLastPresentTimeUs = 0;
94 
95         mCodec = codec;
96 
97         mAvailableInputBufferIndex = -1;
98         mAvailableOutputBufferIndices = new LinkedList<Integer>();
99         mAvailableOutputBufferInfos = new LinkedList<MediaCodec.BufferInfo>();
100         mRenderedVideoFrameTimestampList = new ArrayList<Long>();
101 
102         mPresentationTimeUs = 0;
103 
104         mFirstTunnelFrameReady = false;
105 
106         String mime = mFormat.getString(MediaFormat.KEY_MIME);
107         Log.d(TAG, "CodecState::CodecState " + mime);
108         mIsAudio = mime.startsWith("audio/");
109 
110         if (mTunneled && !mIsAudio) {
111             mOnFrameRenderedListener = new OnFrameRenderedListener();
112             codec.setOnFrameRenderedListener(mOnFrameRenderedListener,
113                                              new Handler(Looper.getMainLooper()));
114             mOnFirstTunnelFrameReadyListener = new OnFirstTunnelFrameReadyListener();
115             codec.setOnFirstTunnelFrameReadyListener(new Handler(Looper.getMainLooper()),
116                     mOnFirstTunnelFrameReadyListener);
117         }
118     }
119 
release()120     public void release() {
121         mCodec.stop();
122         mCodecInputBuffers = null;
123         mCodecOutputBuffers = null;
124         mOutputFormat = null;
125 
126         mAvailableOutputBufferIndices.clear();
127         mAvailableOutputBufferInfos.clear();
128 
129         mAvailableInputBufferIndex = -1;
130         mAvailableOutputBufferIndices = null;
131         mAvailableOutputBufferInfos = null;
132 
133         if (mOnFrameRenderedListener != null) {
134             mCodec.setOnFrameRenderedListener(null, null);
135             mOnFrameRenderedListener = null;
136         }
137         if (mOnFirstTunnelFrameReadyListener != null) {
138             mCodec.setOnFirstTunnelFrameReadyListener(null, null);
139             mOnFirstTunnelFrameReadyListener = null;
140         }
141 
142         mCodec.release();
143         mCodec = null;
144 
145         if (mAudioTrack != null) {
146             mAudioTrack.release();
147             mAudioTrack = null;
148         }
149     }
150 
start()151     public void start() {
152         mCodec.start();
153         mCodecInputBuffers = mCodec.getInputBuffers();
154         if (!mTunneled || mIsAudio) {
155             mCodecOutputBuffers = mCodec.getOutputBuffers();
156         }
157 
158         if (mAudioTrack != null) {
159             mAudioTrack.play();
160         }
161     }
162 
pause()163     public void pause() {
164         if (mAudioTrack != null) {
165             mAudioTrack.pause();
166         }
167     }
168 
getCurrentPositionUs()169     public long getCurrentPositionUs() {
170         return mPresentationTimeUs;
171     }
172 
flush()173     public void flush() {
174         if (!mTunneled || mIsAudio) {
175             mAvailableOutputBufferIndices.clear();
176             mAvailableOutputBufferInfos.clear();
177         }
178 
179         mAvailableInputBufferIndex = -1;
180         mSawInputEOS = false;
181         mSawOutputEOS = false;
182 
183         if (mAudioTrack != null
184                 && mAudioTrack.getPlayState() != AudioTrack.PLAYSTATE_PLAYING) {
185             mAudioTrack.flush();
186         }
187 
188         mCodec.flush();
189         mPresentationTimeUs = 0;
190         mRenderedVideoFrameTimestampList = new ArrayList<Long>();
191         mFirstTunnelFrameReady = false;
192     }
193 
isEnded()194     public boolean isEnded() {
195         return mSawInputEOS && mSawOutputEOS;
196     }
197 
198     /** @see #doSomeWork(Boolean) */
doSomeWork()199     public Long doSomeWork() {
200         return doSomeWork(false /* mustWait */);
201     }
202 
203     /**
204      * {@code doSomeWork} is the worker function that does all buffer handling and decoding works.
205      * It first reads data from {@link MediaExtractor} and pushes it into {@link MediaCodec}; it
206      * then dequeues buffer from {@link MediaCodec}, consumes it and pushes back to its own buffer
207      * queue for next round reading data from {@link MediaExtractor}.
208      *
209      * @param boolean  Whether to block on input buffer retrieval
210      *
211      * @return timestamp of the queued frame, if any.
212      */
doSomeWork(boolean mustWait)213     public Long doSomeWork(boolean mustWait) {
214         // Extract input data, if relevant
215         Long sampleTime = null;
216         if (mAvailableInputBufferIndex == -1) {
217             int indexInput = mCodec.dequeueInputBuffer(mustWait ? -1 : 0 /* timeoutUs */);
218             if (indexInput != MediaCodec.INFO_TRY_AGAIN_LATER) {
219                 mAvailableInputBufferIndex = indexInput;
220             }
221         }
222         if (mAvailableInputBufferIndex != -1) {
223             sampleTime = feedInputBuffer(mAvailableInputBufferIndex);
224             if (sampleTime != null) {
225                 mAvailableInputBufferIndex = -1;
226             }
227         }
228 
229         // Queue output data, if relevant
230         if (mIsAudio || !mTunneled) {
231             MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
232             int indexOutput = mCodec.dequeueOutputBuffer(info, 0 /* timeoutUs */);
233 
234             if (indexOutput == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
235                 mOutputFormat = mCodec.getOutputFormat();
236                 onOutputFormatChanged();
237             } else if (indexOutput == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
238                 mCodecOutputBuffers = mCodec.getOutputBuffers();
239             } else if (indexOutput != MediaCodec.INFO_TRY_AGAIN_LATER) {
240                 mAvailableOutputBufferIndices.add(indexOutput);
241                 mAvailableOutputBufferInfos.add(info);
242             }
243 
244             while (drainOutputBuffer()) {
245             }
246         }
247 
248         return sampleTime;
249     }
250 
setLoopEnabled(boolean enabled)251     public void setLoopEnabled(boolean enabled) {
252         mLoopEnabled = enabled;
253     }
254 
255     /**
256      * Extracts some data from the configured MediaExtractor and feeds it to the configured
257      * MediaCodec.
258      *
259      * Returns the timestamp of the queued buffer, if any.
260      * Returns null once all data has been extracted and queued.
261      */
feedInputBuffer(int inputBufferIndex)262     private Long feedInputBuffer(int inputBufferIndex)
263             throws MediaCodec.CryptoException, IllegalStateException {
264         if (mSawInputEOS || inputBufferIndex == -1) {
265             return null;
266         }
267 
268         // stalls read if audio queue is larger than 2MB full so we will not occupy too much heap
269         if (mLimitQueueDepth && mAudioTrack != null &&
270                 mAudioTrack.getNumBytesQueued() > 2 * 1024 * 1024) {
271             return null;
272         }
273 
274         ByteBuffer codecData = mCodecInputBuffers[inputBufferIndex];
275 
276         int trackIndex = mExtractor.getSampleTrackIndex();
277 
278         if (trackIndex == mTrackIndex) {
279             int sampleSize =
280                 mExtractor.readSampleData(codecData, 0 /* offset */);
281 
282             long sampleTime = mExtractor.getSampleTime();
283 
284             int sampleFlags = mExtractor.getSampleFlags();
285 
286             if (sampleSize <= 0) {
287                 Log.d(TAG, "sampleSize: " + sampleSize + " trackIndex:" + trackIndex +
288                         " sampleTime:" + sampleTime + " sampleFlags:" + sampleFlags);
289                 mSawInputEOS = true;
290                 return null;
291             }
292 
293             if (mTunneled && !mIsAudio) {
294                 if (mFirstSampleTimeUs == -1) {
295                     mFirstSampleTimeUs = sampleTime;
296                 }
297                 sampleTime -= mFirstSampleTimeUs;
298             }
299 
300             mLastPresentTimeUs = mPlaybackStartTimeUs + sampleTime;
301 
302             if ((sampleFlags & MediaExtractor.SAMPLE_FLAG_ENCRYPTED) != 0) {
303                 MediaCodec.CryptoInfo info = new MediaCodec.CryptoInfo();
304                 mExtractor.getSampleCryptoInfo(info);
305 
306                 mCodec.queueSecureInputBuffer(
307                         inputBufferIndex, 0 /* offset */, info, mLastPresentTimeUs, 0 /* flags */);
308             } else {
309                 mCodec.queueInputBuffer(
310                         inputBufferIndex, 0 /* offset */, sampleSize, mLastPresentTimeUs, 0 /* flags */);
311             }
312 
313             mExtractor.advance();
314             return mLastPresentTimeUs;
315         } else if (trackIndex < 0) {
316             Log.d(TAG, "saw input EOS on track " + mTrackIndex);
317 
318             if (mLoopEnabled) {
319                 Log.d(TAG, "looping from the beginning");
320                 mExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
321                 mPlaybackStartTimeUs = mLastPresentTimeUs;
322                 return null;
323             }
324 
325             mSawInputEOS = true;
326             mCodec.queueInputBuffer(
327                     inputBufferIndex, 0 /* offset */, 0 /* sampleSize */,
328                     0 /* sampleTime */, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
329         }
330 
331         return null;
332     }
333 
onOutputFormatChanged()334     private void onOutputFormatChanged() {
335         String mime = mOutputFormat.getString(MediaFormat.KEY_MIME);
336         // b/9250789
337         Log.d(TAG, "CodecState::onOutputFormatChanged " + mime);
338 
339         mIsAudio = false;
340         if (mime.startsWith("audio/")) {
341             mIsAudio = true;
342             int sampleRate =
343                 mOutputFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE);
344 
345             int channelCount =
346                 mOutputFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT);
347 
348             Log.d(TAG, "CodecState::onOutputFormatChanged Audio" +
349                     " sampleRate:" + sampleRate + " channels:" + channelCount);
350             // We do a check here after we receive data from MediaExtractor and before
351             // we pass them down to AudioTrack. If MediaExtractor works properly, this
352             // check is not necessary, however, in our tests, we found that there
353             // are a few cases where ch=0 and samplerate=0 were returned by MediaExtractor.
354             if (channelCount < 1 || channelCount > 8 ||
355                     sampleRate < 8000 || sampleRate > 128000) {
356                 return;
357             }
358             mAudioTrack = new NonBlockingAudioTrack(sampleRate, channelCount,
359                                     mTunneled, mAudioSessionId);
360             mAudioTrack.play();
361         }
362 
363         if (mime.startsWith("video/")) {
364             int width = mOutputFormat.getInteger(MediaFormat.KEY_WIDTH);
365             int height = mOutputFormat.getInteger(MediaFormat.KEY_HEIGHT);
366             Log.d(TAG, "CodecState::onOutputFormatChanged Video" +
367                     " width:" + width + " height:" + height);
368         }
369     }
370 
371     /** Returns true if more output data could be drained. */
drainOutputBuffer()372     private boolean drainOutputBuffer() {
373         if (mSawOutputEOS || mAvailableOutputBufferIndices.isEmpty()) {
374             return false;
375         }
376 
377         int index = mAvailableOutputBufferIndices.peekFirst().intValue();
378         MediaCodec.BufferInfo info = mAvailableOutputBufferInfos.peekFirst();
379 
380         if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
381             Log.d(TAG, "saw output EOS on track " + mTrackIndex);
382 
383             mSawOutputEOS = true;
384 
385             // Do not stop audio track here. Video presentation may not finish
386             // yet, stopping the audio track now would result in getAudioTimeUs
387             // returning 0 and prevent video samples from being presented.
388             // We stop the audio track before the playback thread exits.
389             return false;
390         }
391 
392         long realTimeUs =
393             mMediaTimeProvider.getRealTimeUsForMediaTime(info.presentationTimeUs);
394 
395         long nowUs = mMediaTimeProvider.getNowUs();
396 
397         long lateUs = nowUs - realTimeUs;
398 
399         if (mAudioTrack != null) {
400             ByteBuffer buffer = mCodecOutputBuffers[index];
401             byte[] audioArray = new byte[info.size];
402             buffer.get(audioArray);
403             buffer.clear();
404 
405             mAudioTrack.write(ByteBuffer.wrap(audioArray), info.size,
406                     info.presentationTimeUs*1000);
407 
408             mCodec.releaseOutputBuffer(index, false /* render */);
409 
410             mPresentationTimeUs = info.presentationTimeUs;
411 
412             mAvailableOutputBufferIndices.removeFirst();
413             mAvailableOutputBufferInfos.removeFirst();
414             return true;
415         } else {
416             // video
417             boolean render;
418 
419             if (lateUs < -45000) {
420                 // too early;
421                 return false;
422             } else if (lateUs > 30000) {
423                 Log.d(TAG, "video late by " + lateUs + " us.");
424                 render = false;
425             } else {
426                 render = true;
427                 mPresentationTimeUs = info.presentationTimeUs;
428             }
429 
430             mCodec.releaseOutputBuffer(index, render);
431 
432             mAvailableOutputBufferIndices.removeFirst();
433             mAvailableOutputBufferInfos.removeFirst();
434             return true;
435         }
436     }
437 
438     /** Callback called by the renderer in tunneling mode. */
439     private class OnFrameRenderedListener implements MediaCodec.OnFrameRenderedListener {
440         private static final long TUNNELING_EOS_PRESENTATION_TIME_US = Long.MAX_VALUE;
441 
442         @Override
onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime)443         public void onFrameRendered(MediaCodec codec, long presentationTimeUs, long nanoTime) {
444             if (this != mOnFrameRenderedListener) {
445                 return; // stale event
446             }
447             if (presentationTimeUs == TUNNELING_EOS_PRESENTATION_TIME_US) {
448                  mSawOutputEOS = true;
449             } else {
450                  mPresentationTimeUs = presentationTimeUs;
451             }
452             mRenderedVideoFrameTimestampList.add(presentationTimeUs);
453         }
454     }
455 
getAudioTimeUs()456     public long getAudioTimeUs() {
457         if (mAudioTrack == null) {
458             return 0;
459         }
460 
461         return mAudioTrack.getAudioTimeUs();
462     }
463 
464     /** Callback called in tunnel mode when video peek is ready */
465     private class OnFirstTunnelFrameReadyListener
466         implements MediaCodec.OnFirstTunnelFrameReadyListener {
467 
468         @Override
onFirstTunnelFrameReady(MediaCodec codec)469         public void onFirstTunnelFrameReady(MediaCodec codec) {
470             if (this != mOnFirstTunnelFrameReadyListener) {
471                 return; // stale event
472             }
473             mFirstTunnelFrameReady = true;
474         }
475     }
476 
477     /**
478      * If a video codec, returns the list of rendered frames' timestamps.
479      * Otherwise, returns an empty list.
480      */
getRenderedVideoFrameTimestampList()481     public ArrayList<Long> getRenderedVideoFrameTimestampList() {
482         return new ArrayList<Long>(mRenderedVideoFrameTimestampList);
483     }
484 
485     /** Process the attached {@link AudioTrack}, if any. */
processAudioTrack()486     public void processAudioTrack() {
487         if (mAudioTrack != null) {
488             mAudioTrack.process();
489         }
490     }
491 
getTimestamp()492     public AudioTimestamp getTimestamp() {
493         if (mAudioTrack == null) {
494             return null;
495         }
496 
497         return mAudioTrack.getTimestamp();
498     }
499 
500 
501     /** Stop the attached {@link AudioTrack}, if any. */
stopAudioTrack()502     public void stopAudioTrack() {
503         if (mAudioTrack != null) {
504             mAudioTrack.stop();
505         }
506     }
507 
508     /** Start associated audio track, if any. */
playAudioTrack()509     public void playAudioTrack() {
510         if (mAudioTrack != null) {
511             mAudioTrack.play();
512         }
513     }
514 
setOutputSurface(Surface surface)515     public void setOutputSurface(Surface surface) {
516         if (mAudioTrack != null) {
517             throw new UnsupportedOperationException("Cannot set surface on audio codec");
518         }
519         mCodec.setOutputSurface(surface);
520     }
521 
522     /** Configure video peek. */
setVideoPeek(boolean enable)523     public void setVideoPeek(boolean enable) {
524         Bundle parameters = new Bundle();
525         parameters.putInt(MediaCodec.PARAMETER_KEY_TUNNEL_PEEK, enable ? 1 : 0);
526         mCodec.setParameters(parameters);
527     }
528 
529     /** In tunnel mode, queries whether the first video frame is ready for video peek. */
isFirstTunnelFrameReady()530     public boolean isFirstTunnelFrameReady() {
531         return mFirstTunnelFrameReady;
532     }
533 }
534