1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.media.cts;
18 
19 import com.android.cts.media.R;
20 
21 import android.content.Context;
22 import android.content.pm.PackageManager;
23 import android.content.res.AssetFileDescriptor;
24 import android.content.res.Resources;
25 import android.cts.util.MediaUtils;
26 import android.graphics.ImageFormat;
27 import android.media.Image;
28 import android.media.AudioManager;
29 import android.media.MediaCodec;
30 import android.media.MediaCodecList;
31 import android.media.MediaCodecInfo;
32 import android.media.MediaCodecInfo.CodecCapabilities;
33 import android.media.MediaExtractor;
34 import android.media.MediaFormat;
35 import android.util.Log;
36 import android.view.Surface;
37 import android.net.Uri;
38 
39 import java.io.BufferedInputStream;
40 import java.io.IOException;
41 import java.io.InputStream;
42 import java.nio.ByteBuffer;
43 import java.util.ArrayList;
44 import java.util.Arrays;
45 import java.util.List;
46 import java.util.zip.CRC32;
47 import java.util.concurrent.TimeUnit;
48 
49 public class DecoderTest extends MediaPlayerTestBase {
50     private static final String TAG = "DecoderTest";
51 
52     private static final int RESET_MODE_NONE = 0;
53     private static final int RESET_MODE_RECONFIGURE = 1;
54     private static final int RESET_MODE_FLUSH = 2;
55     private static final int RESET_MODE_EOS_FLUSH = 3;
56 
57     private static final String[] CSD_KEYS = new String[] { "csd-0", "csd-1" };
58 
59     private static final int CONFIG_MODE_NONE = 0;
60     private static final int CONFIG_MODE_QUEUE = 1;
61 
62     private Resources mResources;
63     short[] mMasterBuffer;
64 
65     private MediaCodecTunneledPlayer mMediaCodecPlayer;
66     private static final int SLEEP_TIME_MS = 1000;
67     private static final long PLAY_TIME_MS = TimeUnit.MILLISECONDS.convert(1, TimeUnit.MINUTES);
68     private static final Uri AUDIO_URL = Uri.parse(
69             "http://redirector.c.youtube.com/videoplayback?id=c80658495af60617"
70                 + "&itag=18&source=youtube&ip=0.0.0.0&ipbits=0&expire=19000000000"
71                 + "&sparams=ip,ipbits,expire,id,itag,source"
72                 + "&signature=46A04ED550CA83B79B60060BA80C79FDA5853D26."
73                 + "49582D382B4A9AFAA163DED38D2AE531D85603C0"
74                 + "&key=ik0&user=android-device-test");  // H.264 Base + AAC
75     private static final Uri VIDEO_URL = Uri.parse(
76             "http://redirector.c.youtube.com/videoplayback?id=c80658495af60617"
77                 + "&itag=18&source=youtube&ip=0.0.0.0&ipbits=0&expire=19000000000"
78                 + "&sparams=ip,ipbits,expire,id,itag,source"
79                 + "&signature=46A04ED550CA83B79B60060BA80C79FDA5853D26."
80                 + "49582D382B4A9AFAA163DED38D2AE531D85603C0"
81                 + "&key=ik0&user=android-device-test");  // H.264 Base + AAC
82 
83     @Override
setUp()84     protected void setUp() throws Exception {
85         super.setUp();
86         mResources = mContext.getResources();
87 
88         // read master file into memory
89         AssetFileDescriptor masterFd = mResources.openRawResourceFd(R.raw.sinesweepraw);
90         long masterLength = masterFd.getLength();
91         mMasterBuffer = new short[(int) (masterLength / 2)];
92         InputStream is = masterFd.createInputStream();
93         BufferedInputStream bis = new BufferedInputStream(is);
94         for (int i = 0; i < mMasterBuffer.length; i++) {
95             int lo = bis.read();
96             int hi = bis.read();
97             if (hi >= 128) {
98                 hi -= 256;
99             }
100             int sample = hi * 256 + lo;
101             mMasterBuffer[i] = (short) sample;
102         }
103         bis.close();
104         masterFd.close();
105     }
106 
107     // TODO: add similar tests for other audio and video formats
testBug11696552()108     public void testBug11696552() throws Exception {
109         MediaCodec mMediaCodec = MediaCodec.createDecoderByType(MediaFormat.MIMETYPE_AUDIO_AAC);
110         MediaFormat mFormat = MediaFormat.createAudioFormat(
111                 MediaFormat.MIMETYPE_AUDIO_AAC, 48000 /* frequency */, 2 /* channels */);
112         mFormat.setByteBuffer("csd-0", ByteBuffer.wrap( new byte [] {0x13, 0x10} ));
113         mFormat.setInteger(MediaFormat.KEY_IS_ADTS, 1);
114         mMediaCodec.configure(mFormat, null, null, 0);
115         mMediaCodec.start();
116         int index = mMediaCodec.dequeueInputBuffer(250000);
117         mMediaCodec.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
118         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
119         mMediaCodec.dequeueOutputBuffer(info, 250000);
120     }
121 
122     // The allowed errors in the following tests are the actual maximum measured
123     // errors with the standard decoders, plus 10%.
124     // This should allow for some variation in decoders, while still detecting
125     // phase and delay errors, channel swap, etc.
testDecodeMp3Lame()126     public void testDecodeMp3Lame() throws Exception {
127         decode(R.raw.sinesweepmp3lame, 804.f);
128         testTimeStampOrdering(R.raw.sinesweepmp3lame);
129     }
testDecodeMp3Smpb()130     public void testDecodeMp3Smpb() throws Exception {
131         decode(R.raw.sinesweepmp3smpb, 413.f);
132         testTimeStampOrdering(R.raw.sinesweepmp3smpb);
133     }
testDecodeM4a()134     public void testDecodeM4a() throws Exception {
135         decode(R.raw.sinesweepm4a, 124.f);
136         testTimeStampOrdering(R.raw.sinesweepm4a);
137     }
testDecodeOgg()138     public void testDecodeOgg() throws Exception {
139         decode(R.raw.sinesweepogg, 168.f);
140         testTimeStampOrdering(R.raw.sinesweepogg);
141     }
testDecodeWav()142     public void testDecodeWav() throws Exception {
143         decode(R.raw.sinesweepwav, 0.0f);
144         testTimeStampOrdering(R.raw.sinesweepwav);
145     }
testDecodeFlac()146     public void testDecodeFlac() throws Exception {
147         decode(R.raw.sinesweepflac, 0.0f);
148         testTimeStampOrdering(R.raw.sinesweepflac);
149     }
150 
testDecodeMonoMp3()151     public void testDecodeMonoMp3() throws Exception {
152         monoTest(R.raw.monotestmp3, 44100);
153         testTimeStampOrdering(R.raw.monotestmp3);
154     }
155 
testDecodeMonoM4a()156     public void testDecodeMonoM4a() throws Exception {
157         monoTest(R.raw.monotestm4a, 44100);
158         testTimeStampOrdering(R.raw.monotestm4a);
159     }
160 
testDecodeMonoOgg()161     public void testDecodeMonoOgg() throws Exception {
162         monoTest(R.raw.monotestogg, 44100);
163         testTimeStampOrdering(R.raw.monotestogg);
164     }
165 
testDecodeMonoGsm()166     public void testDecodeMonoGsm() throws Exception {
167         if (MediaUtils.hasCodecsForResource(mContext, R.raw.monotestgsm)) {
168             monoTest(R.raw.monotestgsm, 8000);
169             testTimeStampOrdering(R.raw.monotestgsm);
170         } else {
171             MediaUtils.skipTest("not mandatory");
172         }
173     }
174 
testDecodeAacTs()175     public void testDecodeAacTs() throws Exception {
176         testTimeStampOrdering(R.raw.sinesweeptsaac);
177     }
178 
testDecode51M4a()179     public void testDecode51M4a() throws Exception {
180         decodeToMemory(R.raw.sinesweep51m4a, RESET_MODE_NONE, CONFIG_MODE_NONE, -1, null);
181     }
182 
testTimeStampOrdering(int res)183     private void testTimeStampOrdering(int res) throws Exception {
184         List<Long> timestamps = new ArrayList<Long>();
185         decodeToMemory(res, RESET_MODE_NONE, CONFIG_MODE_NONE, -1, timestamps);
186         Long lastTime = Long.MIN_VALUE;
187         for (int i = 0; i < timestamps.size(); i++) {
188             Long thisTime = timestamps.get(i);
189             assertTrue("timetravel occurred: " + lastTime + " > " + thisTime, thisTime >= lastTime);
190             lastTime = thisTime;
191         }
192     }
193 
testTrackSelection()194     public void testTrackSelection() throws Exception {
195         testTrackSelection(R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz);
196         testTrackSelection(
197                 R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz_fragmented);
198         testTrackSelection(
199                 R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz_dash);
200     }
201 
testBFrames()202     public void testBFrames() throws Exception {
203         int testsRun =
204             testBFrames(R.raw.video_h264_main_b_frames) +
205             testBFrames(R.raw.video_h264_main_b_frames_frag);
206         if (testsRun == 0) {
207             MediaUtils.skipTest("no codec found");
208         }
209     }
210 
testBFrames(int res)211     public int testBFrames(int res) throws Exception {
212         AssetFileDescriptor fd = mResources.openRawResourceFd(res);
213         MediaExtractor ex = new MediaExtractor();
214         ex.setDataSource(fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength());
215         MediaFormat format = ex.getTrackFormat(0);
216         String mime = format.getString(MediaFormat.KEY_MIME);
217         assertTrue("not a video track. Wrong test file?", mime.startsWith("video/"));
218         if (!MediaUtils.canDecode(format)) {
219             ex.release();
220             fd.close();
221             return 0; // skip
222         }
223         MediaCodec dec = MediaCodec.createDecoderByType(mime);
224         Surface s = getActivity().getSurfaceHolder().getSurface();
225         dec.configure(format, s, null, 0);
226         dec.start();
227         ByteBuffer[] buf = dec.getInputBuffers();
228         ex.selectTrack(0);
229         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
230         long lastPresentationTimeUsFromExtractor = -1;
231         long lastPresentationTimeUsFromDecoder = -1;
232         boolean inputoutoforder = false;
233         while(true) {
234             int flags = ex.getSampleFlags();
235             long time = ex.getSampleTime();
236             if (time >= 0 && time < lastPresentationTimeUsFromExtractor) {
237                 inputoutoforder = true;
238             }
239             lastPresentationTimeUsFromExtractor = time;
240             int bufidx = dec.dequeueInputBuffer(5000);
241             if (bufidx >= 0) {
242                 int n = ex.readSampleData(buf[bufidx], 0);
243                 if (n < 0) {
244                     flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM;
245                     time = 0;
246                     n = 0;
247                 }
248                 dec.queueInputBuffer(bufidx, 0, n, time, flags);
249                 ex.advance();
250             }
251             int status = dec.dequeueOutputBuffer(info, 5000);
252             if (status >= 0) {
253                 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
254                     break;
255                 }
256                 assertTrue("out of order timestamp from decoder",
257                         info.presentationTimeUs > lastPresentationTimeUsFromDecoder);
258                 dec.releaseOutputBuffer(status, true);
259                 lastPresentationTimeUsFromDecoder = info.presentationTimeUs;
260             }
261         }
262         assertTrue("extractor timestamps were ordered, wrong test file?", inputoutoforder);
263         dec.release();
264         ex.release();
265         fd.close();
266         return 1;
267       }
268 
testTrackSelection(int resid)269     private void testTrackSelection(int resid) throws Exception {
270         AssetFileDescriptor fd1 = null;
271         try {
272             fd1 = mResources.openRawResourceFd(resid);
273             MediaExtractor ex1 = new MediaExtractor();
274             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
275 
276             ByteBuffer buf1 = ByteBuffer.allocate(1024*1024);
277             ArrayList<Integer> vid = new ArrayList<Integer>();
278             ArrayList<Integer> aud = new ArrayList<Integer>();
279 
280             // scan the file once and build lists of audio and video samples
281             ex1.selectTrack(0);
282             ex1.selectTrack(1);
283             while(true) {
284                 int n1 = ex1.readSampleData(buf1, 0);
285                 if (n1 < 0) {
286                     break;
287                 }
288                 int idx = ex1.getSampleTrackIndex();
289                 if (idx == 0) {
290                     vid.add(n1);
291                 } else if (idx == 1) {
292                     aud.add(n1);
293                 } else {
294                     fail("unexpected track index: " + idx);
295                 }
296                 ex1.advance();
297             }
298 
299             // read the video track once, then rewind and do it again, and
300             // verify we get the right samples
301             ex1.release();
302             ex1 = new MediaExtractor();
303             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
304             ex1.selectTrack(0);
305             for (int i = 0; i < 2; i++) {
306                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
307                 int idx = 0;
308                 while(true) {
309                     int n1 = ex1.readSampleData(buf1, 0);
310                     if (n1 < 0) {
311                         assertEquals(vid.size(), idx);
312                         break;
313                     }
314                     assertEquals(vid.get(idx++).intValue(), n1);
315                     ex1.advance();
316                 }
317             }
318 
319             // read the audio track once, then rewind and do it again, and
320             // verify we get the right samples
321             ex1.release();
322             ex1 = new MediaExtractor();
323             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
324             ex1.selectTrack(1);
325             for (int i = 0; i < 2; i++) {
326                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
327                 int idx = 0;
328                 while(true) {
329                     int n1 = ex1.readSampleData(buf1, 0);
330                     if (n1 < 0) {
331                         assertEquals(aud.size(), idx);
332                         break;
333                     }
334                     assertEquals(aud.get(idx++).intValue(), n1);
335                     ex1.advance();
336                 }
337             }
338 
339             // read the video track first, then rewind and get the audio track instead, and
340             // verify we get the right samples
341             ex1.release();
342             ex1 = new MediaExtractor();
343             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
344             for (int i = 0; i < 2; i++) {
345                 ex1.selectTrack(i);
346                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
347                 int idx = 0;
348                 while(true) {
349                     int n1 = ex1.readSampleData(buf1, 0);
350                     if (i == 0) {
351                         if (n1 < 0) {
352                             assertEquals(vid.size(), idx);
353                             break;
354                         }
355                         assertEquals(vid.get(idx++).intValue(), n1);
356                     } else if (i == 1) {
357                         if (n1 < 0) {
358                             assertEquals(aud.size(), idx);
359                             break;
360                         }
361                         assertEquals(aud.get(idx++).intValue(), n1);
362                     } else {
363                         fail("unexpected track index: " + idx);
364                     }
365                     ex1.advance();
366                 }
367                 ex1.unselectTrack(i);
368             }
369 
370             // read the video track first, then rewind, enable the audio track in addition
371             // to the video track, and verify we get the right samples
372             ex1.release();
373             ex1 = new MediaExtractor();
374             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
375             for (int i = 0; i < 2; i++) {
376                 ex1.selectTrack(i);
377                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
378                 int vididx = 0;
379                 int audidx = 0;
380                 while(true) {
381                     int n1 = ex1.readSampleData(buf1, 0);
382                     if (n1 < 0) {
383                         // we should have read all audio and all video samples at this point
384                         assertEquals(vid.size(), vididx);
385                         if (i == 1) {
386                             assertEquals(aud.size(), audidx);
387                         }
388                         break;
389                     }
390                     int trackidx = ex1.getSampleTrackIndex();
391                     if (trackidx == 0) {
392                         assertEquals(vid.get(vididx++).intValue(), n1);
393                     } else if (trackidx == 1) {
394                         assertEquals(aud.get(audidx++).intValue(), n1);
395                     } else {
396                         fail("unexpected track index: " + trackidx);
397                     }
398                     ex1.advance();
399                 }
400             }
401 
402             // read both tracks from the start, then rewind and verify we get the right
403             // samples both times
404             ex1.release();
405             ex1 = new MediaExtractor();
406             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
407             for (int i = 0; i < 2; i++) {
408                 ex1.selectTrack(0);
409                 ex1.selectTrack(1);
410                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
411                 int vididx = 0;
412                 int audidx = 0;
413                 while(true) {
414                     int n1 = ex1.readSampleData(buf1, 0);
415                     if (n1 < 0) {
416                         // we should have read all audio and all video samples at this point
417                         assertEquals(vid.size(), vididx);
418                         assertEquals(aud.size(), audidx);
419                         break;
420                     }
421                     int trackidx = ex1.getSampleTrackIndex();
422                     if (trackidx == 0) {
423                         assertEquals(vid.get(vididx++).intValue(), n1);
424                     } else if (trackidx == 1) {
425                         assertEquals(aud.get(audidx++).intValue(), n1);
426                     } else {
427                         fail("unexpected track index: " + trackidx);
428                     }
429                     ex1.advance();
430                 }
431             }
432 
433         } finally {
434             if (fd1 != null) {
435                 fd1.close();
436             }
437         }
438     }
439 
testDecodeFragmented()440     public void testDecodeFragmented() throws Exception {
441         testDecodeFragmented(R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz,
442                 R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz_fragmented);
443         testDecodeFragmented(R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz,
444                 R.raw.video_480x360_mp4_h264_1350kbps_30fps_aac_stereo_128kbps_44100hz_dash);
445     }
446 
testDecodeFragmented(int reference, int teststream)447     private void testDecodeFragmented(int reference, int teststream) throws Exception {
448         AssetFileDescriptor fd1 = null;
449         AssetFileDescriptor fd2 = null;
450         try {
451             fd1 = mResources.openRawResourceFd(reference);
452             MediaExtractor ex1 = new MediaExtractor();
453             ex1.setDataSource(fd1.getFileDescriptor(), fd1.getStartOffset(), fd1.getLength());
454 
455             fd2 = mResources.openRawResourceFd(teststream);
456             MediaExtractor ex2 = new MediaExtractor();
457             ex2.setDataSource(fd2.getFileDescriptor(), fd2.getStartOffset(), fd2.getLength());
458 
459             assertEquals("different track count", ex1.getTrackCount(), ex2.getTrackCount());
460 
461             ByteBuffer buf1 = ByteBuffer.allocate(1024*1024);
462             ByteBuffer buf2 = ByteBuffer.allocate(1024*1024);
463 
464             for (int i = 0; i < ex1.getTrackCount(); i++) {
465                 // note: this assumes the tracks are reported in the order in which they appear
466                 // in the file.
467                 ex1.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
468                 ex1.selectTrack(i);
469                 ex2.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
470                 ex2.selectTrack(i);
471 
472                 while(true) {
473                     int n1 = ex1.readSampleData(buf1, 0);
474                     int n2 = ex2.readSampleData(buf2, 0);
475                     assertEquals("different buffer size on track " + i, n1, n2);
476 
477                     if (n1 < 0) {
478                         break;
479                     }
480                     // see bug 13008204
481                     buf1.limit(n1);
482                     buf2.limit(n2);
483                     buf1.rewind();
484                     buf2.rewind();
485 
486                     assertEquals("limit does not match return value on track " + i,
487                             n1, buf1.limit());
488                     assertEquals("limit does not match return value on track " + i,
489                             n2, buf2.limit());
490 
491                     assertEquals("buffer data did not match on track " + i, buf1, buf2);
492 
493                     ex1.advance();
494                     ex2.advance();
495                 }
496                 ex1.unselectTrack(i);
497                 ex2.unselectTrack(i);
498             }
499         } finally {
500             if (fd1 != null) {
501                 fd1.close();
502             }
503             if (fd2 != null) {
504                 fd2.close();
505             }
506         }
507     }
508 
509 
monoTest(int res, int expectedLength)510     private void monoTest(int res, int expectedLength) throws Exception {
511         short [] mono = decodeToMemory(res, RESET_MODE_NONE, CONFIG_MODE_NONE, -1, null);
512         if (mono.length == expectedLength) {
513             // expected
514         } else if (mono.length == expectedLength * 2) {
515             // the decoder output 2 channels instead of 1, check that the left and right channel
516             // are identical
517             for (int i = 0; i < mono.length; i += 2) {
518                 assertEquals("mismatched samples at " + i, mono[i], mono[i+1]);
519             }
520         } else {
521             fail("wrong number of samples: " + mono.length);
522         }
523 
524         short [] mono2 = decodeToMemory(res, RESET_MODE_RECONFIGURE, CONFIG_MODE_NONE, -1, null);
525 
526         assertEquals("count different after reconfigure: ", mono.length, mono2.length);
527         for (int i = 0; i < mono.length; i++) {
528             assertEquals("samples at " + i + " don't match", mono[i], mono2[i]);
529         }
530 
531         short [] mono3 = decodeToMemory(res, RESET_MODE_FLUSH, CONFIG_MODE_NONE, -1, null);
532 
533         assertEquals("count different after flush: ", mono.length, mono3.length);
534         for (int i = 0; i < mono.length; i++) {
535             assertEquals("samples at " + i + " don't match", mono[i], mono3[i]);
536         }
537     }
538 
539     /**
540      * @param testinput the file to decode
541      * @param maxerror the maximum allowed root mean squared error
542      * @throws IOException
543      */
decode(int testinput, float maxerror)544     private void decode(int testinput, float maxerror) throws IOException {
545 
546         short[] decoded = decodeToMemory(testinput, RESET_MODE_NONE, CONFIG_MODE_NONE, -1, null);
547 
548         assertEquals("wrong data size", mMasterBuffer.length, decoded.length);
549 
550         long totalErrorSquared = 0;
551 
552         for (int i = 0; i < decoded.length; i++) {
553             short sample = decoded[i];
554             short mastersample = mMasterBuffer[i];
555             int d = sample - mastersample;
556             totalErrorSquared += d * d;
557         }
558 
559         long avgErrorSquared = (totalErrorSquared / decoded.length);
560         double rmse = Math.sqrt(avgErrorSquared);
561         assertTrue("decoding error too big: " + rmse, rmse <= maxerror);
562 
563         int[] resetModes = new int[] { RESET_MODE_NONE, RESET_MODE_RECONFIGURE,
564                 RESET_MODE_FLUSH, RESET_MODE_EOS_FLUSH };
565         int[] configModes = new int[] { CONFIG_MODE_NONE, CONFIG_MODE_QUEUE };
566 
567         for (int conf : configModes) {
568             for (int reset : resetModes) {
569                 if (conf == CONFIG_MODE_NONE && reset == RESET_MODE_NONE) {
570                     // default case done outside of loop
571                     continue;
572                 }
573                 if (conf == CONFIG_MODE_QUEUE && !hasAudioCsd(testinput)) {
574                     continue;
575                 }
576 
577                 String params = String.format("(using reset: %d, config: %s)", reset, conf);
578                 short[] decoded2 = decodeToMemory(testinput, reset, conf, -1, null);
579                 assertEquals("count different with reconfigure" + params,
580                         decoded.length, decoded2.length);
581                 for (int i = 0; i < decoded.length; i++) {
582                     assertEquals("samples don't match" + params, decoded[i], decoded2[i]);
583                 }
584             }
585         }
586     }
587 
hasAudioCsd(int testinput)588     private boolean hasAudioCsd(int testinput) throws IOException {
589         AssetFileDescriptor fd = null;
590         try {
591 
592             fd = mResources.openRawResourceFd(testinput);
593             MediaExtractor extractor = new MediaExtractor();
594             extractor.setDataSource(fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength());
595             MediaFormat format = extractor.getTrackFormat(0);
596 
597             return format.containsKey(CSD_KEYS[0]);
598 
599         } finally {
600             if (fd != null) {
601                 fd.close();
602             }
603         }
604     }
605 
decodeToMemory(int testinput, int resetMode, int configMode, int eossample, List<Long> timestamps)606     private short[] decodeToMemory(int testinput, int resetMode, int configMode,
607             int eossample, List<Long> timestamps) throws IOException {
608 
609         String localTag = TAG + "#decodeToMemory";
610         Log.v(localTag, String.format("reset = %d; config: %s", resetMode, configMode));
611         short [] decoded = new short[0];
612         int decodedIdx = 0;
613 
614         AssetFileDescriptor testFd = mResources.openRawResourceFd(testinput);
615 
616         MediaExtractor extractor;
617         MediaCodec codec;
618         ByteBuffer[] codecInputBuffers;
619         ByteBuffer[] codecOutputBuffers;
620 
621         extractor = new MediaExtractor();
622         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
623                 testFd.getLength());
624         testFd.close();
625 
626         assertEquals("wrong number of tracks", 1, extractor.getTrackCount());
627         MediaFormat format = extractor.getTrackFormat(0);
628         String mime = format.getString(MediaFormat.KEY_MIME);
629         assertTrue("not an audio file", mime.startsWith("audio/"));
630 
631         MediaFormat configFormat = format;
632         codec = MediaCodec.createDecoderByType(mime);
633         if (configMode == CONFIG_MODE_QUEUE && format.containsKey(CSD_KEYS[0])) {
634             configFormat = MediaFormat.createAudioFormat(mime,
635                     format.getInteger(MediaFormat.KEY_SAMPLE_RATE),
636                     format.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
637 
638             configFormat.setLong(MediaFormat.KEY_DURATION,
639                     format.getLong(MediaFormat.KEY_DURATION));
640             String[] keys = new String[] { "max-input-size", "encoder-delay", "encoder-padding" };
641             for (String k : keys) {
642                 if (format.containsKey(k)) {
643                     configFormat.setInteger(k, format.getInteger(k));
644                 }
645             }
646         }
647         Log.v(localTag, "configuring with " + configFormat);
648         codec.configure(configFormat, null /* surface */, null /* crypto */, 0 /* flags */);
649 
650         codec.start();
651         codecInputBuffers = codec.getInputBuffers();
652         codecOutputBuffers = codec.getOutputBuffers();
653 
654         if (resetMode == RESET_MODE_RECONFIGURE) {
655             codec.stop();
656             codec.configure(configFormat, null /* surface */, null /* crypto */, 0 /* flags */);
657             codec.start();
658             codecInputBuffers = codec.getInputBuffers();
659             codecOutputBuffers = codec.getOutputBuffers();
660         } else if (resetMode == RESET_MODE_FLUSH) {
661             codec.flush();
662         }
663 
664         extractor.selectTrack(0);
665 
666         if (configMode == CONFIG_MODE_QUEUE) {
667             queueConfig(codec, format);
668         }
669 
670         // start decoding
671         final long kTimeOutUs = 5000;
672         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
673         boolean sawInputEOS = false;
674         boolean sawOutputEOS = false;
675         int noOutputCounter = 0;
676         int samplecounter = 0;
677         while (!sawOutputEOS && noOutputCounter < 50) {
678             noOutputCounter++;
679             if (!sawInputEOS) {
680                 int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
681 
682                 if (inputBufIndex >= 0) {
683                     ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
684 
685                     int sampleSize =
686                         extractor.readSampleData(dstBuf, 0 /* offset */);
687 
688                     long presentationTimeUs = 0;
689 
690                     if (sampleSize < 0 && eossample > 0) {
691                         fail("test is broken: never reached eos sample");
692                     }
693                     if (sampleSize < 0) {
694                         Log.d(TAG, "saw input EOS.");
695                         sawInputEOS = true;
696                         sampleSize = 0;
697                     } else {
698                         if (samplecounter == eossample) {
699                             sawInputEOS = true;
700                         }
701                         samplecounter++;
702                         presentationTimeUs = extractor.getSampleTime();
703                     }
704                     codec.queueInputBuffer(
705                             inputBufIndex,
706                             0 /* offset */,
707                             sampleSize,
708                             presentationTimeUs,
709                             sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
710 
711                     if (!sawInputEOS) {
712                         extractor.advance();
713                     }
714                 }
715             }
716 
717             int res = codec.dequeueOutputBuffer(info, kTimeOutUs);
718 
719             if (res >= 0) {
720                 //Log.d(TAG, "got frame, size " + info.size + "/" + info.presentationTimeUs);
721 
722                 if (info.size > 0) {
723                     noOutputCounter = 0;
724                     if (timestamps != null) {
725                         timestamps.add(info.presentationTimeUs);
726                     }
727                 }
728                 if (info.size > 0 &&
729                         resetMode != RESET_MODE_NONE && resetMode != RESET_MODE_EOS_FLUSH) {
730                     // once we've gotten some data out of the decoder, reset and start again
731                     if (resetMode == RESET_MODE_RECONFIGURE) {
732                         codec.stop();
733                         codec.configure(configFormat, null /* surface */, null /* crypto */,
734                                 0 /* flags */);
735                         codec.start();
736                         codecInputBuffers = codec.getInputBuffers();
737                         codecOutputBuffers = codec.getOutputBuffers();
738                         if (configMode == CONFIG_MODE_QUEUE) {
739                             queueConfig(codec, format);
740                         }
741                     } else /* resetMode == RESET_MODE_FLUSH */ {
742                         codec.flush();
743                     }
744                     resetMode = RESET_MODE_NONE;
745                     extractor.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
746                     sawInputEOS = false;
747                     samplecounter = 0;
748                     if (timestamps != null) {
749                         timestamps.clear();
750                     }
751                     continue;
752                 }
753 
754                 int outputBufIndex = res;
755                 ByteBuffer buf = codecOutputBuffers[outputBufIndex];
756 
757                 if (decodedIdx + (info.size / 2) >= decoded.length) {
758                     decoded = Arrays.copyOf(decoded, decodedIdx + (info.size / 2));
759                 }
760 
761                 buf.position(info.offset);
762                 for (int i = 0; i < info.size; i += 2) {
763                     decoded[decodedIdx++] = buf.getShort();
764                 }
765 
766                 codec.releaseOutputBuffer(outputBufIndex, false /* render */);
767 
768                 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
769                     Log.d(TAG, "saw output EOS.");
770                     if (resetMode == RESET_MODE_EOS_FLUSH) {
771                         resetMode = RESET_MODE_NONE;
772                         codec.flush();
773                         extractor.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
774                         sawInputEOS = false;
775                         samplecounter = 0;
776                         decoded = new short[0];
777                         decodedIdx = 0;
778                         if (timestamps != null) {
779                             timestamps.clear();
780                         }
781                     } else {
782                         sawOutputEOS = true;
783                     }
784                 }
785             } else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
786                 codecOutputBuffers = codec.getOutputBuffers();
787 
788                 Log.d(TAG, "output buffers have changed.");
789             } else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
790                 MediaFormat oformat = codec.getOutputFormat();
791 
792                 Log.d(TAG, "output format has changed to " + oformat);
793             } else {
794                 Log.d(TAG, "dequeueOutputBuffer returned " + res);
795             }
796         }
797         if (noOutputCounter >= 50) {
798             fail("decoder stopped outputing data");
799         }
800 
801         codec.stop();
802         codec.release();
803         return decoded;
804     }
805 
queueConfig(MediaCodec codec, MediaFormat format)806     private void queueConfig(MediaCodec codec, MediaFormat format) {
807         for (String csdKey : CSD_KEYS) {
808             if (!format.containsKey(csdKey)) {
809                 continue;
810             }
811             ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
812             int inputBufIndex = codec.dequeueInputBuffer(-1);
813             if (inputBufIndex < 0) {
814                 fail("failed to queue configuration buffer " + csdKey);
815             } else {
816                 ByteBuffer csd = (ByteBuffer) format.getByteBuffer(csdKey).rewind();
817                 Log.v(TAG + "#queueConfig", String.format("queueing %s:%s", csdKey, csd));
818                 codecInputBuffers[inputBufIndex].put(csd);
819                 codec.queueInputBuffer(
820                         inputBufIndex,
821                         0 /* offset */,
822                         csd.limit(),
823                         0 /* presentation time (us) */,
824                         MediaCodec.BUFFER_FLAG_CODEC_CONFIG);
825             }
826         }
827     }
828 
testDecodeWithEOSOnLastBuffer()829     public void testDecodeWithEOSOnLastBuffer() throws Exception {
830         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepm4a);
831         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepmp3lame);
832         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepmp3smpb);
833         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepwav);
834         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepflac);
835         testDecodeWithEOSOnLastBuffer(R.raw.sinesweepogg);
836     }
837 
838     /* setting EOS on the last full input buffer should be equivalent to setting EOS on an empty
839      * input buffer after all the full ones. */
testDecodeWithEOSOnLastBuffer(int res)840     private void testDecodeWithEOSOnLastBuffer(int res) throws Exception {
841         int numsamples = countSamples(res);
842         assertTrue(numsamples != 0);
843 
844         List<Long> timestamps1 = new ArrayList<Long>();
845         short[] decode1 = decodeToMemory(res, RESET_MODE_NONE, CONFIG_MODE_NONE, -1, timestamps1);
846 
847         List<Long> timestamps2 = new ArrayList<Long>();
848         short[] decode2 = decodeToMemory(res, RESET_MODE_NONE, CONFIG_MODE_NONE, numsamples - 1,
849                 timestamps2);
850 
851         // check that the data and the timestamps are the same for EOS-on-last and EOS-after-last
852         assertEquals(decode1.length, decode2.length);
853         assertTrue(Arrays.equals(decode1, decode2));
854         assertEquals(timestamps1.size(), timestamps2.size());
855         assertTrue(timestamps1.equals(timestamps2));
856 
857         // ... and that this is also true when reconfiguring the codec
858         timestamps2.clear();
859         decode2 = decodeToMemory(res, RESET_MODE_RECONFIGURE, CONFIG_MODE_NONE, -1, timestamps2);
860         assertTrue(Arrays.equals(decode1, decode2));
861         assertTrue(timestamps1.equals(timestamps2));
862         timestamps2.clear();
863         decode2 = decodeToMemory(res, RESET_MODE_RECONFIGURE, CONFIG_MODE_NONE, numsamples - 1,
864                 timestamps2);
865         assertEquals(decode1.length, decode2.length);
866         assertTrue(Arrays.equals(decode1, decode2));
867         assertTrue(timestamps1.equals(timestamps2));
868 
869         // ... and that this is also true when flushing the codec
870         timestamps2.clear();
871         decode2 = decodeToMemory(res, RESET_MODE_FLUSH, CONFIG_MODE_NONE, -1, timestamps2);
872         assertTrue(Arrays.equals(decode1, decode2));
873         assertTrue(timestamps1.equals(timestamps2));
874         timestamps2.clear();
875         decode2 = decodeToMemory(res, RESET_MODE_FLUSH, CONFIG_MODE_NONE, numsamples - 1,
876                 timestamps2);
877         assertEquals(decode1.length, decode2.length);
878         assertTrue(Arrays.equals(decode1, decode2));
879         assertTrue(timestamps1.equals(timestamps2));
880     }
881 
countSamples(int res)882     private int countSamples(int res) throws IOException {
883         AssetFileDescriptor testFd = mResources.openRawResourceFd(res);
884 
885         MediaExtractor extractor = new MediaExtractor();
886         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
887                 testFd.getLength());
888         testFd.close();
889         extractor.selectTrack(0);
890         int numsamples = 0;
891         while (extractor.advance()) {
892             numsamples++;
893         }
894         return numsamples;
895     }
896 
testDecode(int testVideo, int frameNum)897     private void testDecode(int testVideo, int frameNum) throws Exception {
898         if (!MediaUtils.checkCodecForResource(mContext, testVideo, 0 /* track */)) {
899             return; // skip
900         }
901 
902         // Decode to Surface.
903         Surface s = getActivity().getSurfaceHolder().getSurface();
904         int frames1 = countFrames(testVideo, RESET_MODE_NONE, -1 /* eosframe */, s);
905         assertEquals("wrong number of frames decoded", frameNum, frames1);
906 
907         // Decode to buffer.
908         int frames2 = countFrames(testVideo, RESET_MODE_NONE, -1 /* eosframe */, null);
909         assertEquals("different number of frames when using Surface", frames1, frames2);
910     }
911 
testCodecBasicH264()912     public void testCodecBasicH264() throws Exception {
913         testDecode(R.raw.video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz, 240);
914     }
915 
testCodecBasicHEVC()916     public void testCodecBasicHEVC() throws Exception {
917         testDecode(R.raw.video_1280x720_mp4_hevc_1150kbps_30fps_aac_stereo_128kbps_48000hz, 300);
918     }
919 
testCodecBasicH263()920     public void testCodecBasicH263() throws Exception {
921         testDecode(R.raw.video_176x144_3gp_h263_300kbps_12fps_aac_stereo_128kbps_22050hz, 122);
922     }
923 
testCodecBasicMpeg4()924     public void testCodecBasicMpeg4() throws Exception {
925         testDecode(R.raw.video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz, 249);
926     }
927 
testCodecBasicVP8()928     public void testCodecBasicVP8() throws Exception {
929         testDecode(R.raw.video_480x360_webm_vp8_333kbps_25fps_vorbis_stereo_128kbps_48000hz, 240);
930     }
931 
testCodecBasicVP9()932     public void testCodecBasicVP9() throws Exception {
933         testDecode(R.raw.video_480x360_webm_vp9_333kbps_25fps_vorbis_stereo_128kbps_48000hz, 240);
934     }
935 
testH264Decode320x240()936     public void testH264Decode320x240() throws Exception {
937         testDecode(R.raw.video_320x240_mp4_h264_800kbps_30fps_aac_stereo_128kbps_44100hz, 299);
938     }
939 
testH264Decode720x480()940     public void testH264Decode720x480() throws Exception {
941         testDecode(R.raw.video_720x480_mp4_h264_2048kbps_30fps_aac_stereo_128kbps_44100hz, 299);
942     }
943 
testH264Decode30fps1280x720Tv()944     public void testH264Decode30fps1280x720Tv() throws Exception {
945         if (checkTv()) {
946             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_AVC, 1280, 720, 30));
947         }
948     }
949 
testH264SecureDecode30fps1280x720Tv()950     public void testH264SecureDecode30fps1280x720Tv() throws Exception {
951         if (checkTv()) {
952             verifySecureVideoDecodeSupport(MediaFormat.MIMETYPE_VIDEO_AVC, 1280, 720, 30);
953         }
954     }
955 
testH264Decode30fps1280x720()956     public void testH264Decode30fps1280x720() throws Exception {
957         testDecode(R.raw.video_1280x720_mp4_h264_8192kbps_30fps_aac_stereo_128kbps_44100hz, 299);
958     }
959 
testH264Decode60fps1280x720Tv()960     public void testH264Decode60fps1280x720Tv() throws Exception {
961         if (checkTv()) {
962             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_AVC, 1280, 720, 60));
963         }
964     }
965 
testH264SecureDecode60fps1280x720Tv()966     public void testH264SecureDecode60fps1280x720Tv() throws Exception {
967         if (checkTv()) {
968             verifySecureVideoDecodeSupport(MediaFormat.MIMETYPE_VIDEO_AVC, 1280, 720, 60);
969         }
970     }
971 
testH264Decode60fps1280x720()972     public void testH264Decode60fps1280x720() throws Exception {
973         testDecode(R.raw.video_1280x720_mp4_h264_8192kbps_60fps_aac_stereo_128kbps_44100hz, 596);
974     }
975 
testH264Decode30fps1920x1080Tv()976     public void testH264Decode30fps1920x1080Tv() throws Exception {
977         if (checkTv()) {
978             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080, 30));
979         }
980     }
981 
testH264SecureDecode30fps1920x1080Tv()982     public void testH264SecureDecode30fps1920x1080Tv() throws Exception {
983         if (checkTv()) {
984             verifySecureVideoDecodeSupport(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080, 30);
985         }
986     }
987 
testH264Decode30fps1920x1080()988     public void testH264Decode30fps1920x1080() throws Exception {
989         testDecode(R.raw.video_1920x1080_mp4_h264_20480kbps_30fps_aac_stereo_128kbps_44100hz, 299);
990     }
991 
testH264Decode60fps1920x1080Tv()992     public void testH264Decode60fps1920x1080Tv() throws Exception {
993         if (checkTv()) {
994             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080, 60));
995         }
996     }
997 
testH264SecureDecode60fps1920x1080Tv()998     public void testH264SecureDecode60fps1920x1080Tv() throws Exception {
999         if (checkTv()) {
1000             verifySecureVideoDecodeSupport(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080, 60);
1001         }
1002     }
1003 
testH264Decode60fps1920x1080()1004     public void testH264Decode60fps1920x1080() throws Exception {
1005         testDecode(R.raw.video_1920x1080_mp4_h264_20480kbps_60fps_aac_stereo_128kbps_44100hz, 596);
1006     }
1007 
testVP8Decode320x240()1008     public void testVP8Decode320x240() throws Exception {
1009         testDecode(R.raw.video_320x240_webm_vp8_800kbps_30fps_vorbis_stereo_128kbps_44100hz, 249);
1010     }
1011 
testVP8Decode640x360()1012     public void testVP8Decode640x360() throws Exception {
1013         testDecode(R.raw.video_640x360_webm_vp8_2048kbps_30fps_vorbis_stereo_128kbps_48000hz, 249);
1014     }
1015 
testVP8Decode30fps1280x720Tv()1016     public void testVP8Decode30fps1280x720Tv() throws Exception {
1017         if (checkTv()) {
1018             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_VP8, 1280, 720, 30));
1019         }
1020     }
1021 
testVP8Decode30fps1280x720()1022     public void testVP8Decode30fps1280x720() throws Exception {
1023         testDecode(R.raw.video_1280x720_webm_vp8_8192kbps_30fps_vorbis_stereo_128kbps_48000hz, 249);
1024     }
1025 
testVP8Decode60fps1280x720Tv()1026     public void testVP8Decode60fps1280x720Tv() throws Exception {
1027         if (checkTv()) {
1028             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_VP8, 1280, 720, 60));
1029         }
1030     }
1031 
testVP8Decode60fps1280x720()1032     public void testVP8Decode60fps1280x720() throws Exception {
1033         testDecode(R.raw.video_1280x720_webm_vp8_8192kbps_60fps_vorbis_stereo_128kbps_48000hz, 249);
1034     }
1035 
testVP8Decode30fps1920x1080Tv()1036     public void testVP8Decode30fps1920x1080Tv() throws Exception {
1037         if (checkTv()) {
1038             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_VP8, 1920, 1080, 30));
1039         }
1040     }
1041 
testVP8Decode30fps1920x1080()1042     public void testVP8Decode30fps1920x1080() throws Exception {
1043         testDecode(R.raw.video_1920x1080_webm_vp8_20480kbps_30fps_vorbis_stereo_128kbps_48000hz,
1044                 249);
1045     }
1046 
testVP8Decode60fps1920x1080Tv()1047     public void testVP8Decode60fps1920x1080Tv() throws Exception {
1048         if (checkTv()) {
1049             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_VP8, 1920, 1080, 60));
1050         }
1051     }
1052 
testVP8Decode60fps1920x1080()1053     public void testVP8Decode60fps1920x1080() throws Exception {
1054         testDecode(R.raw.video_1920x1080_webm_vp8_20480kbps_60fps_vorbis_stereo_128kbps_44100hz,
1055                 249);
1056     }
1057 
testVP9Decode320x240()1058     public void testVP9Decode320x240() throws Exception {
1059         testDecode(R.raw.video_320x240_webm_vp9_600kbps_30fps_vorbis_stereo_128kbps_48000hz, 249);
1060     }
1061 
testVP9Decode640x360()1062     public void testVP9Decode640x360() throws Exception {
1063         testDecode(R.raw.video_640x360_webm_vp9_1600kbps_30fps_vorbis_stereo_128kbps_48000hz, 249);
1064     }
1065 
testVP9Decode30fps1280x720Tv()1066     public void testVP9Decode30fps1280x720Tv() throws Exception {
1067         if (checkTv()) {
1068             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_VP9, 1280, 720, 30));
1069         }
1070     }
1071 
testVP9Decode30fps1280x720()1072     public void testVP9Decode30fps1280x720() throws Exception {
1073         testDecode(R.raw.video_1280x720_webm_vp9_4096kbps_30fps_vorbis_stereo_128kbps_44100hz, 249);
1074     }
1075 
testVP9Decode30fps1920x1080()1076     public void testVP9Decode30fps1920x1080() throws Exception {
1077         testDecode(R.raw.video_1920x1080_webm_vp9_10240kbps_30fps_vorbis_stereo_128kbps_48000hz,
1078                 249);
1079     }
1080 
testVP9Decode30fps3840x2160()1081     public void testVP9Decode30fps3840x2160() throws Exception {
1082         testDecode(R.raw.video_3840x2160_webm_vp9_20480kbps_30fps_vorbis_stereo_128kbps_48000hz,
1083                 249);
1084     }
1085 
testHEVCDecode352x288()1086     public void testHEVCDecode352x288() throws Exception {
1087         testDecode(R.raw.video_352x288_mp4_hevc_600kbps_30fps_aac_stereo_128kbps_44100hz, 299);
1088     }
1089 
testHEVCDecode720x480()1090     public void testHEVCDecode720x480() throws Exception {
1091         testDecode(R.raw.video_720x480_mp4_hevc_1638kbps_30fps_aac_stereo_128kbps_44100hz, 299);
1092     }
1093 
testHEVCDecode30fps1280x720Tv()1094     public void testHEVCDecode30fps1280x720Tv() throws Exception {
1095         if (checkTv()) {
1096             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_HEVC, 1280, 720, 30));
1097         }
1098     }
1099 
testHEVCDecode30fps1280x720()1100     public void testHEVCDecode30fps1280x720() throws Exception {
1101         testDecode(R.raw.video_1280x720_mp4_hevc_4096kbps_30fps_aac_stereo_128kbps_44100hz, 299);
1102     }
1103 
testHEVCDecode30fps1920x1080Tv()1104     public void testHEVCDecode30fps1920x1080Tv() throws Exception {
1105         if (checkTv()) {
1106             assertTrue(MediaUtils.canDecodeVideo(MediaFormat.MIMETYPE_VIDEO_HEVC, 1920, 1080, 30));
1107         }
1108     }
1109 
testHEVCDecode30fps1920x1080()1110     public void testHEVCDecode30fps1920x1080() throws Exception {
1111         testDecode(R.raw.video_1920x1080_mp4_hevc_10240kbps_30fps_aac_stereo_128kbps_44100hz, 299);
1112     }
1113 
testHEVCDecode30fps3840x2160()1114     public void testHEVCDecode30fps3840x2160() throws Exception {
1115         testDecode(R.raw.video_3840x2160_mp4_hevc_20480kbps_30fps_aac_stereo_128kbps_44100hz, 299);
1116     }
1117 
testCodecEarlyEOS(int resid, int eosFrame)1118     private void testCodecEarlyEOS(int resid, int eosFrame) throws Exception {
1119         if (!MediaUtils.checkCodecForResource(mContext, resid, 0 /* track */)) {
1120             return; // skip
1121         }
1122         Surface s = getActivity().getSurfaceHolder().getSurface();
1123         int frames1 = countFrames(resid, RESET_MODE_NONE, eosFrame, s);
1124         assertEquals("wrong number of frames decoded", eosFrame, frames1);
1125     }
1126 
testCodecEarlyEOSH263()1127     public void testCodecEarlyEOSH263() throws Exception {
1128         testCodecEarlyEOS(
1129                 R.raw.video_176x144_3gp_h263_300kbps_12fps_aac_stereo_128kbps_22050hz,
1130                 64 /* eosframe */);
1131     }
1132 
testCodecEarlyEOSH264()1133     public void testCodecEarlyEOSH264() throws Exception {
1134         testCodecEarlyEOS(
1135                 R.raw.video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz,
1136                 120 /* eosframe */);
1137     }
1138 
testCodecEarlyEOSHEVC()1139     public void testCodecEarlyEOSHEVC() throws Exception {
1140         testCodecEarlyEOS(
1141                 R.raw.video_480x360_mp4_hevc_650kbps_30fps_aac_stereo_128kbps_48000hz,
1142                 120 /* eosframe */);
1143     }
1144 
testCodecEarlyEOSMpeg4()1145     public void testCodecEarlyEOSMpeg4() throws Exception {
1146         testCodecEarlyEOS(
1147                 R.raw.video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz,
1148                 120 /* eosframe */);
1149     }
1150 
testCodecEarlyEOSVP8()1151     public void testCodecEarlyEOSVP8() throws Exception {
1152         testCodecEarlyEOS(
1153                 R.raw.video_480x360_webm_vp8_333kbps_25fps_vorbis_stereo_128kbps_48000hz,
1154                 120 /* eosframe */);
1155     }
1156 
testCodecEarlyEOSVP9()1157     public void testCodecEarlyEOSVP9() throws Exception {
1158         testCodecEarlyEOS(
1159                 R.raw.video_480x360_webm_vp9_333kbps_25fps_vorbis_stereo_128kbps_48000hz,
1160                 120 /* eosframe */);
1161     }
1162 
testCodecResetsH264WithoutSurface()1163     public void testCodecResetsH264WithoutSurface() throws Exception {
1164         testCodecResets(
1165                 R.raw.video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz, null);
1166     }
1167 
testCodecResetsH264WithSurface()1168     public void testCodecResetsH264WithSurface() throws Exception {
1169         Surface s = getActivity().getSurfaceHolder().getSurface();
1170         testCodecResets(
1171                 R.raw.video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz, s);
1172     }
1173 
testCodecResetsHEVCWithoutSurface()1174     public void testCodecResetsHEVCWithoutSurface() throws Exception {
1175         testCodecResets(
1176                 R.raw.video_1280x720_mp4_hevc_1150kbps_30fps_aac_stereo_128kbps_48000hz, null);
1177     }
1178 
testCodecResetsHEVCWithSurface()1179     public void testCodecResetsHEVCWithSurface() throws Exception {
1180         Surface s = getActivity().getSurfaceHolder().getSurface();
1181         testCodecResets(
1182                 R.raw.video_1280x720_mp4_hevc_1150kbps_30fps_aac_stereo_128kbps_48000hz, s);
1183     }
1184 
testCodecResetsH263WithoutSurface()1185     public void testCodecResetsH263WithoutSurface() throws Exception {
1186         testCodecResets(
1187                 R.raw.video_176x144_3gp_h263_300kbps_12fps_aac_stereo_128kbps_22050hz, null);
1188     }
1189 
testCodecResetsH263WithSurface()1190     public void testCodecResetsH263WithSurface() throws Exception {
1191         Surface s = getActivity().getSurfaceHolder().getSurface();
1192         testCodecResets(
1193                 R.raw.video_176x144_3gp_h263_300kbps_12fps_aac_stereo_128kbps_22050hz, s);
1194     }
1195 
testCodecResetsMpeg4WithoutSurface()1196     public void testCodecResetsMpeg4WithoutSurface() throws Exception {
1197         testCodecResets(
1198                 R.raw.video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz, null);
1199     }
1200 
testCodecResetsMpeg4WithSurface()1201     public void testCodecResetsMpeg4WithSurface() throws Exception {
1202         Surface s = getActivity().getSurfaceHolder().getSurface();
1203         testCodecResets(
1204                 R.raw.video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz, s);
1205     }
1206 
testCodecResetsVP8WithoutSurface()1207     public void testCodecResetsVP8WithoutSurface() throws Exception {
1208         testCodecResets(
1209                 R.raw.video_480x360_webm_vp8_333kbps_25fps_vorbis_stereo_128kbps_48000hz, null);
1210     }
1211 
testCodecResetsVP8WithSurface()1212     public void testCodecResetsVP8WithSurface() throws Exception {
1213         Surface s = getActivity().getSurfaceHolder().getSurface();
1214         testCodecResets(
1215                 R.raw.video_480x360_webm_vp8_333kbps_25fps_vorbis_stereo_128kbps_48000hz, s);
1216     }
1217 
testCodecResetsVP9WithoutSurface()1218     public void testCodecResetsVP9WithoutSurface() throws Exception {
1219         testCodecResets(
1220                 R.raw.video_480x360_webm_vp9_333kbps_25fps_vorbis_stereo_128kbps_48000hz, null);
1221     }
1222 
testCodecResetsVP9WithSurface()1223     public void testCodecResetsVP9WithSurface() throws Exception {
1224         Surface s = getActivity().getSurfaceHolder().getSurface();
1225         testCodecResets(
1226                 R.raw.video_480x360_webm_vp9_333kbps_25fps_vorbis_stereo_128kbps_48000hz, s);
1227     }
1228 
1229 //    public void testCodecResetsOgg() throws Exception {
1230 //        testCodecResets(R.raw.sinesweepogg, null);
1231 //    }
1232 
testCodecResetsMp3()1233     public void testCodecResetsMp3() throws Exception {
1234         testCodecReconfig(R.raw.sinesweepmp3lame);
1235         // NOTE: replacing testCodecReconfig call soon
1236 //        testCodecResets(R.raw.sinesweepmp3lame, null);
1237     }
1238 
testCodecResetsM4a()1239     public void testCodecResetsM4a() throws Exception {
1240         testCodecReconfig(R.raw.sinesweepm4a);
1241         // NOTE: replacing testCodecReconfig call soon
1242 //        testCodecResets(R.raw.sinesweepm4a, null);
1243     }
1244 
testCodecReconfig(int audio)1245     private void testCodecReconfig(int audio) throws Exception {
1246         int size1 = countSize(audio, RESET_MODE_NONE, -1 /* eosframe */);
1247         int size2 = countSize(audio, RESET_MODE_RECONFIGURE, -1 /* eosframe */);
1248         assertEquals("different output size when using reconfigured codec", size1, size2);
1249     }
1250 
testCodecResets(int video, Surface s)1251     private void testCodecResets(int video, Surface s) throws Exception {
1252         if (!MediaUtils.checkCodecForResource(mContext, video, 0 /* track */)) {
1253             return; // skip
1254         }
1255 
1256         int frames1 = countFrames(video, RESET_MODE_NONE, -1 /* eosframe */, s);
1257         int frames2 = countFrames(video, RESET_MODE_RECONFIGURE, -1 /* eosframe */, s);
1258         int frames3 = countFrames(video, RESET_MODE_FLUSH, -1 /* eosframe */, s);
1259         assertEquals("different number of frames when using reconfigured codec", frames1, frames2);
1260         assertEquals("different number of frames when using flushed codec", frames1, frames3);
1261     }
1262 
verifySecureVideoDecodeSupport(String mime, int width, int height, float rate)1263     private static void verifySecureVideoDecodeSupport(String mime, int width, int height, float rate) {
1264         MediaFormat baseFormat = new MediaFormat();
1265         baseFormat.setString(MediaFormat.KEY_MIME, mime);
1266         baseFormat.setFeatureEnabled(CodecCapabilities.FEATURE_SecurePlayback, true);
1267 
1268         MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
1269         format.setFeatureEnabled(CodecCapabilities.FEATURE_SecurePlayback, true);
1270         format.setFloat(MediaFormat.KEY_FRAME_RATE, rate);
1271 
1272         MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS);
1273         if (mcl.findDecoderForFormat(baseFormat) == null) {
1274             MediaUtils.skipTest("no secure decoder for " + mime);
1275             return;
1276         }
1277         assertNotNull("no decoder for " + format, mcl.findDecoderForFormat(format));
1278     }
1279 
createDecoder(String mime)1280     private static MediaCodec createDecoder(String mime) {
1281         try {
1282             if (false) {
1283                 // change to force testing software codecs
1284                 if (mime.contains("avc")) {
1285                     return MediaCodec.createByCodecName("OMX.google.h264.decoder");
1286                 } else if (mime.contains("hevc")) {
1287                     return MediaCodec.createByCodecName("OMX.google.hevc.decoder");
1288                 } else if (mime.contains("3gpp")) {
1289                     return MediaCodec.createByCodecName("OMX.google.h263.decoder");
1290                 } else if (mime.contains("mp4v")) {
1291                     return MediaCodec.createByCodecName("OMX.google.mpeg4.decoder");
1292                 } else if (mime.contains("vp8")) {
1293                     return MediaCodec.createByCodecName("OMX.google.vp8.decoder");
1294                 } else if (mime.contains("vp9")) {
1295                     return MediaCodec.createByCodecName("OMX.google.vp9.decoder");
1296                 }
1297             }
1298             return MediaCodec.createDecoderByType(mime);
1299         } catch (Exception e) {
1300             return null;
1301         }
1302     }
1303 
createDecoder(MediaFormat format)1304     private static MediaCodec createDecoder(MediaFormat format) {
1305         return MediaUtils.getDecoder(format);
1306     }
1307 
1308     // for video
countFrames(int video, int resetMode, int eosframe, Surface s)1309     private int countFrames(int video, int resetMode, int eosframe, Surface s)
1310             throws Exception {
1311         AssetFileDescriptor testFd = mResources.openRawResourceFd(video);
1312         MediaExtractor extractor = new MediaExtractor();
1313         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
1314                 testFd.getLength());
1315         extractor.selectTrack(0);
1316 
1317         int numframes = decodeWithChecks(extractor, CHECKFLAG_RETURN_OUTPUTFRAMES
1318                 | CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH, resetMode, s,
1319                 eosframe, null, null);
1320 
1321         extractor.release();
1322         testFd.close();
1323         return numframes;
1324     }
1325 
1326     // for audio
countSize(int audio, int resetMode, int eosframe)1327     private int countSize(int audio, int resetMode, int eosframe)
1328             throws Exception {
1329         AssetFileDescriptor testFd = mResources.openRawResourceFd(audio);
1330         MediaExtractor extractor = new MediaExtractor();
1331         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
1332                 testFd.getLength());
1333         extractor.selectTrack(0);
1334 
1335         // fails CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH
1336         int outputSize = decodeWithChecks(extractor, CHECKFLAG_RETURN_OUTPUTSIZE, resetMode, null,
1337                 eosframe, null, null);
1338 
1339         extractor.release();
1340         testFd.close();
1341         return outputSize;
1342     }
1343 
testEOSBehavior(int movie, int stopatsample)1344     private void testEOSBehavior(int movie, int stopatsample) throws Exception {
1345         testEOSBehavior(movie, new int[] {stopatsample});
1346     }
1347 
testEOSBehavior(int movie, int[] stopAtSample)1348     private void testEOSBehavior(int movie, int[] stopAtSample) throws Exception {
1349         Surface s = null;
1350         AssetFileDescriptor testFd = mResources.openRawResourceFd(movie);
1351         MediaExtractor extractor = new MediaExtractor();
1352         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
1353                 testFd.getLength());
1354         extractor.selectTrack(0); // consider variable looping on track
1355         MediaFormat format = extractor.getTrackFormat(0);
1356         if (!MediaUtils.checkDecoderForFormat(format)) {
1357             return; // skip
1358         }
1359         List<Long> outputChecksums = new ArrayList<Long>();
1360         List<Long> outputTimestamps = new ArrayList<Long>();
1361         Arrays.sort(stopAtSample);
1362         int last = stopAtSample.length - 1;
1363 
1364         // decode reference (longest sequence to stop at + 100) and
1365         // store checksums/pts in outputChecksums and outputTimestamps
1366         // (will fail CHECKFLAG_COMPAREINPUTOUTPUTSAMPLEMATCH)
1367         decodeWithChecks(extractor,
1368                 CHECKFLAG_SETCHECKSUM | CHECKFLAG_SETPTS | CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH,
1369                 RESET_MODE_NONE, s,
1370                 stopAtSample[last] + 100, outputChecksums, outputTimestamps);
1371 
1372         // decode stopAtSample requests in reverse order (longest to
1373         // shortest) and compare to reference checksums/pts in
1374         // outputChecksums and outputTimestamps
1375         for (int i = last; i >= 0; --i) {
1376             if (true) { // reposition extractor
1377                 extractor.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
1378             } else { // create new extractor
1379                 extractor.release();
1380                 extractor = new MediaExtractor();
1381                 extractor.setDataSource(testFd.getFileDescriptor(),
1382                         testFd.getStartOffset(), testFd.getLength());
1383                 extractor.selectTrack(0); // consider variable looping on track
1384             }
1385             decodeWithChecks(extractor,
1386                     CHECKFLAG_COMPARECHECKSUM | CHECKFLAG_COMPAREPTS
1387                     | CHECKFLAG_COMPAREINPUTOUTPUTSAMPLEMATCH
1388                     | CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH,
1389                     RESET_MODE_NONE, s,
1390                     stopAtSample[i], outputChecksums, outputTimestamps);
1391         }
1392         extractor.release();
1393         testFd.close();
1394     }
1395 
1396     private static final int CHECKFLAG_SETCHECKSUM = 1 << 0;
1397     private static final int CHECKFLAG_COMPARECHECKSUM = 1 << 1;
1398     private static final int CHECKFLAG_SETPTS = 1 << 2;
1399     private static final int CHECKFLAG_COMPAREPTS = 1 << 3;
1400     private static final int CHECKFLAG_COMPAREINPUTOUTPUTSAMPLEMATCH = 1 << 4;
1401     private static final int CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH = 1 << 5;
1402     private static final int CHECKFLAG_RETURN_OUTPUTFRAMES = 1 << 6;
1403     private static final int CHECKFLAG_RETURN_OUTPUTSIZE = 1 << 7;
1404 
1405     /**
1406      * Decodes frames with parameterized checks and return values.
1407      * The integer return can be selected through the checkFlags variable.
1408      */
decodeWithChecks(MediaExtractor extractor, int checkFlags, int resetMode, Surface surface, int stopAtSample, List<Long> outputChecksums, List<Long> outputTimestamps)1409     private static int decodeWithChecks(MediaExtractor extractor, int checkFlags, int resetMode,
1410             Surface surface, int stopAtSample,
1411             List<Long> outputChecksums, List<Long> outputTimestamps)
1412             throws Exception {
1413         int trackIndex = extractor.getSampleTrackIndex();
1414         MediaFormat format = extractor.getTrackFormat(trackIndex);
1415         String mime = format.getString(MediaFormat.KEY_MIME);
1416         boolean isAudio = mime.startsWith("audio/");
1417         ByteBuffer[] codecInputBuffers;
1418         ByteBuffer[] codecOutputBuffers;
1419 
1420         MediaCodec codec = createDecoder(format);
1421         Log.i("@@@@", "using codec: " + codec.getName());
1422         codec.configure(format, surface, null /* crypto */, 0 /* flags */);
1423         codec.start();
1424         codecInputBuffers = codec.getInputBuffers();
1425         codecOutputBuffers = codec.getOutputBuffers();
1426 
1427         if (resetMode == RESET_MODE_RECONFIGURE) {
1428             codec.stop();
1429             codec.configure(format, surface, null /* crypto */, 0 /* flags */);
1430             codec.start();
1431             codecInputBuffers = codec.getInputBuffers();
1432             codecOutputBuffers = codec.getOutputBuffers();
1433         } else if (resetMode == RESET_MODE_FLUSH) {
1434             codec.flush();
1435         }
1436 
1437         // start decode loop
1438         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
1439 
1440         final long kTimeOutUs = 5000; // 5ms timeout
1441         boolean sawInputEOS = false;
1442         boolean sawOutputEOS = false;
1443         int deadDecoderCounter = 0;
1444         int samplenum = 0;
1445         int numframes = 0;
1446         int outputSize = 0;
1447         int width = 0;
1448         int height = 0;
1449         boolean dochecksum = false;
1450         ArrayList<Long> timestamps = new ArrayList<Long>();
1451         if ((checkFlags & CHECKFLAG_SETPTS) != 0) {
1452             outputTimestamps.clear();
1453         }
1454         if ((checkFlags & CHECKFLAG_SETCHECKSUM) != 0) {
1455             outputChecksums.clear();
1456         }
1457         while (!sawOutputEOS && deadDecoderCounter < 100) {
1458             // handle input
1459             if (!sawInputEOS) {
1460                 int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
1461 
1462                 if (inputBufIndex >= 0) {
1463                     ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
1464 
1465                     int sampleSize =
1466                             extractor.readSampleData(dstBuf, 0 /* offset */);
1467                     long presentationTimeUs = extractor.getSampleTime();
1468                     boolean advanceDone = extractor.advance();
1469                     // int flags = extractor.getSampleFlags();
1470                     // Log.i("@@@@", "read sample " + samplenum + ":" +
1471                     // extractor.getSampleFlags()
1472                     // + " @ " + extractor.getSampleTime() + " size " +
1473                     // sampleSize);
1474                     assertEquals("extractor.advance() should match end of stream", sampleSize >= 0,
1475                             advanceDone);
1476 
1477                     if (sampleSize < 0) {
1478                         Log.d(TAG, "saw input EOS.");
1479                         sawInputEOS = true;
1480                         assertEquals("extractor.readSampleData() must return -1 at end of stream",
1481                                 -1, sampleSize);
1482                         assertEquals("extractor.getSampleTime() must return -1 at end of stream",
1483                                 -1, presentationTimeUs);
1484                         sampleSize = 0; // required otherwise queueInputBuffer
1485                                         // returns invalid.
1486                     } else {
1487                         timestamps.add(presentationTimeUs);
1488                         samplenum++; // increment before comparing with stopAtSample
1489                         if (samplenum == stopAtSample) {
1490                             Log.d(TAG, "saw input EOS (stop at sample).");
1491                             sawInputEOS = true; // tag this sample as EOS
1492                         }
1493                     }
1494                     codec.queueInputBuffer(
1495                             inputBufIndex,
1496                             0 /* offset */,
1497                             sampleSize,
1498                             presentationTimeUs,
1499                             sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);
1500                 } else {
1501                     assertEquals(
1502                             "codec.dequeueInputBuffer() unrecognized return value: " + inputBufIndex,
1503                             MediaCodec.INFO_TRY_AGAIN_LATER, inputBufIndex);
1504                 }
1505             }
1506 
1507             // handle output
1508             int outputBufIndex = codec.dequeueOutputBuffer(info, kTimeOutUs);
1509 
1510             deadDecoderCounter++;
1511             if (outputBufIndex >= 0) {
1512                 if (info.size > 0) { // Disregard 0-sized buffers at the end.
1513                     deadDecoderCounter = 0;
1514                     if (resetMode != RESET_MODE_NONE) {
1515                         // once we've gotten some data out of the decoder, reset
1516                         // and start again
1517                         if (resetMode == RESET_MODE_RECONFIGURE) {
1518                             codec.stop();
1519                             codec.configure(format, surface /* surface */, null /* crypto */,
1520                                     0 /* flags */);
1521                             codec.start();
1522                             codecInputBuffers = codec.getInputBuffers();
1523                             codecOutputBuffers = codec.getOutputBuffers();
1524                         } else if (resetMode == RESET_MODE_FLUSH) {
1525                             codec.flush();
1526                         } else {
1527                             fail("unknown resetMode: " + resetMode);
1528                         }
1529                         // restart at beginning, clear resetMode
1530                         resetMode = RESET_MODE_NONE;
1531                         extractor.seekTo(0, MediaExtractor.SEEK_TO_NEXT_SYNC);
1532                         sawInputEOS = false;
1533                         numframes = 0;
1534                         timestamps.clear();
1535                         if ((checkFlags & CHECKFLAG_SETPTS) != 0) {
1536                             outputTimestamps.clear();
1537                         }
1538                         if ((checkFlags & CHECKFLAG_SETCHECKSUM) != 0) {
1539                             outputChecksums.clear();
1540                         }
1541                         continue;
1542                     }
1543                     if ((checkFlags & CHECKFLAG_COMPAREPTS) != 0) {
1544                         assertTrue("number of frames (" + numframes
1545                                 + ") exceeds number of reference timestamps",
1546                                 numframes < outputTimestamps.size());
1547                         assertEquals("frame ts mismatch at frame " + numframes,
1548                                 (long) outputTimestamps.get(numframes), info.presentationTimeUs);
1549                     } else if ((checkFlags & CHECKFLAG_SETPTS) != 0) {
1550                         outputTimestamps.add(info.presentationTimeUs);
1551                     }
1552                     if ((checkFlags & (CHECKFLAG_SETCHECKSUM | CHECKFLAG_COMPARECHECKSUM)) != 0) {
1553                         long sum = 0;   // note: checksum is 0 if buffer format unrecognized
1554                         if (dochecksum) {
1555                             Image image = codec.getOutputImage(outputBufIndex);
1556                             // use image to do crc if it's available
1557                             // fall back to buffer if image is not available
1558                             if (image != null) {
1559                                 sum = checksum(image);
1560                             } else {
1561                                 // TODO: add stride - right now just use info.size (as before)
1562                                 //sum = checksum(codecOutputBuffers[outputBufIndex], width, height,
1563                                 //        stride);
1564                                 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufIndex);
1565                                 outputBuffer.position(info.offset);
1566                                 sum = checksum(outputBuffer, info.size);
1567                             }
1568                         }
1569                         if ((checkFlags & CHECKFLAG_COMPARECHECKSUM) != 0) {
1570                             assertTrue("number of frames (" + numframes
1571                                     + ") exceeds number of reference checksums",
1572                                     numframes < outputChecksums.size());
1573                             Log.d(TAG, "orig checksum: " + outputChecksums.get(numframes)
1574                                     + " new checksum: " + sum);
1575                             assertEquals("frame data mismatch at frame " + numframes,
1576                                     (long) outputChecksums.get(numframes), sum);
1577                         } else if ((checkFlags & CHECKFLAG_SETCHECKSUM) != 0) {
1578                             outputChecksums.add(sum);
1579                         }
1580                     }
1581                     if ((checkFlags & CHECKFLAG_COMPAREINPUTOUTPUTPTSMATCH) != 0) {
1582                         assertTrue("output timestamp " + info.presentationTimeUs
1583                                 + " without corresponding input timestamp"
1584                                 , timestamps.remove(info.presentationTimeUs));
1585                     }
1586                     outputSize += info.size;
1587                     numframes++;
1588                 }
1589                 // Log.d(TAG, "got frame, size " + info.size + "/" +
1590                 // info.presentationTimeUs +
1591                 // "/" + numframes + "/" + info.flags);
1592                 codec.releaseOutputBuffer(outputBufIndex, true /* render */);
1593                 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
1594                     Log.d(TAG, "saw output EOS.");
1595                     sawOutputEOS = true;
1596                 }
1597             } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
1598                 codecOutputBuffers = codec.getOutputBuffers();
1599                 Log.d(TAG, "output buffers have changed.");
1600             } else if (outputBufIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1601                 MediaFormat oformat = codec.getOutputFormat();
1602                 if (oformat.containsKey(MediaFormat.KEY_COLOR_FORMAT) &&
1603                         oformat.containsKey(MediaFormat.KEY_WIDTH) &&
1604                         oformat.containsKey(MediaFormat.KEY_HEIGHT)) {
1605                     int colorFormat = oformat.getInteger(MediaFormat.KEY_COLOR_FORMAT);
1606                     width = oformat.getInteger(MediaFormat.KEY_WIDTH);
1607                     height = oformat.getInteger(MediaFormat.KEY_HEIGHT);
1608                     dochecksum = isRecognizedFormat(colorFormat); // only checksum known raw
1609                                                                   // buf formats
1610                     Log.d(TAG, "checksum fmt: " + colorFormat + " dim " + width + "x" + height);
1611                 } else {
1612                     dochecksum = false; // check with audio later
1613                     width = height = 0;
1614                     Log.d(TAG, "output format has changed to (unknown video) " + oformat);
1615                 }
1616             } else {
1617                 assertEquals(
1618                         "codec.dequeueOutputBuffer() unrecognized return index: "
1619                                 + outputBufIndex,
1620                         MediaCodec.INFO_TRY_AGAIN_LATER, outputBufIndex);
1621             }
1622         }
1623         codec.stop();
1624         codec.release();
1625 
1626         assertTrue("last frame didn't have EOS", sawOutputEOS);
1627         if ((checkFlags & CHECKFLAG_COMPAREINPUTOUTPUTSAMPLEMATCH) != 0) {
1628             assertEquals("I!=O", samplenum, numframes);
1629             if (stopAtSample != 0) {
1630                 assertEquals("did not stop with right number of frames", stopAtSample, numframes);
1631             }
1632         }
1633         return (checkFlags & CHECKFLAG_RETURN_OUTPUTSIZE) != 0 ? outputSize :
1634                 (checkFlags & CHECKFLAG_RETURN_OUTPUTFRAMES) != 0 ? numframes :
1635                         0;
1636     }
1637 
testEOSBehaviorH264()1638     public void testEOSBehaviorH264() throws Exception {
1639         // this video has an I frame at 44
1640         testEOSBehavior(R.raw.video_480x360_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz,
1641                 new int[] {44, 45, 55});
1642     }
testEOSBehaviorHEVC()1643     public void testEOSBehaviorHEVC() throws Exception {
1644         testEOSBehavior(R.raw.video_480x360_mp4_hevc_650kbps_30fps_aac_stereo_128kbps_48000hz, 17);
1645         testEOSBehavior(R.raw.video_480x360_mp4_hevc_650kbps_30fps_aac_stereo_128kbps_48000hz, 23);
1646         testEOSBehavior(R.raw.video_480x360_mp4_hevc_650kbps_30fps_aac_stereo_128kbps_48000hz, 49);
1647     }
1648 
testEOSBehaviorH263()1649     public void testEOSBehaviorH263() throws Exception {
1650         // this video has an I frame every 12 frames.
1651         testEOSBehavior(R.raw.video_176x144_3gp_h263_300kbps_12fps_aac_stereo_128kbps_22050hz,
1652                 new int[] {24, 25, 48, 50});
1653     }
1654 
testEOSBehaviorMpeg4()1655     public void testEOSBehaviorMpeg4() throws Exception {
1656         // this video has an I frame every 12 frames
1657         testEOSBehavior(R.raw.video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz,
1658                 new int[] {24, 25, 48, 50, 2});
1659     }
1660 
testEOSBehaviorVP8()1661     public void testEOSBehaviorVP8() throws Exception {
1662         // this video has an I frame at 46
1663         testEOSBehavior(R.raw.video_480x360_webm_vp8_333kbps_25fps_vorbis_stereo_128kbps_48000hz,
1664                 new int[] {46, 47, 57, 45});
1665     }
1666 
testEOSBehaviorVP9()1667     public void testEOSBehaviorVP9() throws Exception {
1668         // this video has an I frame at 44
1669         testEOSBehavior(R.raw.video_480x360_webm_vp9_333kbps_25fps_vorbis_stereo_128kbps_48000hz,
1670                 new int[] {44, 45, 55, 43});
1671     }
1672 
1673     /* from EncodeDecodeTest */
isRecognizedFormat(int colorFormat)1674     private static boolean isRecognizedFormat(int colorFormat) {
1675         // Log.d(TAG, "color format: " + String.format("0x%08x", colorFormat));
1676         switch (colorFormat) {
1677         // these are the formats we know how to handle for this test
1678             case CodecCapabilities.COLOR_FormatYUV420Planar:
1679             case CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
1680             case CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
1681             case CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
1682             case CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
1683             case CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar:
1684                 /*
1685                  * TODO: Check newer formats or ignore.
1686                  * OMX_SEC_COLOR_FormatNV12Tiled = 0x7FC00002
1687                  * OMX_QCOM_COLOR_FormatYUV420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03: N4/N7_2
1688                  * OMX_QCOM_COLOR_FormatYUV420PackedSemiPlanar32m = 0x7FA30C04: N5
1689                  */
1690                 return true;
1691             default:
1692                 return false;
1693         }
1694     }
1695 
checksum(ByteBuffer buf, int size)1696     private static long checksum(ByteBuffer buf, int size) {
1697         int cap = buf.capacity();
1698         assertTrue("checksum() params are invalid: size = " + size + " cap = " + cap,
1699                 size > 0 && size <= cap);
1700         CRC32 crc = new CRC32();
1701         if (buf.hasArray()) {
1702             crc.update(buf.array(), buf.position() + buf.arrayOffset(), size);
1703         } else {
1704             int pos = buf.position();
1705             final int rdsize = Math.min(4096, size);
1706             byte bb[] = new byte[rdsize];
1707             int chk;
1708             for (int i = 0; i < size; i += chk) {
1709                 chk = Math.min(rdsize, size - i);
1710                 buf.get(bb, 0, chk);
1711                 crc.update(bb, 0, chk);
1712             }
1713             buf.position(pos);
1714         }
1715         return crc.getValue();
1716     }
1717 
checksum(ByteBuffer buf, int width, int height, int stride)1718     private static long checksum(ByteBuffer buf, int width, int height, int stride) {
1719         int cap = buf.capacity();
1720         assertTrue("checksum() params are invalid: w x h , s = "
1721                 + width + " x " + height + " , " + stride + " cap = " + cap,
1722                 width > 0 && width <= stride && height > 0 && height * stride <= cap);
1723         // YUV 4:2:0 should generally have a data storage height 1.5x greater
1724         // than the declared image height, representing the UV planes.
1725         //
1726         // We only check Y frame for now. Somewhat unknown with tiling effects.
1727         //
1728         //long tm = System.nanoTime();
1729         final int lineinterval = 1; // line sampling frequency
1730         CRC32 crc = new CRC32();
1731         if (buf.hasArray()) {
1732             byte b[] = buf.array();
1733             int offs = buf.arrayOffset();
1734             for (int i = 0; i < height; i += lineinterval) {
1735                 crc.update(b, i * stride + offs, width);
1736             }
1737         } else { // almost always ends up here due to direct buffers
1738             int pos = buf.position();
1739             if (true) { // this {} is 80x times faster than else {} below.
1740                 byte[] bb = new byte[width]; // local line buffer
1741                 for (int i = 0; i < height; i += lineinterval) {
1742                     buf.position(pos + i * stride);
1743                     buf.get(bb, 0, width);
1744                     crc.update(bb, 0, width);
1745                 }
1746             } else {
1747                 for (int i = 0; i < height; i += lineinterval) {
1748                     buf.position(pos + i * stride);
1749                     for (int j = 0; j < width; ++j) {
1750                         crc.update(buf.get());
1751                     }
1752                 }
1753             }
1754             buf.position(pos);
1755         }
1756         //tm = System.nanoTime() - tm;
1757         //Log.d(TAG, "checksum time " + tm);
1758         return crc.getValue();
1759     }
1760 
checksum(Image image)1761     private static long checksum(Image image) {
1762         int format = image.getFormat();
1763         assertEquals("unsupported image format", ImageFormat.YUV_420_888, format);
1764 
1765         CRC32 crc = new CRC32();
1766 
1767         int imageWidth = image.getWidth();
1768         int imageHeight = image.getHeight();
1769 
1770         Image.Plane[] planes = image.getPlanes();
1771         for (int i = 0; i < planes.length; ++i) {
1772             ByteBuffer buf = planes[i].getBuffer();
1773 
1774             int width, height, rowStride, pixelStride, x, y;
1775             rowStride = planes[i].getRowStride();
1776             pixelStride = planes[i].getPixelStride();
1777             if (i == 0) {
1778                 width = imageWidth;
1779                 height = imageHeight;
1780             } else {
1781                 width = imageWidth / 2;
1782                 height = imageHeight /2;
1783             }
1784             // local contiguous pixel buffer
1785             byte[] bb = new byte[width * height];
1786             if (buf.hasArray()) {
1787                 byte b[] = buf.array();
1788                 int offs = buf.arrayOffset();
1789                 if (pixelStride == 1) {
1790                     for (y = 0; y < height; ++y) {
1791                         System.arraycopy(bb, y * width, b, y * rowStride + offs, width);
1792                     }
1793                 } else {
1794                     // do it pixel-by-pixel
1795                     for (y = 0; y < height; ++y) {
1796                         int lineOffset = offs + y * rowStride;
1797                         for (x = 0; x < width; ++x) {
1798                             bb[y * width + x] = b[lineOffset + x * pixelStride];
1799                         }
1800                     }
1801                 }
1802             } else { // almost always ends up here due to direct buffers
1803                 int pos = buf.position();
1804                 if (pixelStride == 1) {
1805                     for (y = 0; y < height; ++y) {
1806                         buf.position(pos + y * rowStride);
1807                         buf.get(bb, y * width, width);
1808                     }
1809                 } else {
1810                     // local line buffer
1811                     byte[] lb = new byte[rowStride];
1812                     // do it pixel-by-pixel
1813                     for (y = 0; y < height; ++y) {
1814                         buf.position(pos + y * rowStride);
1815                         // we're only guaranteed to have pixelStride * (width - 1) + 1 bytes
1816                         buf.get(lb, 0, pixelStride * (width - 1) + 1);
1817                         for (x = 0; x < width; ++x) {
1818                             bb[y * width + x] = lb[x * pixelStride];
1819                         }
1820                     }
1821                 }
1822                 buf.position(pos);
1823             }
1824             crc.update(bb, 0, width * height);
1825         }
1826 
1827         return crc.getValue();
1828     }
1829 
testFlush()1830     public void testFlush() throws Exception {
1831         testFlush(R.raw.loudsoftwav);
1832         testFlush(R.raw.loudsoftogg);
1833         testFlush(R.raw.loudsoftmp3);
1834         testFlush(R.raw.loudsoftaac);
1835         testFlush(R.raw.loudsoftfaac);
1836         testFlush(R.raw.loudsoftitunes);
1837     }
1838 
testFlush(int resource)1839     private void testFlush(int resource) throws Exception {
1840 
1841         AssetFileDescriptor testFd = mResources.openRawResourceFd(resource);
1842 
1843         MediaExtractor extractor;
1844         MediaCodec codec;
1845         ByteBuffer[] codecInputBuffers;
1846         ByteBuffer[] codecOutputBuffers;
1847 
1848         extractor = new MediaExtractor();
1849         extractor.setDataSource(testFd.getFileDescriptor(), testFd.getStartOffset(),
1850                 testFd.getLength());
1851         testFd.close();
1852 
1853         assertEquals("wrong number of tracks", 1, extractor.getTrackCount());
1854         MediaFormat format = extractor.getTrackFormat(0);
1855         String mime = format.getString(MediaFormat.KEY_MIME);
1856         assertTrue("not an audio file", mime.startsWith("audio/"));
1857 
1858         codec = MediaCodec.createDecoderByType(mime);
1859         assertNotNull("couldn't find codec " + mime, codec);
1860 
1861         codec.configure(format, null /* surface */, null /* crypto */, 0 /* flags */);
1862         codec.start();
1863         codecInputBuffers = codec.getInputBuffers();
1864         codecOutputBuffers = codec.getOutputBuffers();
1865 
1866         extractor.selectTrack(0);
1867 
1868         // decode a bit of the first part of the file, and verify the amplitude
1869         short maxvalue1 = getAmplitude(extractor, codec);
1870 
1871         // flush the codec and seek the extractor a different position, then decode a bit more
1872         // and check the amplitude
1873         extractor.seekTo(8000000, 0);
1874         codec.flush();
1875         short maxvalue2 = getAmplitude(extractor, codec);
1876 
1877         assertTrue("first section amplitude too low", maxvalue1 > 20000);
1878         assertTrue("second section amplitude too high", maxvalue2 < 5000);
1879         codec.stop();
1880         codec.release();
1881 
1882     }
1883 
getAmplitude(MediaExtractor extractor, MediaCodec codec)1884     private short getAmplitude(MediaExtractor extractor, MediaCodec codec) {
1885         short maxvalue = 0;
1886         int numBytesDecoded = 0;
1887         final long kTimeOutUs = 5000;
1888         ByteBuffer[] codecInputBuffers = codec.getInputBuffers();
1889         ByteBuffer[] codecOutputBuffers = codec.getOutputBuffers();
1890         MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
1891 
1892         while(numBytesDecoded < 44100 * 2) {
1893             int inputBufIndex = codec.dequeueInputBuffer(kTimeOutUs);
1894 
1895             if (inputBufIndex >= 0) {
1896                 ByteBuffer dstBuf = codecInputBuffers[inputBufIndex];
1897 
1898                 int sampleSize = extractor.readSampleData(dstBuf, 0 /* offset */);
1899                 long presentationTimeUs = extractor.getSampleTime();
1900 
1901                 codec.queueInputBuffer(
1902                         inputBufIndex,
1903                         0 /* offset */,
1904                         sampleSize,
1905                         presentationTimeUs,
1906                         0 /* flags */);
1907 
1908                 extractor.advance();
1909             }
1910             int res = codec.dequeueOutputBuffer(info, kTimeOutUs);
1911 
1912             if (res >= 0) {
1913 
1914                 int outputBufIndex = res;
1915                 ByteBuffer buf = codecOutputBuffers[outputBufIndex];
1916 
1917                 buf.position(info.offset);
1918                 for (int i = 0; i < info.size; i += 2) {
1919                     short sample = buf.getShort();
1920                     if (maxvalue < sample) {
1921                         maxvalue = sample;
1922                     }
1923                     int idx = (numBytesDecoded + i) / 2;
1924                 }
1925 
1926                 numBytesDecoded += info.size;
1927 
1928                 codec.releaseOutputBuffer(outputBufIndex, false /* render */);
1929             } else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
1930                 codecOutputBuffers = codec.getOutputBuffers();
1931             } else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
1932                 MediaFormat oformat = codec.getOutputFormat();
1933             }
1934         }
1935         return maxvalue;
1936     }
1937 
1938     /* return true if a particular video feature is supported for the given mimetype */
isVideoFeatureSupported(String mimeType, String feature)1939     private boolean isVideoFeatureSupported(String mimeType, String feature) {
1940         MediaFormat format = MediaFormat.createVideoFormat( mimeType, 1920, 1080);
1941         format.setFeatureEnabled(feature, true);
1942         MediaCodecList mcl = new MediaCodecList(MediaCodecList.ALL_CODECS);
1943         String codecName = mcl.findDecoderForFormat(format);
1944         return (codecName == null) ? false : true;
1945     }
1946 
1947 
1948     /**
1949      * Test tunneled video playback mode if supported
1950      */
testTunneledVideoPlayback()1951     public void testTunneledVideoPlayback() throws Exception {
1952         if (!isVideoFeatureSupported(MediaFormat.MIMETYPE_VIDEO_AVC,
1953                 CodecCapabilities.FEATURE_TunneledPlayback)) {
1954             MediaUtils.skipTest(TAG, "No tunneled video playback codec found!");
1955             return;
1956         }
1957 
1958         AudioManager am = (AudioManager)mContext.getSystemService(Context.AUDIO_SERVICE);
1959         mMediaCodecPlayer = new MediaCodecTunneledPlayer(
1960                 getActivity().getSurfaceHolder(), true, am.generateAudioSessionId());
1961 
1962         mMediaCodecPlayer.setAudioDataSource(AUDIO_URL, null);
1963         mMediaCodecPlayer.setVideoDataSource(VIDEO_URL, null);
1964         assertTrue("MediaCodecPlayer.start() failed!", mMediaCodecPlayer.start());
1965         assertTrue("MediaCodecPlayer.prepare() failed!", mMediaCodecPlayer.prepare());
1966 
1967         // starts video playback
1968         mMediaCodecPlayer.startThread();
1969 
1970         long timeOut = System.currentTimeMillis() + 4*PLAY_TIME_MS;
1971         while (timeOut > System.currentTimeMillis() && !mMediaCodecPlayer.isEnded()) {
1972             Thread.sleep(SLEEP_TIME_MS);
1973             if (mMediaCodecPlayer.getCurrentPosition() >= mMediaCodecPlayer.getDuration() ) {
1974                 Log.d(TAG, "testTunneledVideoPlayback -- current pos = " +
1975                         mMediaCodecPlayer.getCurrentPosition() +
1976                         ">= duration = " + mMediaCodecPlayer.getDuration());
1977                 break;
1978             }
1979         }
1980         assertTrue("Tunneled video playback timeout exceeded!",
1981                 timeOut > System.currentTimeMillis());
1982 
1983         Log.d(TAG, "playVideo player.reset()");
1984         mMediaCodecPlayer.reset();
1985     }
1986 
1987     /**
1988      * Test tunneled video playback flush if supported
1989      */
testTunneledVideoFlush()1990     public void testTunneledVideoFlush() throws Exception {
1991         if (!isVideoFeatureSupported(MediaFormat.MIMETYPE_VIDEO_AVC,
1992                 CodecCapabilities.FEATURE_TunneledPlayback)) {
1993             MediaUtils.skipTest(TAG, "No tunneled video playback codec found!");
1994             return;
1995         }
1996 
1997         AudioManager am = (AudioManager)mContext.getSystemService(Context.AUDIO_SERVICE);
1998         mMediaCodecPlayer = new MediaCodecTunneledPlayer(
1999                 getActivity().getSurfaceHolder(), true, am.generateAudioSessionId());
2000 
2001         mMediaCodecPlayer.setAudioDataSource(AUDIO_URL, null);
2002         mMediaCodecPlayer.setVideoDataSource(VIDEO_URL, null);
2003         assertTrue("MediaCodecPlayer.start() failed!", mMediaCodecPlayer.start());
2004         assertTrue("MediaCodecPlayer.prepare() failed!", mMediaCodecPlayer.prepare());
2005 
2006         // starts video playback
2007         mMediaCodecPlayer.startThread();
2008         Thread.sleep(SLEEP_TIME_MS);
2009         mMediaCodecPlayer.pause();
2010         mMediaCodecPlayer.flush();
2011         Thread.sleep(SLEEP_TIME_MS);
2012         mMediaCodecPlayer.reset();
2013     }
2014 }
2015 
2016