1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.cts; 18 19 import android.content.res.AssetFileDescriptor; 20 import android.media.MediaCodec; 21 import android.media.MediaCodec.BufferInfo; 22 import android.media.MediaExtractor; 23 import android.media.MediaFormat; 24 import android.media.MediaPlayer; 25 import android.media.cts.TestUtils.Monitor; 26 import android.net.Uri; 27 import android.os.Build; 28 import android.os.ParcelFileDescriptor; 29 import android.platform.test.annotations.AppModeFull; 30 import android.platform.test.annotations.Presubmit; 31 import android.platform.test.annotations.RequiresDevice; 32 import android.util.Log; 33 import android.view.Surface; 34 import android.webkit.cts.CtsTestServer; 35 36 import androidx.test.filters.SmallTest; 37 38 import com.android.compatibility.common.util.ApiLevelUtil; 39 import com.android.compatibility.common.util.MediaUtils; 40 41 import org.apache.http.Header; 42 import org.apache.http.HttpRequest; 43 import org.apache.http.impl.DefaultHttpServerConnection; 44 import org.apache.http.impl.io.SocketOutputBuffer; 45 import org.apache.http.io.SessionOutputBuffer; 46 import org.apache.http.params.HttpParams; 47 import org.apache.http.util.CharArrayBuffer; 48 49 import java.io.File; 50 import java.io.FileDescriptor; 51 import java.io.FileNotFoundException; 52 import java.io.IOException; 53 import java.net.Socket; 54 import java.nio.ByteBuffer; 55 import java.util.ArrayList; 56 import java.util.Arrays; 57 import java.util.HashMap; 58 import java.util.Map; 59 import java.util.Set; 60 import java.util.UUID; 61 import java.util.concurrent.CountDownLatch; 62 import java.util.concurrent.TimeUnit; 63 import java.util.zip.Adler32; 64 65 @SmallTest 66 @RequiresDevice 67 @AppModeFull(reason = "TODO: evaluate and port to instant") 68 public class NativeDecoderTest extends MediaPlayerTestBase { 69 private static final String TAG = "DecoderTest"; 70 71 private static final int RESET_MODE_NONE = 0; 72 private static final int RESET_MODE_RECONFIGURE = 1; 73 private static final int RESET_MODE_FLUSH = 2; 74 private static final int RESET_MODE_EOS_FLUSH = 3; 75 76 private static final String[] CSD_KEYS = new String[] { "csd-0", "csd-1" }; 77 78 private static final int CONFIG_MODE_NONE = 0; 79 private static final int CONFIG_MODE_QUEUE = 1; 80 81 private static boolean sIsAtLeastS = ApiLevelUtil.isAtLeast(Build.VERSION_CODES.S); 82 83 static final String mInpPrefix = WorkDir.getMediaDirString(); 84 short[] mMasterBuffer; 85 86 /** Load jni on initialization */ 87 static { 88 Log.i("@@@", "before loadlibrary"); 89 System.loadLibrary("ctsmediacodec_jni"); 90 Log.i("@@@", "after loadlibrary"); 91 } 92 93 @Override setUp()94 protected void setUp() throws Exception { 95 super.setUp(); 96 97 } 98 99 // check that native extractor behavior matches java extractor 100 compareArrays(String message, int[] a1, int[] a2)101 private void compareArrays(String message, int[] a1, int[] a2) { 102 if (a1 == a2) { 103 return; 104 } 105 106 assertNotNull(message + ": array 1 is null", a1); 107 assertNotNull(message + ": array 2 is null", a2); 108 109 assertEquals(message + ": arraylengths differ", a1.length, a2.length); 110 int length = a1.length; 111 112 for (int i = 0; i < length; i++) 113 if (a1[i] != a2[i]) { 114 Log.i("@@@@", Arrays.toString(a1)); 115 Log.i("@@@@", Arrays.toString(a2)); 116 fail(message + ": at index " + i); 117 } 118 } 119 SKIP_testExtractor()120 public void SKIP_testExtractor() throws Exception { 121 // duplicate of CtsMediaV2TestCases:ExtractorTest$FunctionalityTest#testExtract where 122 // checksum is computed over track format attributes, track buffer and buffer 123 // info in both SDK and NDK side and checked for equality 124 testExtractor("sinesweepogg.ogg"); 125 testExtractor("sinesweepoggmkv.mkv"); 126 testExtractor("sinesweepoggmp4.mp4"); 127 testExtractor("sinesweepmp3lame.mp3"); 128 testExtractor("sinesweepmp3smpb.mp3"); 129 testExtractor("sinesweepopus.mkv"); 130 testExtractor("sinesweepopusmp4.mp4"); 131 testExtractor("sinesweepm4a.m4a"); 132 testExtractor("sinesweepflacmkv.mkv"); 133 testExtractor("sinesweepflac.flac"); 134 testExtractor("sinesweepflacmp4.mp4"); 135 testExtractor("sinesweepwav.wav"); 136 137 testExtractor("video_1280x720_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz.mp4"); 138 testExtractor("bbb_s3_1280x720_webm_vp8_8mbps_60fps_opus_6ch_384kbps_48000hz.webm"); 139 testExtractor("bbb_s4_1280x720_webm_vp9_0p31_4mbps_30fps_opus_stereo_128kbps_48000hz.webm"); 140 testExtractor("video_1280x720_webm_av1_2000kbps_30fps_vorbis_stereo_128kbps_48000hz.webm"); 141 testExtractor("video_176x144_3gp_h263_300kbps_12fps_aac_mono_24kbps_11025hz.3gp"); 142 testExtractor("video_480x360_mp4_mpeg2_1500kbps_30fps_aac_stereo_128kbps_48000hz.mp4"); 143 testExtractor("video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz.mp4"); 144 145 CtsTestServer foo = new CtsTestServer(mContext); 146 testExtractor(foo.getAssetUrl("noiseandchirps.ogg"), null, null); 147 testExtractor(foo.getAssetUrl("ringer.mp3"), null, null); 148 testExtractor(foo.getRedirectingAssetUrl("ringer.mp3"), null, null); 149 150 String[] keys = new String[] {"header0", "header1"}; 151 String[] values = new String[] {"value0", "value1"}; 152 testExtractor(foo.getAssetUrl("noiseandchirps.ogg"), keys, values); 153 HttpRequest req = foo.getLastRequest("noiseandchirps.ogg"); 154 for (int i = 0; i < keys.length; i++) { 155 String key = keys[i]; 156 String value = values[i]; 157 Header[] header = req.getHeaders(key); 158 assertTrue("expecting " + key + ":" + value + ", saw " + Arrays.toString(header), 159 header.length == 1 && header[0].getValue().equals(value)); 160 } 161 162 String[] emptyArray = new String[0]; 163 testExtractor(foo.getAssetUrl("noiseandchirps.ogg"), emptyArray, emptyArray); 164 } 165 166 /** 167 * |keys| and |values| should be arrays of the same length. 168 * 169 * If keys or values is null, test {@link MediaExtractor#setDataSource(String)} 170 * and NDK counter part, i.e. set data source without headers. 171 * 172 * If keys or values is zero length, test {@link MediaExtractor#setDataSource(String, Map))} 173 * and NDK counter part with null headers. 174 * 175 */ testExtractor(String path, String[] keys, String[] values)176 private void testExtractor(String path, String[] keys, String[] values) throws Exception { 177 int[] jsizes = getSampleSizes(path, keys, values); 178 int[] nsizes = getSampleSizesNativePath(path, keys, values, /* testNativeSource = */ false); 179 int[] nsizes2 = getSampleSizesNativePath(path, keys, values, /* testNativeSource = */ true); 180 181 compareArrays("different samplesizes", jsizes, nsizes); 182 compareArrays("different samplesizes native source", jsizes, nsizes2); 183 } 184 getAssetFileDescriptorFor(final String res)185 protected static AssetFileDescriptor getAssetFileDescriptorFor(final String res) 186 throws FileNotFoundException { 187 Preconditions.assertTestFileExists(mInpPrefix + res); 188 File inpFile = new File(mInpPrefix + res); 189 ParcelFileDescriptor parcelFD = 190 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY); 191 return new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize()); 192 } 193 testExtractor(final String res)194 private void testExtractor(final String res) throws Exception { 195 AssetFileDescriptor fd = getAssetFileDescriptorFor(res); 196 197 int[] jsizes = getSampleSizes( 198 fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength()); 199 int[] nsizes = getSampleSizesNative( 200 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength()); 201 202 fd.close(); 203 compareArrays("different samples", jsizes, nsizes); 204 } 205 getSampleSizes(String path, String[] keys, String[] values)206 private static int[] getSampleSizes(String path, String[] keys, String[] values) throws IOException { 207 MediaExtractor ex = new MediaExtractor(); 208 if (keys == null || values == null) { 209 ex.setDataSource(path); 210 } else { 211 Map<String, String> headers = null; 212 int numheaders = Math.min(keys.length, values.length); 213 for (int i = 0; i < numheaders; i++) { 214 if (headers == null) { 215 headers = new HashMap<>(); 216 } 217 String key = keys[i]; 218 String value = values[i]; 219 headers.put(key, value); 220 } 221 ex.setDataSource(path, headers); 222 } 223 224 return getSampleSizes(ex); 225 } 226 getSampleSizes(FileDescriptor fd, long offset, long size)227 private static int[] getSampleSizes(FileDescriptor fd, long offset, long size) 228 throws IOException { 229 MediaExtractor ex = new MediaExtractor(); 230 ex.setDataSource(fd, offset, size); 231 return getSampleSizes(ex); 232 } 233 getSampleSizes(MediaExtractor ex)234 private static int[] getSampleSizes(MediaExtractor ex) { 235 ArrayList<Integer> foo = new ArrayList<Integer>(); 236 ByteBuffer buf = ByteBuffer.allocate(1024*1024); 237 int numtracks = ex.getTrackCount(); 238 assertTrue("no tracks", numtracks > 0); 239 foo.add(numtracks); 240 for (int i = 0; i < numtracks; i++) { 241 MediaFormat format = ex.getTrackFormat(i); 242 String mime = format.getString(MediaFormat.KEY_MIME); 243 if (mime.startsWith("audio/")) { 244 foo.add(0); 245 foo.add(format.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 246 foo.add(format.getInteger(MediaFormat.KEY_CHANNEL_COUNT)); 247 foo.add((int)format.getLong(MediaFormat.KEY_DURATION)); 248 } else if (mime.startsWith("video/")) { 249 foo.add(1); 250 foo.add(format.getInteger(MediaFormat.KEY_WIDTH)); 251 foo.add(format.getInteger(MediaFormat.KEY_HEIGHT)); 252 foo.add((int)format.getLong(MediaFormat.KEY_DURATION)); 253 } else { 254 fail("unexpected mime type: " + mime); 255 } 256 ex.selectTrack(i); 257 } 258 while(true) { 259 int n = ex.readSampleData(buf, 0); 260 if (n < 0) { 261 break; 262 } 263 foo.add(n); 264 foo.add(ex.getSampleTrackIndex()); 265 foo.add(ex.getSampleFlags()); 266 foo.add((int)ex.getSampleTime()); // just the low bits should be OK 267 byte foobar[] = new byte[n]; 268 buf.get(foobar, 0, n); 269 foo.add((int)adler32(foobar)); 270 ex.advance(); 271 } 272 273 int [] ret = new int[foo.size()]; 274 for (int i = 0; i < ret.length; i++) { 275 ret[i] = foo.get(i); 276 } 277 return ret; 278 } 279 getSampleSizesNative(int fd, long offset, long size)280 private static native int[] getSampleSizesNative(int fd, long offset, long size); getSampleSizesNativePath( String path, String[] keys, String[] values, boolean testNativeSource)281 private static native int[] getSampleSizesNativePath( 282 String path, String[] keys, String[] values, boolean testNativeSource); 283 284 @Presubmit SKIP_testExtractorFileDurationNative()285 public void SKIP_testExtractorFileDurationNative() throws Exception { 286 // duplicate of CtsMediaV2TestCases:ExtractorTest$FunctionalityTest#testExtract where 287 // checksum is computed over track format attributes, track buffer and buffer 288 // info in both SDK and NDK side and checked for equality. KEY_DURATION for each track is 289 // part of the checksum. 290 testExtractorFileDurationNative( 291 "video_1280x720_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz.mp4"); 292 } 293 testExtractorFileDurationNative(final String res)294 private void testExtractorFileDurationNative(final String res) throws Exception { 295 AssetFileDescriptor fd = getAssetFileDescriptorFor(res); 296 long durationUs = getExtractorFileDurationNative( 297 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength()); 298 299 MediaExtractor ex = new MediaExtractor(); 300 ex.setDataSource(fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength()); 301 302 int numtracks = ex.getTrackCount(); 303 long aDurationUs = -1, vDurationUs = -1; 304 for (int i = 0; i < numtracks; i++) { 305 MediaFormat format = ex.getTrackFormat(i); 306 String mime = format.getString(MediaFormat.KEY_MIME); 307 if (mime.startsWith("audio/")) { 308 aDurationUs = format.getLong(MediaFormat.KEY_DURATION); 309 } else if (mime.startsWith("video/")) { 310 vDurationUs = format.getLong(MediaFormat.KEY_DURATION); 311 } 312 } 313 314 assertTrue("duration inconsistency", 315 durationUs < 0 || durationUs >= aDurationUs && durationUs >= vDurationUs); 316 317 } 318 getExtractorFileDurationNative(int fd, long offset, long size)319 private static native long getExtractorFileDurationNative(int fd, long offset, long size); 320 321 @Presubmit SKIP_testExtractorCachedDurationNative()322 public void SKIP_testExtractorCachedDurationNative() throws Exception { 323 // duplicate of CtsMediaV2TestCases:ExtractorTest$SetDataSourceTest#testDataSourceNative 324 CtsTestServer foo = new CtsTestServer(mContext); 325 String url = foo.getAssetUrl("ringer.mp3"); 326 long cachedDurationUs = getExtractorCachedDurationNative(url, /* testNativeSource = */ false); 327 assertTrue("cached duration negative", cachedDurationUs >= 0); 328 cachedDurationUs = getExtractorCachedDurationNative(url, /* testNativeSource = */ true); 329 assertTrue("cached duration negative native source", cachedDurationUs >= 0); 330 } 331 getExtractorCachedDurationNative(String uri, boolean testNativeSource)332 private static native long getExtractorCachedDurationNative(String uri, boolean testNativeSource); 333 SKIP_testDecoder()334 public void SKIP_testDecoder() throws Exception { 335 // duplicate of CtsMediaV2TestCases:CodecDecoderTest#testSimpleDecode where checksum is 336 // computed over decoded output in both SDK and NDK side and checked for equality. 337 int testsRun = 338 testDecoder("sinesweepogg.ogg") + 339 testDecoder("sinesweepoggmkv.mkv") + 340 testDecoder("sinesweepoggmp4.mp4") + 341 testDecoder("sinesweepmp3lame.mp3") + 342 testDecoder("sinesweepmp3smpb.mp3") + 343 testDecoder("sinesweepopus.mkv") + 344 testDecoder("sinesweepopusmp4.mp4") + 345 testDecoder("sinesweepm4a.m4a") + 346 testDecoder("sinesweepflacmkv.mkv") + 347 testDecoder("sinesweepflac.flac") + 348 testDecoder("sinesweepflacmp4.mp4") + 349 testDecoder("sinesweepwav.wav") + 350 testDecoder("video_1280x720_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz.mp4") + 351 testDecoder("bbb_s1_640x360_webm_vp8_2mbps_30fps_vorbis_5ch_320kbps_48000hz.webm") + 352 testDecoder("bbb_s1_640x360_webm_vp9_0p21_1600kbps_30fps_vorbis_stereo_128kbps_48000hz.webm") + 353 testDecoder("video_176x144_3gp_h263_300kbps_12fps_aac_mono_24kbps_11025hz.3gp") + 354 testDecoder("video_480x360_mp4_mpeg2_1500kbps_30fps_aac_stereo_128kbps_48000hz.mp4"); 355 testDecoder("video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz.mp4"); 356 if (testsRun == 0) { 357 MediaUtils.skipTest("no decoders found"); 358 } 359 } 360 testDataSource()361 public void testDataSource() throws Exception { 362 int testsRun = testDecoder( 363 "video_176x144_3gp_h263_300kbps_12fps_aac_mono_24kbps_11025hz.3gp", /* wrapFd */ 364 true, /* useCallback */ false); 365 if (testsRun == 0) { 366 MediaUtils.skipTest("no decoders found"); 367 } 368 } 369 testDataSourceAudioOnly()370 public void testDataSourceAudioOnly() throws Exception { 371 int testsRun = testDecoder( 372 "loudsoftmp3.mp3", 373 /* wrapFd */ true, /* useCallback */ false) + 374 testDecoder( 375 "loudsoftaac.aac", 376 /* wrapFd */ false, /* useCallback */ false); 377 if (testsRun == 0) { 378 MediaUtils.skipTest("no decoders found"); 379 } 380 } 381 testDataSourceWithCallback()382 public void testDataSourceWithCallback() throws Exception { 383 int testsRun = testDecoder( 384 "video_176x144_3gp_h263_300kbps_12fps_aac_mono_24kbps_11025hz.3gp",/* wrapFd */ 385 true, /* useCallback */ true); 386 if (testsRun == 0) { 387 MediaUtils.skipTest("no decoders found"); 388 } 389 } 390 testDecoder(final String res)391 private int testDecoder(final String res) throws Exception { 392 return testDecoder(res, /* wrapFd */ false, /* useCallback */ false); 393 } 394 testDecoder(final String res, boolean wrapFd, boolean useCallback)395 private int testDecoder(final String res, boolean wrapFd, boolean useCallback) 396 throws Exception { 397 Preconditions.assertTestFileExists(mInpPrefix + res); 398 if (!MediaUtils.hasCodecsForResource(mInpPrefix + res)) { 399 return 0; // skip 400 } 401 402 AssetFileDescriptor fd = getAssetFileDescriptorFor(res); 403 404 int[] jdata1 = getDecodedData( 405 fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength()); 406 int[] jdata2 = getDecodedData( 407 fd.getFileDescriptor(), fd.getStartOffset(), fd.getLength()); 408 int[] ndata1 = getDecodedDataNative( 409 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength(), wrapFd, 410 useCallback); 411 int[] ndata2 = getDecodedDataNative( 412 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength(), wrapFd, 413 useCallback); 414 415 fd.close(); 416 compareArrays("inconsistent java decoder", jdata1, jdata2); 417 compareArrays("inconsistent native decoder", ndata1, ndata2); 418 compareArrays("different decoded data", jdata1, ndata1); 419 return 1; 420 } 421 getDecodedData(FileDescriptor fd, long offset, long size)422 private static int[] getDecodedData(FileDescriptor fd, long offset, long size) 423 throws IOException { 424 MediaExtractor ex = new MediaExtractor(); 425 ex.setDataSource(fd, offset, size); 426 return getDecodedData(ex); 427 } getDecodedData(MediaExtractor ex)428 private static int[] getDecodedData(MediaExtractor ex) throws IOException { 429 int numtracks = ex.getTrackCount(); 430 assertTrue("no tracks", numtracks > 0); 431 ArrayList<Integer>[] trackdata = new ArrayList[numtracks]; 432 MediaCodec[] codec = new MediaCodec[numtracks]; 433 MediaFormat[] format = new MediaFormat[numtracks]; 434 ByteBuffer[][] inbuffers = new ByteBuffer[numtracks][]; 435 ByteBuffer[][] outbuffers = new ByteBuffer[numtracks][]; 436 for (int i = 0; i < numtracks; i++) { 437 format[i] = ex.getTrackFormat(i); 438 String mime = format[i].getString(MediaFormat.KEY_MIME); 439 if (mime.startsWith("audio/") || mime.startsWith("video/")) { 440 codec[i] = MediaCodec.createDecoderByType(mime); 441 codec[i].configure(format[i], null, null, 0); 442 codec[i].start(); 443 inbuffers[i] = codec[i].getInputBuffers(); 444 outbuffers[i] = codec[i].getOutputBuffers(); 445 trackdata[i] = new ArrayList<Integer>(); 446 } else { 447 fail("unexpected mime type: " + mime); 448 } 449 ex.selectTrack(i); 450 } 451 452 boolean[] sawInputEOS = new boolean[numtracks]; 453 boolean[] sawOutputEOS = new boolean[numtracks]; 454 int eosCount = 0; 455 BufferInfo info = new BufferInfo(); 456 while(eosCount < numtracks) { 457 int t = ex.getSampleTrackIndex(); 458 if (t >= 0) { 459 assertFalse("saw input EOS twice", sawInputEOS[t]); 460 int bufidx = codec[t].dequeueInputBuffer(5000); 461 if (bufidx >= 0) { 462 Log.i("@@@@", "track " + t + " buffer " + bufidx); 463 ByteBuffer buf = inbuffers[t][bufidx]; 464 int sampleSize = ex.readSampleData(buf, 0); 465 Log.i("@@@@", "read " + sampleSize + " @ " + ex.getSampleTime()); 466 if (sampleSize < 0) { 467 sampleSize = 0; 468 sawInputEOS[t] = true; 469 Log.i("@@@@", "EOS"); 470 //break; 471 } 472 long presentationTimeUs = ex.getSampleTime(); 473 474 codec[t].queueInputBuffer(bufidx, 0, sampleSize, presentationTimeUs, 475 sawInputEOS[t] ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0); 476 ex.advance(); 477 } 478 } else { 479 Log.i("@@@@", "no more input samples"); 480 for (int tt = 0; tt < codec.length; tt++) { 481 if (!sawInputEOS[tt]) { 482 // we ran out of samples without ever signaling EOS to the codec, 483 // so do that now 484 int bufidx = codec[tt].dequeueInputBuffer(5000); 485 if (bufidx >= 0) { 486 codec[tt].queueInputBuffer(bufidx, 0, 0, 0, 487 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 488 sawInputEOS[tt] = true; 489 } 490 } 491 } 492 } 493 494 // see if any of the codecs have data available 495 for (int tt = 0; tt < codec.length; tt++) { 496 if (!sawOutputEOS[tt]) { 497 int status = codec[tt].dequeueOutputBuffer(info, 1); 498 if (status >= 0) { 499 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 500 Log.i("@@@@", "EOS on track " + tt); 501 sawOutputEOS[tt] = true; 502 eosCount++; 503 } 504 Log.i("@@@@", "got decoded buffer for track " + tt + ", size " + info.size); 505 if (info.size > 0) { 506 addSampleData(trackdata[tt], outbuffers[tt][status], info.size, format[tt]); 507 } 508 codec[tt].releaseOutputBuffer(status, false); 509 } else if (status == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 510 Log.i("@@@@", "output buffers changed for track " + tt); 511 outbuffers[tt] = codec[tt].getOutputBuffers(); 512 } else if (status == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 513 format[tt] = codec[tt].getOutputFormat(); 514 Log.i("@@@@", "format changed for track " + t + ": " + format[tt].toString()); 515 } else if (status == MediaCodec.INFO_TRY_AGAIN_LATER) { 516 Log.i("@@@@", "no buffer right now for track " + tt); 517 } else { 518 Log.i("@@@@", "unexpected info code for track " + tt + ": " + status); 519 } 520 } else { 521 Log.i("@@@@", "already at EOS on track " + tt); 522 } 523 } 524 } 525 526 int totalsize = 0; 527 for (int i = 0; i < numtracks; i++) { 528 totalsize += trackdata[i].size(); 529 } 530 int[] trackbytes = new int[totalsize]; 531 int idx = 0; 532 for (int i = 0; i < numtracks; i++) { 533 ArrayList<Integer> src = trackdata[i]; 534 int tracksize = src.size(); 535 for (int j = 0; j < tracksize; j++) { 536 trackbytes[idx++] = src.get(j); 537 } 538 } 539 540 for (int i = 0; i < codec.length; i++) { 541 codec[i].release(); 542 } 543 544 return trackbytes; 545 } 546 addSampleData(ArrayList<Integer> dst, ByteBuffer buf, int size, MediaFormat format)547 static void addSampleData(ArrayList<Integer> dst, 548 ByteBuffer buf, int size, MediaFormat format) throws IOException{ 549 550 Log.i("@@@", "addsample " + dst.size() + "/" + size); 551 int width = format.getInteger(MediaFormat.KEY_WIDTH, size); 552 int stride = format.getInteger(MediaFormat.KEY_STRIDE, width); 553 int height = format.getInteger(MediaFormat.KEY_HEIGHT, 1); 554 byte[] bb = new byte[width * height]; 555 int offset = buf.position(); 556 for (int i = 0; i < height; i++) { 557 buf.position(i * stride + offset); 558 buf.get(bb, i * width, width); 559 } 560 // bb is filled with data 561 long sum = adler32(bb); 562 dst.add( (int) (sum & 0xffffffff)); 563 } 564 565 private final static Adler32 checksummer = new Adler32(); 566 // simple checksum computed over every decoded buffer adler32(byte[] input)567 static int adler32(byte[] input) { 568 checksummer.reset(); 569 checksummer.update(input); 570 int ret = (int) checksummer.getValue(); 571 Log.i("@@@", "adler " + input.length + "/" + ret); 572 return ret; 573 } 574 getDecodedDataNative(int fd, long offset, long size, boolean wrapFd, boolean useCallback)575 private static native int[] getDecodedDataNative(int fd, long offset, long size, boolean wrapFd, 576 boolean useCallback) 577 throws IOException; 578 SKIP_testVideoPlayback()579 public void SKIP_testVideoPlayback() throws Exception { 580 // duplicate of 581 // CtsMediaV2TestCases:CodecDecoderSurfaceTest#testSimpleDecodeToSurfaceNative[*] 582 int testsRun = 583 testVideoPlayback( 584 "video_1280x720_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz.mp4") + 585 testVideoPlayback( 586 "bbb_s1_640x360_webm_vp8_2mbps_30fps_vorbis_5ch_320kbps_48000hz.webm") + 587 testVideoPlayback( 588 "bbb_s1_640x360_webm_vp9_0p21_1600kbps_30fps_vorbis_stereo_128kbps_48000hz.webm") + 589 testVideoPlayback( 590 "video_640x360_webm_av1_470kbps_30fps_vorbis_stereo_128kbps_48000hz.webm") + 591 testVideoPlayback( 592 "video_176x144_3gp_h263_300kbps_12fps_aac_mono_24kbps_11025hz.3gp") + 593 testVideoPlayback( 594 "video_176x144_mp4_mpeg2_105kbps_25fps_aac_stereo_128kbps_44100hz.mp4") + 595 testVideoPlayback( 596 "video_480x360_mp4_mpeg4_860kbps_25fps_aac_stereo_128kbps_44100hz.mp4"); 597 if (testsRun == 0) { 598 MediaUtils.skipTest("no decoders found"); 599 } 600 } 601 testVideoPlayback(final String res)602 private int testVideoPlayback(final String res) throws Exception { 603 Preconditions.assertTestFileExists(mInpPrefix + res); 604 if (!MediaUtils.checkCodecsForResource(mInpPrefix + res)) { 605 return 0; // skip 606 } 607 608 AssetFileDescriptor fd = getAssetFileDescriptorFor(res); 609 610 boolean ret = testPlaybackNative(mActivity.getSurfaceHolder().getSurface(), 611 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength()); 612 assertTrue("native playback error", ret); 613 return 1; 614 } 615 testPlaybackNative(Surface surface, int fd, long startOffset, long length)616 private static native boolean testPlaybackNative(Surface surface, 617 int fd, long startOffset, long length); 618 619 @Presubmit 620 @NonMediaMainlineTest testMuxerAvc()621 public void testMuxerAvc() throws Exception { 622 // IMPORTANT: this file must not have B-frames 623 testMuxer("video_1280x720_mp4_h264_1000kbps_25fps_aac_stereo_128kbps_44100hz.mp4", false); 624 } 625 626 @NonMediaMainlineTest testMuxerH263()627 public void testMuxerH263() throws Exception { 628 // IMPORTANT: this file must not have B-frames 629 testMuxer("video_176x144_3gp_h263_300kbps_25fps_aac_stereo_128kbps_11025hz.3gp", false); 630 } 631 632 @NonMediaMainlineTest testMuxerHevc()633 public void testMuxerHevc() throws Exception { 634 // IMPORTANT: this file must not have B-frames 635 testMuxer("video_640x360_mp4_hevc_450kbps_no_b.mp4", false); 636 } 637 638 @NonMediaMainlineTest testMuxerVp8()639 public void testMuxerVp8() throws Exception { 640 testMuxer("bbb_s1_640x360_webm_vp8_2mbps_30fps_vorbis_5ch_320kbps_48000hz.webm", true); 641 } 642 643 @NonMediaMainlineTest testMuxerVp9()644 public void testMuxerVp9() throws Exception { 645 testMuxer("video_1280x720_webm_vp9_csd_309kbps_25fps_vorbis_stereo_128kbps_48000hz.webm", 646 true); 647 } 648 649 @NonMediaMainlineTest testMuxerVp9NoCsd()650 public void testMuxerVp9NoCsd() throws Exception { 651 testMuxer("bbb_s1_640x360_webm_vp9_0p21_1600kbps_30fps_vorbis_stereo_128kbps_48000hz.webm", 652 true); 653 } 654 655 @NonMediaMainlineTest testMuxerVp9Hdr()656 public void testMuxerVp9Hdr() throws Exception { 657 testMuxer("video_256x144_webm_vp9_hdr_83kbps_24fps.webm", true); 658 } 659 660 // We do not support MPEG-2 muxing as of yet SKIP_testMuxerMpeg2()661 public void SKIP_testMuxerMpeg2() throws Exception { 662 // IMPORTANT: this file must not have B-frames 663 testMuxer("video_176x144_mp4_mpeg2_105kbps_25fps_aac_stereo_128kbps_44100hz.mp4", false); 664 } 665 666 @NonMediaMainlineTest testMuxerMpeg4()667 public void testMuxerMpeg4() throws Exception { 668 // IMPORTANT: this file must not have B-frames 669 testMuxer("video_176x144_mp4_mpeg4_300kbps_25fps_aac_stereo_128kbps_44100hz.mp4", false); 670 } 671 testMuxer(final String res, boolean webm)672 private void testMuxer(final String res, boolean webm) throws Exception { 673 Preconditions.assertTestFileExists(mInpPrefix + res); 674 if (!MediaUtils.checkCodecsForResource(mInpPrefix + res)) { 675 return; // skip 676 } 677 678 AssetFileDescriptor infd = getAssetFileDescriptorFor(res); 679 680 File base = mContext.getExternalFilesDir(null); 681 String tmpFile = base.getPath() + "/tmp.dat"; 682 Log.i("@@@", "using tmp file " + tmpFile); 683 new File(tmpFile).delete(); 684 ParcelFileDescriptor out = ParcelFileDescriptor.open(new File(tmpFile), 685 ParcelFileDescriptor.MODE_READ_WRITE | ParcelFileDescriptor.MODE_CREATE); 686 687 assertTrue("muxer failed", testMuxerNative( 688 infd.getParcelFileDescriptor().getFd(), infd.getStartOffset(), infd.getLength(), 689 out.getFd(), webm)); 690 691 // compare the original with the remuxed 692 MediaExtractor org = new MediaExtractor(); 693 org.setDataSource(infd.getFileDescriptor(), 694 infd.getStartOffset(), infd.getLength()); 695 696 MediaExtractor remux = new MediaExtractor(); 697 remux.setDataSource(out.getFileDescriptor()); 698 699 assertEquals("mismatched numer of tracks", org.getTrackCount(), remux.getTrackCount()); 700 // allow duration mismatch for webm files as ffmpeg does not consider the duration of the 701 // last frame while libwebm (and our framework) does. 702 final long maxDurationDiffUs = webm ? 50000 : 0; // 50ms for webm 703 for (int i = 0; i < org.getTrackCount(); i++) { 704 MediaFormat format1 = org.getTrackFormat(i); 705 MediaFormat format2 = remux.getTrackFormat(i); 706 Log.i("@@@", "org: " + format1); 707 Log.i("@@@", "remux: " + format2); 708 assertTrue("different formats", compareFormats(format1, format2, maxDurationDiffUs)); 709 } 710 711 org.release(); 712 remux.release(); 713 714 Preconditions.assertTestFileExists(mInpPrefix + res); 715 MediaPlayer player1 = 716 MediaPlayer.create(mContext, Uri.fromFile(new File(mInpPrefix + res))); 717 MediaPlayer player2 = MediaPlayer.create(mContext, Uri.parse("file://" + tmpFile)); 718 assertEquals("duration is different", 719 player1.getDuration(), player2.getDuration(), maxDurationDiffUs * 0.001); 720 player1.release(); 721 player2.release(); 722 new File(tmpFile).delete(); 723 } 724 hexString(ByteBuffer buf)725 private String hexString(ByteBuffer buf) { 726 if (buf == null) { 727 return "(null)"; 728 } 729 final char digits[] = 730 { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; 731 732 StringBuilder hex = new StringBuilder(); 733 for (int i = buf.position(); i < buf.limit(); ++i) { 734 byte c = buf.get(i); 735 hex.append(digits[(c >> 4) & 0xf]); 736 hex.append(digits[c & 0xf]); 737 } 738 return hex.toString(); 739 } 740 741 /** returns: null if key is in neither formats, true if they match and false otherwise */ compareByteBufferInFormats(MediaFormat f1, MediaFormat f2, String key)742 private Boolean compareByteBufferInFormats(MediaFormat f1, MediaFormat f2, String key) { 743 ByteBuffer bufF1 = f1.containsKey(key) ? f1.getByteBuffer(key) : null; 744 ByteBuffer bufF2 = f2.containsKey(key) ? f2.getByteBuffer(key) : null; 745 if (bufF1 == null && bufF2 == null) { 746 return null; 747 } 748 if (bufF1 == null || !bufF1.equals(bufF2)) { 749 Log.i("@@@", "org " + key + ": " + hexString(bufF1)); 750 Log.i("@@@", "rmx " + key + ": " + hexString(bufF2)); 751 return false; 752 } 753 return true; 754 } 755 compareFormats(MediaFormat f1, MediaFormat f2, long maxDurationDiffUs)756 private boolean compareFormats(MediaFormat f1, MediaFormat f2, long maxDurationDiffUs) { 757 final String KEY_DURATION = MediaFormat.KEY_DURATION; 758 759 // allow some difference in durations 760 if (maxDurationDiffUs > 0 761 && f1.containsKey(KEY_DURATION) && f2.containsKey(KEY_DURATION) 762 && Math.abs(f1.getLong(KEY_DURATION) 763 - f2.getLong(KEY_DURATION)) <= maxDurationDiffUs) { 764 f2.setLong(KEY_DURATION, f1.getLong(KEY_DURATION)); 765 } 766 767 // verify hdr-static-info 768 if (Boolean.FALSE.equals(compareByteBufferInFormats(f1, f2, "hdr-static-info"))) { 769 return false; 770 } 771 772 // verify CSDs 773 for (int i = 0;; ++i) { 774 String key = "csd-" + i; 775 Boolean match = compareByteBufferInFormats(f1, f2, key); 776 if (match == null) { 777 break; 778 } else if (match == false) { 779 return false; 780 } 781 } 782 783 // before S, mpeg4 writers jammed a fixed SAR value into the output; 784 // this was fixed in S 785 if (!sIsAtLeastS) { 786 if (f1.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT) 787 && f2.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT)) { 788 f2.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT, 789 f1.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_HEIGHT)); 790 } 791 if (f1.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH) 792 && f2.containsKey(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH)) { 793 f2.setInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH, 794 f1.getInteger(MediaFormat.KEY_PIXEL_ASPECT_RATIO_WIDTH)); 795 } 796 } 797 798 // look for f2 (the new) being a superset (>=) of f1 (the original) 799 // ensure that all of our fields in f1 appear in f2 with the same 800 // value. We allow f2 to contain extra fields. 801 Set<String> keys = f1.getKeys(); 802 for (String key: keys) { 803 if (key == null) { 804 continue; 805 } 806 if (!f2.containsKey(key)) { 807 return false; 808 } 809 int f1Type = f1.getValueTypeForKey(key); 810 if (f1Type != f2.getValueTypeForKey(key)) { 811 return false; 812 } 813 switch (f1Type) { 814 case MediaFormat.TYPE_INTEGER: 815 int f1Int = f1.getInteger(key); 816 int f2Int = f2.getInteger(key); 817 if (f1Int != f2Int) { 818 return false; 819 } 820 break; 821 case MediaFormat.TYPE_LONG: 822 long f1Long = f1.getLong(key); 823 long f2Long = f2.getLong(key); 824 if (f1Long != f2Long) { 825 return false; 826 } 827 break; 828 case MediaFormat.TYPE_FLOAT: 829 float f1Float = f1.getFloat(key); 830 float f2Float = f2.getFloat(key); 831 if (f1Float != f2Float) { 832 return false; 833 } 834 break; 835 case MediaFormat.TYPE_STRING: 836 String f1String = f1.getString(key); 837 String f2String = f2.getString(key); 838 if (!f1String.equals(f2String)) { 839 return false; 840 } 841 break; 842 case MediaFormat.TYPE_BYTE_BUFFER: 843 ByteBuffer f1ByteBuffer = f1.getByteBuffer(key); 844 ByteBuffer f2ByteBuffer = f2.getByteBuffer(key); 845 if (!f1ByteBuffer.equals(f2ByteBuffer)) { 846 return false; 847 } 848 break; 849 default: 850 return false; 851 } 852 } 853 854 // repeat for getFeatures 855 // (which we don't use in this test, but include for completeness) 856 Set<String> features = f1.getFeatures(); 857 for (String key: features) { 858 if (key == null) { 859 continue; 860 } 861 if (!f2.containsKey(key)) { 862 return false; 863 } 864 int f1Type = f1.getValueTypeForKey(key); 865 if (f1Type != f2.getValueTypeForKey(key)) { 866 return false; 867 } 868 switch (f1Type) { 869 case MediaFormat.TYPE_INTEGER: 870 int f1Int = f1.getInteger(key); 871 int f2Int = f2.getInteger(key); 872 if (f1Int != f2Int) { 873 return false; 874 } 875 break; 876 case MediaFormat.TYPE_LONG: 877 long f1Long = f1.getLong(key); 878 long f2Long = f2.getLong(key); 879 if (f1Long != f2Long) { 880 return false; 881 } 882 break; 883 case MediaFormat.TYPE_FLOAT: 884 float f1Float = f1.getFloat(key); 885 float f2Float = f2.getFloat(key); 886 if (f1Float != f2Float) { 887 return false; 888 } 889 break; 890 case MediaFormat.TYPE_STRING: 891 String f1String = f1.getString(key); 892 String f2String = f2.getString(key); 893 if (!f1String.equals(f2String)) { 894 return false; 895 } 896 break; 897 case MediaFormat.TYPE_BYTE_BUFFER: 898 ByteBuffer f1ByteBuffer = f1.getByteBuffer(key); 899 ByteBuffer f2ByteBuffer = f2.getByteBuffer(key); 900 if (!f1ByteBuffer.equals(f2ByteBuffer)) { 901 return false; 902 } 903 break; 904 default: 905 return false; 906 } 907 } 908 909 // not otherwise disqualified 910 return true; 911 } 912 testMuxerNative(int in, long inoffset, long insize, int out, boolean webm)913 private static native boolean testMuxerNative(int in, long inoffset, long insize, 914 int out, boolean webm); 915 916 @Presubmit testFormat()917 public void testFormat() throws Exception { 918 assertTrue("media format fail, see log for details", testFormatNative()); 919 } 920 testFormatNative()921 private static native boolean testFormatNative(); 922 923 @Presubmit testPssh()924 public void testPssh() throws Exception { 925 testPssh("psshtest.mp4"); 926 } 927 testPssh(final String res)928 private void testPssh(final String res) throws Exception { 929 AssetFileDescriptor fd = getAssetFileDescriptorFor(res); 930 931 MediaExtractor ex = new MediaExtractor(); 932 ex.setDataSource(fd.getParcelFileDescriptor().getFileDescriptor(), 933 fd.getStartOffset(), fd.getLength()); 934 testPssh(ex); 935 ex.release(); 936 937 boolean ret = testPsshNative( 938 fd.getParcelFileDescriptor().getFd(), fd.getStartOffset(), fd.getLength()); 939 assertTrue("native pssh error", ret); 940 } 941 testPssh(MediaExtractor ex)942 private static void testPssh(MediaExtractor ex) { 943 Map<UUID, byte[]> map = ex.getPsshInfo(); 944 Set<UUID> keys = map.keySet(); 945 for (UUID uuid: keys) { 946 Log.i("@@@", "uuid: " + uuid + ", data size " + 947 map.get(uuid).length); 948 } 949 } 950 testPsshNative(int fd, long offset, long size)951 private static native boolean testPsshNative(int fd, long offset, long size); 952 testCryptoInfo()953 public void testCryptoInfo() throws Exception { 954 assertTrue("native cryptoinfo failed, see log for details", testCryptoInfoNative()); 955 } 956 testCryptoInfoNative()957 private static native boolean testCryptoInfoNative(); 958 959 @Presubmit testMediaFormat()960 public void testMediaFormat() throws Exception { 961 assertTrue("native mediaformat failed, see log for details", testMediaFormatNative()); 962 } 963 testMediaFormatNative()964 private static native boolean testMediaFormatNative(); 965 966 @Presubmit testAMediaDataSourceClose()967 public void testAMediaDataSourceClose() throws Throwable { 968 969 final CtsTestServer slowServer = new SlowCtsTestServer(); 970 final String url = slowServer.getAssetUrl("noiseandchirps.ogg"); 971 final long ds = createAMediaDataSource(url); 972 final long ex = createAMediaExtractor(); 973 974 try { 975 setAMediaExtractorDataSourceAndFailIfAnr(ex, ds); 976 } finally { 977 slowServer.shutdown(); 978 deleteAMediaExtractor(ex); 979 deleteAMediaDataSource(ds); 980 } 981 982 } 983 setAMediaExtractorDataSourceAndFailIfAnr(final long ex, final long ds)984 private void setAMediaExtractorDataSourceAndFailIfAnr(final long ex, final long ds) 985 throws Throwable { 986 final Monitor setAMediaExtractorDataSourceDone = new Monitor(); 987 final int HEAD_START_MILLIS = 1000; 988 final int ANR_TIMEOUT_MILLIS = 2500; 989 final int JOIN_TIMEOUT_MILLIS = 1500; 990 991 Thread setAMediaExtractorDataSourceThread = new Thread() { 992 public void run() { 993 setAMediaExtractorDataSource(ex, ds); 994 setAMediaExtractorDataSourceDone.signal(); 995 } 996 }; 997 998 try { 999 setAMediaExtractorDataSourceThread.start(); 1000 Thread.sleep(HEAD_START_MILLIS); 1001 closeAMediaDataSource(ds); 1002 boolean closed = setAMediaExtractorDataSourceDone.waitForSignal(ANR_TIMEOUT_MILLIS); 1003 assertTrue("close took longer than " + ANR_TIMEOUT_MILLIS, closed); 1004 } finally { 1005 setAMediaExtractorDataSourceThread.join(JOIN_TIMEOUT_MILLIS); 1006 } 1007 1008 } 1009 1010 private class SlowCtsTestServer extends CtsTestServer { 1011 1012 private static final int SERVER_DELAY_MILLIS = 5000; 1013 private final CountDownLatch mDisconnected = new CountDownLatch(1); 1014 SlowCtsTestServer()1015 SlowCtsTestServer() throws Exception { 1016 super(mContext); 1017 } 1018 1019 @Override createHttpServerConnection()1020 protected DefaultHttpServerConnection createHttpServerConnection() { 1021 return new SlowHttpServerConnection(mDisconnected, SERVER_DELAY_MILLIS); 1022 } 1023 1024 @Override shutdown()1025 public void shutdown() { 1026 mDisconnected.countDown(); 1027 super.shutdown(); 1028 } 1029 } 1030 1031 private static class SlowHttpServerConnection extends DefaultHttpServerConnection { 1032 1033 private final CountDownLatch mDisconnected; 1034 private final int mDelayMillis; 1035 SlowHttpServerConnection(CountDownLatch disconnected, int delayMillis)1036 public SlowHttpServerConnection(CountDownLatch disconnected, int delayMillis) { 1037 mDisconnected = disconnected; 1038 mDelayMillis = delayMillis; 1039 } 1040 1041 @Override createHttpDataTransmitter( Socket socket, int buffersize, HttpParams params)1042 protected SessionOutputBuffer createHttpDataTransmitter( 1043 Socket socket, int buffersize, HttpParams params) throws IOException { 1044 return createSessionOutputBuffer(socket, buffersize, params); 1045 } 1046 createSessionOutputBuffer( Socket socket, int buffersize, HttpParams params)1047 SessionOutputBuffer createSessionOutputBuffer( 1048 Socket socket, int buffersize, HttpParams params) throws IOException { 1049 return new SocketOutputBuffer(socket, buffersize, params) { 1050 @Override 1051 public void write(byte[] b) throws IOException { 1052 write(b, 0, b.length); 1053 } 1054 1055 @Override 1056 public void write(byte[] b, int off, int len) throws IOException { 1057 while (len-- > 0) { 1058 write(b[off++]); 1059 } 1060 } 1061 1062 @Override 1063 public void writeLine(String s) throws IOException { 1064 delay(); 1065 super.writeLine(s); 1066 } 1067 1068 @Override 1069 public void writeLine(CharArrayBuffer buffer) throws IOException { 1070 delay(); 1071 super.writeLine(buffer); 1072 } 1073 1074 @Override 1075 public void write(int b) throws IOException { 1076 delay(); 1077 super.write(b); 1078 } 1079 1080 private void delay() throws IOException { 1081 try { 1082 mDisconnected.await(mDelayMillis, TimeUnit.MILLISECONDS); 1083 } catch (InterruptedException e) { 1084 // Ignored 1085 } 1086 } 1087 1088 }; 1089 } 1090 } 1091 1092 private static native long createAMediaExtractor(); 1093 private static native long createAMediaDataSource(String url); 1094 private static native int setAMediaExtractorDataSource(long ex, long ds); 1095 private static native void closeAMediaDataSource(long ds); 1096 private static native void deleteAMediaExtractor(long ex); 1097 private static native void deleteAMediaDataSource(long ds); 1098 1099 } 1100 1101