1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.cts; 18 19 import android.annotation.TargetApi; 20 import android.content.res.AssetFileDescriptor; 21 import android.media.MediaCodec; 22 import android.media.MediaCodecInfo; 23 import android.media.MediaCodecInfo.CodecCapabilities; 24 import android.media.MediaCodecInfo.CodecProfileLevel; 25 import android.media.MediaCodecList; 26 import android.media.MediaExtractor; 27 import android.media.MediaFormat; 28 import android.media.MediaMuxer; 29 import android.media.MediaPlayer; 30 import android.os.Environment; 31 import android.os.ParcelFileDescriptor; 32 import android.platform.test.annotations.AppModeFull; 33 import android.test.ActivityInstrumentationTestCase2; 34 import android.util.Log; 35 import android.view.Surface; 36 37 import android.media.MediaCodecInfo; 38 import android.media.MediaCodecInfo.CodecCapabilities; 39 import android.media.MediaCodecInfo.CodecProfileLevel; 40 41 import java.io.File; 42 import java.io.FileNotFoundException; 43 import java.io.IOException; 44 import java.nio.ByteBuffer; 45 import java.util.concurrent.atomic.AtomicReference; 46 import java.util.concurrent.CountDownLatch; 47 48 /** 49 * Test for the integration of MediaMuxer and MediaCodec's encoder. 50 * 51 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 52 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 53 * them into a file. 54 * 55 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 56 * fails along the way. 57 * 58 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 59 * MediaMuxer. 60 */ 61 @TargetApi(18) 62 @AppModeFull(reason = "Instant apps cannot access the SD card") 63 public class ExtractDecodeEditEncodeMuxTest 64 extends ActivityInstrumentationTestCase2<MediaStubActivity> { 65 66 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 67 private static final boolean VERBOSE = false; // lots of logging 68 static final String mInpPrefix = WorkDir.getMediaDirString(); 69 70 /** How long to wait for the next buffer to become available. */ 71 private static final int TIMEOUT_USEC = 10000; 72 73 /** Where to output the test files. */ 74 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 75 76 // parameters for the video encoder 77 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 78 private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps 79 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 80 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 81 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 82 83 // parameters for the audio encoder 84 // Advanced Audio Coding 85 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 86 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 87 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 88 private static final int OUTPUT_AUDIO_AAC_PROFILE = 89 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 90 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 91 92 /** 93 * Used for editing the frames. 94 * 95 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 96 */ 97 private static final String FRAGMENT_SHADER = 98 "#extension GL_OES_EGL_image_external : require\n" + 99 "precision mediump float;\n" + 100 "varying vec2 vTextureCoord;\n" + 101 "uniform samplerExternalOES sTexture;\n" + 102 "void main() {\n" + 103 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 104 "}\n"; 105 106 /** Whether to copy the video from the test video. */ 107 private boolean mCopyVideo; 108 /** Whether to copy the audio from the test video. */ 109 private boolean mCopyAudio; 110 /** Whether to verify the audio format. */ 111 private boolean mVerifyAudioFormat; 112 /** Width of the output frames. */ 113 private int mWidth = -1; 114 /** Height of the output frames. */ 115 private int mHeight = -1; 116 117 /** The raw resource used as the input file. */ 118 private String mSourceRes; 119 120 /** The destination file for the encoded output. */ 121 private String mOutputFile; 122 123 private String mOutputVideoMimeType; 124 ExtractDecodeEditEncodeMuxTest()125 public ExtractDecodeEditEncodeMuxTest() { 126 super(MediaStubActivity.class); 127 } 128 testExtractDecodeEditEncodeMuxQCIF()129 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 130 if(!setSize(176, 144)) return; 131 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 132 setCopyVideo(); 133 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 134 TestWrapper.runTest(this); 135 } 136 testExtractDecodeEditEncodeMuxQVGA()137 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 138 if(!setSize(320, 240)) return; 139 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 140 setCopyVideo(); 141 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 142 TestWrapper.runTest(this); 143 } 144 testExtractDecodeEditEncodeMux720p()145 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 146 if(!setSize(1280, 720)) return; 147 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 148 setCopyVideo(); 149 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 150 TestWrapper.runTest(this); 151 } 152 testExtractDecodeEditEncodeMux2160pHevc()153 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 154 if(!setSize(3840, 2160)) return; 155 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 156 setCopyVideo(); 157 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 158 TestWrapper.runTest(this); 159 } 160 testExtractDecodeEditEncodeMuxAudio()161 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 162 if(!setSize(1280, 720)) return; 163 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 164 setCopyAudio(); 165 setVerifyAudioFormat(); 166 TestWrapper.runTest(this); 167 } 168 testExtractDecodeEditEncodeMuxAudioVideo()169 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 170 if(!setSize(1280, 720)) return; 171 setSource("video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz.mp4"); 172 setCopyAudio(); 173 setCopyVideo(); 174 setVerifyAudioFormat(); 175 TestWrapper.runTest(this); 176 } 177 178 /** Wraps testExtractDecodeEditEncodeMux() */ 179 private static class TestWrapper implements Runnable { 180 private Throwable mThrowable; 181 private ExtractDecodeEditEncodeMuxTest mTest; 182 TestWrapper(ExtractDecodeEditEncodeMuxTest test)183 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 184 mTest = test; 185 } 186 187 @Override run()188 public void run() { 189 try { 190 mTest.extractDecodeEditEncodeMux(); 191 } catch (Throwable th) { 192 mThrowable = th; 193 } 194 } 195 196 /** 197 * Entry point. 198 */ runTest(ExtractDecodeEditEncodeMuxTest test)199 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 200 test.setOutputFile(); 201 TestWrapper wrapper = new TestWrapper(test); 202 Thread th = new Thread(wrapper, "codec test"); 203 th.start(); 204 th.join(); 205 if (wrapper.mThrowable != null) { 206 throw wrapper.mThrowable; 207 } 208 } 209 } 210 211 /** 212 * Sets the test to copy the video stream. 213 */ setCopyVideo()214 private void setCopyVideo() { 215 mCopyVideo = true; 216 } 217 218 /** 219 * Sets the test to copy the video stream. 220 */ setCopyAudio()221 private void setCopyAudio() { 222 mCopyAudio = true; 223 } 224 225 /** 226 * Sets the test to verify the output audio format. 227 */ setVerifyAudioFormat()228 private void setVerifyAudioFormat() { 229 mVerifyAudioFormat = true; 230 } 231 232 /** 233 * Sets the desired frame size and returns whether the given resolution is 234 * supported. 235 * 236 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 237 * is supported. For other codecs, always return {@code true}. 238 */ setSize(int width, int height)239 private boolean setSize(int width, int height) { 240 if ((width % 16) != 0 || (height % 16) != 0) { 241 Log.w(TAG, "WARNING: width or height not multiple of 16"); 242 } 243 mWidth = width; 244 mHeight = height; 245 246 // TODO: remove this logic in setSize as it is now handled when configuring codecs 247 return true; 248 } 249 250 /** 251 * Sets the raw resource used as the source video. 252 */ setSource(String res)253 private void setSource(String res) { 254 mSourceRes = res; 255 } 256 257 /** 258 * Sets the name of the output file based on the other parameters. 259 * 260 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(String)}. 261 */ setOutputFile()262 private void setOutputFile() { 263 StringBuilder sb = new StringBuilder(); 264 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 265 sb.append("/cts-media-"); 266 sb.append(getClass().getSimpleName()); 267 assertTrue("should have called setSource() first", mSourceRes != null); 268 sb.append('-'); 269 sb.append(mSourceRes); 270 if (mCopyVideo) { 271 assertTrue("should have called setSize() first", mWidth != -1); 272 assertTrue("should have called setSize() first", mHeight != -1); 273 sb.append('-'); 274 sb.append("video"); 275 sb.append('-'); 276 sb.append(mWidth); 277 sb.append('x'); 278 sb.append(mHeight); 279 } 280 if (mCopyAudio) { 281 sb.append('-'); 282 sb.append("audio"); 283 } 284 sb.append(".mp4"); 285 mOutputFile = sb.toString(); 286 } 287 setVideoMimeType(String mimeType)288 private void setVideoMimeType(String mimeType) { 289 mOutputVideoMimeType = mimeType; 290 } 291 292 /** 293 * Tests encoding and subsequently decoding video from frames generated into a buffer. 294 * <p> 295 * We encode several frames of a video test pattern using MediaCodec, then decode the output 296 * with MediaCodec and do some simple checks. 297 */ extractDecodeEditEncodeMux()298 private void extractDecodeEditEncodeMux() throws Exception { 299 // Exception that may be thrown during release. 300 Exception exception = null; 301 302 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 303 304 // We avoid the device-specific limitations on width and height by using values 305 // that are multiples of 16, which all tested devices seem to be able to handle. 306 MediaFormat outputVideoFormat = 307 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 308 309 // Set some properties. Failing to specify some of these can cause the MediaCodec 310 // configure() call to throw an unhelpful exception. 311 outputVideoFormat.setInteger( 312 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 313 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 314 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 315 outputVideoFormat.setInteger( 316 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 317 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 318 319 String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 320 if (videoEncoderName == null) { 321 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 322 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 323 return; 324 } 325 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 326 327 MediaFormat outputAudioFormat = 328 MediaFormat.createAudioFormat( 329 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 330 OUTPUT_AUDIO_CHANNEL_COUNT); 331 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 332 // TODO: Bug workaround --- uncomment once fixed. 333 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 334 335 String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 336 if (audioEncoderName == null) { 337 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 338 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 339 return; 340 } 341 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 342 343 MediaExtractor videoExtractor = null; 344 MediaExtractor audioExtractor = null; 345 OutputSurface outputSurface = null; 346 MediaCodec videoDecoder = null; 347 MediaCodec audioDecoder = null; 348 MediaCodec videoEncoder = null; 349 MediaCodec audioEncoder = null; 350 MediaMuxer muxer = null; 351 352 InputSurface inputSurface = null; 353 354 try { 355 if (mCopyVideo) { 356 videoExtractor = createExtractor(); 357 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 358 assertTrue("missing video track in test video", videoInputTrack != -1); 359 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 360 361 // Create a MediaCodec for the desired codec, then configure it as an encoder with 362 // our desired properties. Request a Surface to use for input. 363 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 364 videoEncoder = createVideoEncoder( 365 videoEncoderName, outputVideoFormat, inputSurfaceReference); 366 inputSurface = new InputSurface(inputSurfaceReference.get()); 367 inputSurface.makeCurrent(); 368 // Create a MediaCodec for the decoder, based on the extractor's format. 369 outputSurface = new OutputSurface(); 370 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 371 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 372 } 373 374 if (mCopyAudio) { 375 audioExtractor = createExtractor(); 376 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 377 assertTrue("missing audio track in test video", audioInputTrack != -1); 378 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 379 380 // Create a MediaCodec for the desired codec, then configure it as an encoder with 381 // our desired properties. Request a Surface to use for input. 382 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 383 // Create a MediaCodec for the decoder, based on the extractor's format. 384 audioDecoder = createAudioDecoder(mcl, inputFormat); 385 } 386 387 // Creates a muxer but do not start or add tracks just yet. 388 muxer = createMuxer(); 389 390 doExtractDecodeEditEncodeMux( 391 videoExtractor, 392 audioExtractor, 393 videoDecoder, 394 videoEncoder, 395 audioDecoder, 396 audioEncoder, 397 muxer, 398 inputSurface, 399 outputSurface); 400 } finally { 401 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 402 // Try to release everything we acquired, even if one of the releases fails, in which 403 // case we save the first exception we got and re-throw at the end (unless something 404 // other exception has already been thrown). This guarantees the first exception thrown 405 // is reported as the cause of the error, everything is (attempted) to be released, and 406 // all other exceptions appear in the logs. 407 try { 408 if (videoExtractor != null) { 409 videoExtractor.release(); 410 } 411 } catch(Exception e) { 412 Log.e(TAG, "error while releasing videoExtractor", e); 413 if (exception == null) { 414 exception = e; 415 } 416 } 417 try { 418 if (audioExtractor != null) { 419 audioExtractor.release(); 420 } 421 } catch(Exception e) { 422 Log.e(TAG, "error while releasing audioExtractor", e); 423 if (exception == null) { 424 exception = e; 425 } 426 } 427 try { 428 if (videoDecoder != null) { 429 videoDecoder.stop(); 430 videoDecoder.release(); 431 } 432 } catch(Exception e) { 433 Log.e(TAG, "error while releasing videoDecoder", e); 434 if (exception == null) { 435 exception = e; 436 } 437 } 438 try { 439 if (outputSurface != null) { 440 outputSurface.release(); 441 } 442 } catch(Exception e) { 443 Log.e(TAG, "error while releasing outputSurface", e); 444 if (exception == null) { 445 exception = e; 446 } 447 } 448 try { 449 if (videoEncoder != null) { 450 videoEncoder.stop(); 451 videoEncoder.release(); 452 } 453 } catch(Exception e) { 454 Log.e(TAG, "error while releasing videoEncoder", e); 455 if (exception == null) { 456 exception = e; 457 } 458 } 459 try { 460 if (audioDecoder != null) { 461 audioDecoder.stop(); 462 audioDecoder.release(); 463 } 464 } catch(Exception e) { 465 Log.e(TAG, "error while releasing audioDecoder", e); 466 if (exception == null) { 467 exception = e; 468 } 469 } 470 try { 471 if (audioEncoder != null) { 472 audioEncoder.stop(); 473 audioEncoder.release(); 474 } 475 } catch(Exception e) { 476 Log.e(TAG, "error while releasing audioEncoder", e); 477 if (exception == null) { 478 exception = e; 479 } 480 } 481 try { 482 if (muxer != null) { 483 muxer.stop(); 484 muxer.release(); 485 } 486 } catch(Exception e) { 487 Log.e(TAG, "error while releasing muxer", e); 488 if (exception == null) { 489 exception = e; 490 } 491 } 492 try { 493 if (inputSurface != null) { 494 inputSurface.release(); 495 } 496 } catch(Exception e) { 497 Log.e(TAG, "error while releasing inputSurface", e); 498 if (exception == null) { 499 exception = e; 500 } 501 } 502 } 503 if (exception != null) { 504 throw exception; 505 } 506 507 MediaExtractor mediaExtractor = null; 508 try { 509 mediaExtractor = new MediaExtractor(); 510 mediaExtractor.setDataSource(mOutputFile); 511 512 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 513 mediaExtractor.getTrackCount()); 514 if (mVerifyAudioFormat) { 515 boolean foundAudio = false; 516 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 517 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 518 if (isAudioFormat(trackFormat)) { 519 foundAudio = true; 520 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 521 522 // SBR mode halves the sample rate in the format. 523 if (OUTPUT_AUDIO_AAC_PROFILE == 524 MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 525 expectedSampleRate /= 2; 526 } 527 assertEquals("sample rates should match", expectedSampleRate, 528 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 529 } 530 } 531 532 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 533 } 534 } catch (IOException e) { 535 throw new IllegalStateException("exception verifying output file", e); 536 } finally { 537 if (mediaExtractor != null) { 538 mediaExtractor.release(); 539 } 540 } 541 542 // TODO: Check the generated output file's video format and sample data. 543 544 MediaStubActivity activity = getActivity(); 545 final MediaPlayer mp = new MediaPlayer(); 546 final Exception[] exceptionHolder = { null }; 547 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 548 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 549 @Override 550 public boolean onError(MediaPlayer origin, int what, int extra) { 551 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 552 + " extra=" + extra); 553 // Returning false would trigger onCompletion() so that 554 // playbackEndSignal.await() can stop waiting. 555 return false; 556 } 557 }); 558 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 559 @Override 560 public void onCompletion(MediaPlayer origin) { 561 playbackEndSignal.countDown(); 562 } 563 }); 564 try { 565 mp.setDataSource(mOutputFile); 566 mp.setDisplay(activity.getSurfaceHolder()); 567 mp.prepare(); 568 mp.start(); 569 playbackEndSignal.await(); 570 } catch (Exception e) { 571 exceptionHolder[0] = e; 572 } finally { 573 mp.release(); 574 } 575 576 if (exceptionHolder[0] != null) { 577 throw exceptionHolder[0]; 578 } 579 } 580 getAssetFileDescriptorFor(final String res)581 protected AssetFileDescriptor getAssetFileDescriptorFor(final String res) 582 throws FileNotFoundException { 583 Preconditions.assertTestFileExists(mInpPrefix + res); 584 File inpFile = new File(mInpPrefix + res); 585 ParcelFileDescriptor parcelFD = 586 ParcelFileDescriptor.open(inpFile, ParcelFileDescriptor.MODE_READ_ONLY); 587 return new AssetFileDescriptor(parcelFD, 0, parcelFD.getStatSize()); 588 } 589 590 /** 591 * Creates an extractor that reads its frames from {@link #mSourceRes}. 592 */ createExtractor()593 private MediaExtractor createExtractor() throws IOException { 594 MediaExtractor extractor; 595 AssetFileDescriptor srcFd = getAssetFileDescriptorFor(mSourceRes); 596 extractor = new MediaExtractor(); 597 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 598 srcFd.getLength()); 599 return extractor; 600 } 601 602 /** 603 * Creates a decoder for the given format, which outputs to the given surface. 604 * 605 * @param inputFormat the format of the stream to decode 606 * @param surface into which to decode the frames 607 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)608 private MediaCodec createVideoDecoder( 609 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 610 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 611 decoder.configure(inputFormat, surface, null, 0); 612 decoder.start(); 613 return decoder; 614 } 615 616 /** 617 * Creates an encoder for the given format using the specified codec, taking input from a 618 * surface. 619 * 620 * <p>The surface to use as input is stored in the given reference. 621 * 622 * @param codecInfo of the codec to use 623 * @param format of the stream to be produced 624 * @param surfaceReference to store the surface to use as input 625 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)626 private MediaCodec createVideoEncoder( 627 String codecName, 628 MediaFormat format, 629 AtomicReference<Surface> surfaceReference) 630 throws IOException { 631 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 632 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 633 // Must be called before start() is. 634 surfaceReference.set(encoder.createInputSurface()); 635 encoder.start(); 636 return encoder; 637 } 638 639 /** 640 * Creates a decoder for the given format. 641 * 642 * @param inputFormat the format of the stream to decode 643 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)644 private MediaCodec createAudioDecoder( 645 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 646 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 647 decoder.configure(inputFormat, null, null, 0); 648 decoder.start(); 649 return decoder; 650 } 651 652 /** 653 * Creates an encoder for the given format using the specified codec. 654 * 655 * @param codecInfo of the codec to use 656 * @param format of the stream to be produced 657 */ createAudioEncoder(String codecName, MediaFormat format)658 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 659 throws IOException { 660 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 661 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 662 encoder.start(); 663 return encoder; 664 } 665 666 /** 667 * Creates a muxer to write the encoded frames. 668 * 669 * <p>The muxer is not started as it needs to be started only after all streams have been added. 670 */ createMuxer()671 private MediaMuxer createMuxer() throws IOException { 672 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 673 } 674 getAndSelectVideoTrackIndex(MediaExtractor extractor)675 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 676 for (int index = 0; index < extractor.getTrackCount(); ++index) { 677 if (VERBOSE) { 678 Log.d(TAG, "format for track " + index + " is " 679 + getMimeTypeFor(extractor.getTrackFormat(index))); 680 } 681 if (isVideoFormat(extractor.getTrackFormat(index))) { 682 extractor.selectTrack(index); 683 return index; 684 } 685 } 686 return -1; 687 } 688 getAndSelectAudioTrackIndex(MediaExtractor extractor)689 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 690 for (int index = 0; index < extractor.getTrackCount(); ++index) { 691 if (VERBOSE) { 692 Log.d(TAG, "format for track " + index + " is " 693 + getMimeTypeFor(extractor.getTrackFormat(index))); 694 } 695 if (isAudioFormat(extractor.getTrackFormat(index))) { 696 extractor.selectTrack(index); 697 return index; 698 } 699 } 700 return -1; 701 } 702 703 /** 704 * Does the actual work for extracting, decoding, encoding and muxing. 705 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)706 private void doExtractDecodeEditEncodeMux( 707 MediaExtractor videoExtractor, 708 MediaExtractor audioExtractor, 709 MediaCodec videoDecoder, 710 MediaCodec videoEncoder, 711 MediaCodec audioDecoder, 712 MediaCodec audioEncoder, 713 MediaMuxer muxer, 714 InputSurface inputSurface, 715 OutputSurface outputSurface) { 716 ByteBuffer[] videoDecoderInputBuffers = null; 717 ByteBuffer[] videoDecoderOutputBuffers = null; 718 ByteBuffer[] videoEncoderOutputBuffers = null; 719 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 720 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 721 if (mCopyVideo) { 722 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 723 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 724 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 725 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 726 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 727 } 728 ByteBuffer[] audioDecoderInputBuffers = null; 729 ByteBuffer[] audioDecoderOutputBuffers = null; 730 ByteBuffer[] audioEncoderInputBuffers = null; 731 ByteBuffer[] audioEncoderOutputBuffers = null; 732 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 733 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 734 if (mCopyAudio) { 735 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 736 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 737 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 738 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 739 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 740 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 741 } 742 // We will get these from the decoders when notified of a format change. 743 MediaFormat decoderOutputVideoFormat = null; 744 MediaFormat decoderOutputAudioFormat = null; 745 // We will get these from the encoders when notified of a format change. 746 MediaFormat encoderOutputVideoFormat = null; 747 MediaFormat encoderOutputAudioFormat = null; 748 // We will determine these once we have the output format. 749 int outputVideoTrack = -1; 750 int outputAudioTrack = -1; 751 // Whether things are done on the video side. 752 boolean videoExtractorDone = false; 753 boolean videoDecoderDone = false; 754 boolean videoEncoderDone = false; 755 // Whether things are done on the audio side. 756 boolean audioExtractorDone = false; 757 boolean audioDecoderDone = false; 758 boolean audioEncoderDone = false; 759 // The audio decoder output buffer to process, -1 if none. 760 int pendingAudioDecoderOutputBufferIndex = -1; 761 762 boolean muxing = false; 763 764 int videoExtractedFrameCount = 0; 765 int videoDecodedFrameCount = 0; 766 int videoEncodedFrameCount = 0; 767 768 int audioExtractedFrameCount = 0; 769 int audioDecodedFrameCount = 0; 770 int audioEncodedFrameCount = 0; 771 772 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 773 if (VERBOSE) { 774 Log.d(TAG, String.format( 775 "loop: " 776 777 + "V(%b){" 778 + "extracted:%d(done:%b) " 779 + "decoded:%d(done:%b) " 780 + "encoded:%d(done:%b)} " 781 782 + "A(%b){" 783 + "extracted:%d(done:%b) " 784 + "decoded:%d(done:%b) " 785 + "encoded:%d(done:%b) " 786 + "pending:%d} " 787 788 + "muxing:%b(V:%d,A:%d)", 789 790 mCopyVideo, 791 videoExtractedFrameCount, videoExtractorDone, 792 videoDecodedFrameCount, videoDecoderDone, 793 videoEncodedFrameCount, videoEncoderDone, 794 795 mCopyAudio, 796 audioExtractedFrameCount, audioExtractorDone, 797 audioDecodedFrameCount, audioDecoderDone, 798 audioEncodedFrameCount, audioEncoderDone, 799 pendingAudioDecoderOutputBufferIndex, 800 801 muxing, outputVideoTrack, outputAudioTrack)); 802 } 803 804 // Extract video from file and feed to decoder. 805 // Do not extract video if we have determined the output format but we are not yet 806 // ready to mux the frames. 807 while (mCopyVideo && !videoExtractorDone 808 && (encoderOutputVideoFormat == null || muxing)) { 809 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 810 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 811 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 812 break; 813 } 814 if (VERBOSE) { 815 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 816 } 817 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 818 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 819 long presentationTime = videoExtractor.getSampleTime(); 820 int flags = videoExtractor.getSampleFlags(); 821 if (VERBOSE) { 822 Log.d(TAG, "video extractor: returned buffer of size " + size); 823 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 824 } 825 videoExtractorDone = !videoExtractor.advance(); 826 if (videoExtractorDone) { 827 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 828 flags = flags | MediaCodec.BUFFER_FLAG_END_OF_STREAM; 829 } 830 if (size >= 0) { 831 videoDecoder.queueInputBuffer( 832 decoderInputBufferIndex, 833 0, 834 size, 835 presentationTime, 836 flags); 837 videoExtractedFrameCount++; 838 } 839 // We extracted a frame, let's try something else next. 840 break; 841 } 842 843 // Extract audio from file and feed to decoder. 844 // Do not extract audio if we have determined the output format but we are not yet 845 // ready to mux the frames. 846 while (mCopyAudio && !audioExtractorDone 847 && (encoderOutputAudioFormat == null || muxing)) { 848 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 849 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 850 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 851 break; 852 } 853 if (VERBOSE) { 854 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 855 } 856 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 857 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 858 long presentationTime = audioExtractor.getSampleTime(); 859 if (VERBOSE) { 860 Log.d(TAG, "audio extractor: returned buffer of size " + size); 861 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 862 } 863 if (size >= 0) { 864 audioDecoder.queueInputBuffer( 865 decoderInputBufferIndex, 866 0, 867 size, 868 presentationTime, 869 audioExtractor.getSampleFlags()); 870 } 871 audioExtractorDone = !audioExtractor.advance(); 872 if (audioExtractorDone) { 873 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 874 audioDecoder.queueInputBuffer( 875 decoderInputBufferIndex, 876 0, 877 0, 878 0, 879 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 880 } 881 audioExtractedFrameCount++; 882 // We extracted a frame, let's try something else next. 883 break; 884 } 885 886 // Poll output frames from the video decoder and feed the encoder. 887 while (mCopyVideo && !videoDecoderDone 888 && (encoderOutputVideoFormat == null || muxing)) { 889 int decoderOutputBufferIndex = 890 videoDecoder.dequeueOutputBuffer( 891 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 892 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 893 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 894 break; 895 } 896 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 897 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 898 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 899 break; 900 } 901 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 902 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 903 if (VERBOSE) { 904 Log.d(TAG, "video decoder: output format changed: " 905 + decoderOutputVideoFormat); 906 } 907 break; 908 } 909 if (VERBOSE) { 910 Log.d(TAG, "video decoder: returned output buffer: " 911 + decoderOutputBufferIndex); 912 Log.d(TAG, "video decoder: returned buffer of size " 913 + videoDecoderOutputBufferInfo.size); 914 } 915 ByteBuffer decoderOutputBuffer = 916 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 917 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 918 != 0) { 919 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 920 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 921 break; 922 } 923 if (VERBOSE) { 924 Log.d(TAG, "video decoder: returned buffer for time " 925 + videoDecoderOutputBufferInfo.presentationTimeUs); 926 } 927 boolean render = videoDecoderOutputBufferInfo.size != 0; 928 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 929 if (render) { 930 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 931 outputSurface.awaitNewImage(); 932 // Edit the frame and send it to the encoder. 933 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 934 outputSurface.drawImage(); 935 inputSurface.setPresentationTime( 936 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 937 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 938 inputSurface.swapBuffers(); 939 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 940 videoDecodedFrameCount++; 941 } 942 if ((videoDecoderOutputBufferInfo.flags 943 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 944 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 945 videoDecoderDone = true; 946 videoEncoder.signalEndOfInputStream(); 947 } 948 // We extracted a pending frame, let's try something else next. 949 break; 950 } 951 952 // Poll output frames from the audio decoder. 953 // Do not poll if we already have a pending buffer to feed to the encoder. 954 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 955 && (encoderOutputAudioFormat == null || muxing)) { 956 int decoderOutputBufferIndex = 957 audioDecoder.dequeueOutputBuffer( 958 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 959 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 960 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 961 break; 962 } 963 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 964 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 965 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 966 break; 967 } 968 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 969 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 970 if (VERBOSE) { 971 Log.d(TAG, "audio decoder: output format changed: " 972 + decoderOutputAudioFormat); 973 } 974 break; 975 } 976 if (VERBOSE) { 977 Log.d(TAG, "audio decoder: returned output buffer: " 978 + decoderOutputBufferIndex); 979 } 980 if (VERBOSE) { 981 Log.d(TAG, "audio decoder: returned buffer of size " 982 + audioDecoderOutputBufferInfo.size); 983 } 984 ByteBuffer decoderOutputBuffer = 985 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 986 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 987 != 0) { 988 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 989 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 990 break; 991 } 992 if (VERBOSE) { 993 Log.d(TAG, "audio decoder: returned buffer for time " 994 + audioDecoderOutputBufferInfo.presentationTimeUs); 995 } 996 if (VERBOSE) { 997 Log.d(TAG, "audio decoder: output buffer is now pending: " 998 + pendingAudioDecoderOutputBufferIndex); 999 } 1000 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 1001 audioDecodedFrameCount++; 1002 // We extracted a pending frame, let's try something else next. 1003 break; 1004 } 1005 1006 // Feed the pending decoded audio buffer to the audio encoder. 1007 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1008 if (VERBOSE) { 1009 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1010 + pendingAudioDecoderOutputBufferIndex); 1011 } 1012 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1013 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1014 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1015 break; 1016 } 1017 if (VERBOSE) { 1018 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1019 } 1020 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1021 int size = audioDecoderOutputBufferInfo.size; 1022 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1023 if (VERBOSE) { 1024 Log.d(TAG, "audio decoder: processing pending buffer: " 1025 + pendingAudioDecoderOutputBufferIndex); 1026 } 1027 if (VERBOSE) { 1028 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1029 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1030 } 1031 if (size >= 0) { 1032 ByteBuffer decoderOutputBuffer = 1033 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1034 .duplicate(); 1035 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1036 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1037 encoderInputBuffer.position(0); 1038 encoderInputBuffer.put(decoderOutputBuffer); 1039 1040 audioEncoder.queueInputBuffer( 1041 encoderInputBufferIndex, 1042 0, 1043 size, 1044 presentationTime, 1045 audioDecoderOutputBufferInfo.flags); 1046 } 1047 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1048 pendingAudioDecoderOutputBufferIndex = -1; 1049 if ((audioDecoderOutputBufferInfo.flags 1050 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1051 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1052 audioDecoderDone = true; 1053 } 1054 // We enqueued a pending frame, let's try something else next. 1055 break; 1056 } 1057 1058 // Poll frames from the video encoder and send them to the muxer. 1059 while (mCopyVideo && !videoEncoderDone 1060 && (encoderOutputVideoFormat == null || muxing)) { 1061 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1062 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1063 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1064 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1065 break; 1066 } 1067 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1068 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1069 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1070 break; 1071 } 1072 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1073 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1074 if (outputVideoTrack >= 0) { 1075 fail("video encoder changed its output format again?"); 1076 } 1077 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1078 break; 1079 } 1080 assertTrue("should have added track before processing output", muxing); 1081 if (VERBOSE) { 1082 Log.d(TAG, "video encoder: returned output buffer: " 1083 + encoderOutputBufferIndex); 1084 Log.d(TAG, "video encoder: returned buffer of size " 1085 + videoEncoderOutputBufferInfo.size); 1086 } 1087 ByteBuffer encoderOutputBuffer = 1088 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1089 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1090 != 0) { 1091 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1092 // Simply ignore codec config buffers. 1093 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1094 break; 1095 } 1096 if (VERBOSE) { 1097 Log.d(TAG, "video encoder: returned buffer for time " 1098 + videoEncoderOutputBufferInfo.presentationTimeUs); 1099 } 1100 if (videoEncoderOutputBufferInfo.size != 0) { 1101 muxer.writeSampleData( 1102 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1103 videoEncodedFrameCount++; 1104 } 1105 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1106 != 0) { 1107 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1108 videoEncoderDone = true; 1109 } 1110 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1111 // We enqueued an encoded frame, let's try something else next. 1112 break; 1113 } 1114 1115 // Poll frames from the audio encoder and send them to the muxer. 1116 while (mCopyAudio && !audioEncoderDone 1117 && (encoderOutputAudioFormat == null || muxing)) { 1118 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1119 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1120 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1121 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1122 break; 1123 } 1124 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1125 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1126 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1127 break; 1128 } 1129 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1130 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1131 if (outputAudioTrack >= 0) { 1132 fail("audio encoder changed its output format again?"); 1133 } 1134 1135 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1136 break; 1137 } 1138 assertTrue("should have added track before processing output", muxing); 1139 if (VERBOSE) { 1140 Log.d(TAG, "audio encoder: returned output buffer: " 1141 + encoderOutputBufferIndex); 1142 Log.d(TAG, "audio encoder: returned buffer of size " 1143 + audioEncoderOutputBufferInfo.size); 1144 } 1145 ByteBuffer encoderOutputBuffer = 1146 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1147 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1148 != 0) { 1149 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1150 // Simply ignore codec config buffers. 1151 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1152 break; 1153 } 1154 if (VERBOSE) { 1155 Log.d(TAG, "audio encoder: returned buffer for time " 1156 + audioEncoderOutputBufferInfo.presentationTimeUs); 1157 } 1158 if (audioEncoderOutputBufferInfo.size != 0) { 1159 muxer.writeSampleData( 1160 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1161 } 1162 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1163 != 0) { 1164 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1165 audioEncoderDone = true; 1166 } 1167 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1168 audioEncodedFrameCount++; 1169 // We enqueued an encoded frame, let's try something else next. 1170 break; 1171 } 1172 1173 if (!muxing 1174 && (!mCopyAudio || encoderOutputAudioFormat != null) 1175 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1176 if (mCopyVideo) { 1177 Log.d(TAG, "muxer: adding video track."); 1178 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1179 } 1180 if (mCopyAudio) { 1181 Log.d(TAG, "muxer: adding audio track."); 1182 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1183 } 1184 Log.d(TAG, "muxer: starting"); 1185 muxer.start(); 1186 muxing = true; 1187 } 1188 } 1189 1190 // Basic validation checks. 1191 if (mCopyVideo) { 1192 assertEquals("encoded and decoded video frame counts should match", 1193 videoDecodedFrameCount, videoEncodedFrameCount); 1194 assertTrue("decoded frame count should be less than extracted frame count", 1195 videoDecodedFrameCount <= videoExtractedFrameCount); 1196 } 1197 if (mCopyAudio) { 1198 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1199 } 1200 } 1201 isVideoFormat(MediaFormat format)1202 private static boolean isVideoFormat(MediaFormat format) { 1203 return getMimeTypeFor(format).startsWith("video/"); 1204 } 1205 isAudioFormat(MediaFormat format)1206 private static boolean isAudioFormat(MediaFormat format) { 1207 return getMimeTypeFor(format).startsWith("audio/"); 1208 } 1209 getMimeTypeFor(MediaFormat format)1210 private static String getMimeTypeFor(MediaFormat format) { 1211 return format.getString(MediaFormat.KEY_MIME); 1212 } 1213 1214 /** 1215 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1216 * found. 1217 */ selectCodec(String mimeType)1218 private static MediaCodecInfo selectCodec(String mimeType) { 1219 int numCodecs = MediaCodecList.getCodecCount(); 1220 for (int i = 0; i < numCodecs; i++) { 1221 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1222 1223 if (codecInfo.isAlias()) { 1224 continue; 1225 } 1226 if (!codecInfo.isEncoder()) { 1227 continue; 1228 } 1229 1230 String[] types = codecInfo.getSupportedTypes(); 1231 for (int j = 0; j < types.length; j++) { 1232 if (types[j].equalsIgnoreCase(mimeType)) { 1233 return codecInfo; 1234 } 1235 } 1236 } 1237 return null; 1238 } 1239 } 1240