1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.cts; 18 19 import android.annotation.TargetApi; 20 import android.content.res.AssetFileDescriptor; 21 import android.content.Context; 22 import android.media.MediaCodec; 23 import android.media.MediaCodecInfo; 24 import android.media.MediaCodecInfo.CodecCapabilities; 25 import android.media.MediaCodecInfo.CodecProfileLevel; 26 import android.media.MediaCodecList; 27 import android.media.MediaExtractor; 28 import android.media.MediaFormat; 29 import android.media.MediaMuxer; 30 import android.media.MediaPlayer; 31 import android.os.Environment; 32 import android.test.ActivityInstrumentationTestCase2; 33 import android.util.Log; 34 import android.view.Surface; 35 36 import android.media.MediaCodecInfo; 37 import android.media.MediaCodecInfo.CodecCapabilities; 38 import android.media.MediaCodecInfo.CodecProfileLevel; 39 40 import android.media.cts.R; 41 42 import java.io.File; 43 import java.io.IOException; 44 import java.nio.ByteBuffer; 45 import java.util.concurrent.atomic.AtomicReference; 46 import java.util.concurrent.CountDownLatch; 47 48 /** 49 * Test for the integration of MediaMuxer and MediaCodec's encoder. 50 * 51 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 52 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 53 * them into a file. 54 * 55 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 56 * fails along the way. 57 * 58 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 59 * MediaMuxer. 60 */ 61 @TargetApi(18) 62 public class ExtractDecodeEditEncodeMuxTest 63 extends ActivityInstrumentationTestCase2<MediaStubActivity> { 64 65 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 66 private static final boolean VERBOSE = false; // lots of logging 67 68 /** How long to wait for the next buffer to become available. */ 69 private static final int TIMEOUT_USEC = 10000; 70 71 /** Where to output the test files. */ 72 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 73 74 // parameters for the video encoder 75 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 76 private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps 77 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 78 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 79 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 80 81 // parameters for the audio encoder 82 // Advanced Audio Coding 83 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 84 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 85 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 86 private static final int OUTPUT_AUDIO_AAC_PROFILE = 87 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 88 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 89 90 /** 91 * Used for editing the frames. 92 * 93 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 94 */ 95 private static final String FRAGMENT_SHADER = 96 "#extension GL_OES_EGL_image_external : require\n" + 97 "precision mediump float;\n" + 98 "varying vec2 vTextureCoord;\n" + 99 "uniform samplerExternalOES sTexture;\n" + 100 "void main() {\n" + 101 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 102 "}\n"; 103 104 /** Whether to copy the video from the test video. */ 105 private boolean mCopyVideo; 106 /** Whether to copy the audio from the test video. */ 107 private boolean mCopyAudio; 108 /** Whether to verify the audio format. */ 109 private boolean mVerifyAudioFormat; 110 /** Width of the output frames. */ 111 private int mWidth = -1; 112 /** Height of the output frames. */ 113 private int mHeight = -1; 114 115 /** The raw resource used as the input file. */ 116 private int mSourceResId; 117 118 /** The destination file for the encoded output. */ 119 private String mOutputFile; 120 121 private String mOutputVideoMimeType; 122 ExtractDecodeEditEncodeMuxTest()123 public ExtractDecodeEditEncodeMuxTest() { 124 super(MediaStubActivity.class); 125 } 126 testExtractDecodeEditEncodeMuxQCIF()127 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 128 if(!setSize(176, 144)) return; 129 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 130 setCopyVideo(); 131 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 132 TestWrapper.runTest(this); 133 } 134 testExtractDecodeEditEncodeMuxQVGA()135 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 136 if(!setSize(320, 240)) return; 137 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 138 setCopyVideo(); 139 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 140 TestWrapper.runTest(this); 141 } 142 testExtractDecodeEditEncodeMux720p()143 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 144 if(!setSize(1280, 720)) return; 145 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 146 setCopyVideo(); 147 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 148 TestWrapper.runTest(this); 149 } 150 testExtractDecodeEditEncodeMux2160pHevc()151 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 152 if(!setSize(3840, 2160)) return; 153 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 154 setCopyVideo(); 155 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 156 TestWrapper.runTest(this); 157 } 158 testExtractDecodeEditEncodeMuxAudio()159 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 160 if(!setSize(1280, 720)) return; 161 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 162 setCopyAudio(); 163 setVerifyAudioFormat(); 164 TestWrapper.runTest(this); 165 } 166 testExtractDecodeEditEncodeMuxAudioVideo()167 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 168 if(!setSize(1280, 720)) return; 169 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 170 setCopyAudio(); 171 setCopyVideo(); 172 setVerifyAudioFormat(); 173 TestWrapper.runTest(this); 174 } 175 176 /** Wraps testExtractDecodeEditEncodeMux() */ 177 private static class TestWrapper implements Runnable { 178 private Throwable mThrowable; 179 private ExtractDecodeEditEncodeMuxTest mTest; 180 TestWrapper(ExtractDecodeEditEncodeMuxTest test)181 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 182 mTest = test; 183 } 184 185 @Override run()186 public void run() { 187 try { 188 mTest.extractDecodeEditEncodeMux(); 189 } catch (Throwable th) { 190 mThrowable = th; 191 } 192 } 193 194 /** 195 * Entry point. 196 */ runTest(ExtractDecodeEditEncodeMuxTest test)197 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 198 test.setOutputFile(); 199 TestWrapper wrapper = new TestWrapper(test); 200 Thread th = new Thread(wrapper, "codec test"); 201 th.start(); 202 th.join(); 203 if (wrapper.mThrowable != null) { 204 throw wrapper.mThrowable; 205 } 206 } 207 } 208 209 /** 210 * Sets the test to copy the video stream. 211 */ setCopyVideo()212 private void setCopyVideo() { 213 mCopyVideo = true; 214 } 215 216 /** 217 * Sets the test to copy the video stream. 218 */ setCopyAudio()219 private void setCopyAudio() { 220 mCopyAudio = true; 221 } 222 223 /** 224 * Sets the test to verify the output audio format. 225 */ setVerifyAudioFormat()226 private void setVerifyAudioFormat() { 227 mVerifyAudioFormat = true; 228 } 229 230 /** 231 * Sets the desired frame size and returns whether the given resolution is 232 * supported. 233 * 234 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 235 * is supported. For other codecs, always return {@code true}. 236 */ setSize(int width, int height)237 private boolean setSize(int width, int height) { 238 if ((width % 16) != 0 || (height % 16) != 0) { 239 Log.w(TAG, "WARNING: width or height not multiple of 16"); 240 } 241 mWidth = width; 242 mHeight = height; 243 244 // TODO: remove this logic in setSize as it is now handled when configuring codecs 245 return true; 246 } 247 248 /** 249 * Sets the raw resource used as the source video. 250 */ setSource(int resId)251 private void setSource(int resId) { 252 mSourceResId = resId; 253 } 254 255 /** 256 * Sets the name of the output file based on the other parameters. 257 * 258 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(int)}. 259 */ setOutputFile()260 private void setOutputFile() { 261 StringBuilder sb = new StringBuilder(); 262 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 263 sb.append("/cts-media-"); 264 sb.append(getClass().getSimpleName()); 265 assertTrue("should have called setSource() first", mSourceResId != -1); 266 sb.append('-'); 267 sb.append(mSourceResId); 268 if (mCopyVideo) { 269 assertTrue("should have called setSize() first", mWidth != -1); 270 assertTrue("should have called setSize() first", mHeight != -1); 271 sb.append('-'); 272 sb.append("video"); 273 sb.append('-'); 274 sb.append(mWidth); 275 sb.append('x'); 276 sb.append(mHeight); 277 } 278 if (mCopyAudio) { 279 sb.append('-'); 280 sb.append("audio"); 281 } 282 sb.append(".mp4"); 283 mOutputFile = sb.toString(); 284 } 285 setVideoMimeType(String mimeType)286 private void setVideoMimeType(String mimeType) { 287 mOutputVideoMimeType = mimeType; 288 } 289 290 /** 291 * Tests encoding and subsequently decoding video from frames generated into a buffer. 292 * <p> 293 * We encode several frames of a video test pattern using MediaCodec, then decode the output 294 * with MediaCodec and do some simple checks. 295 */ extractDecodeEditEncodeMux()296 private void extractDecodeEditEncodeMux() throws Exception { 297 // Exception that may be thrown during release. 298 Exception exception = null; 299 300 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 301 302 // We avoid the device-specific limitations on width and height by using values 303 // that are multiples of 16, which all tested devices seem to be able to handle. 304 MediaFormat outputVideoFormat = 305 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 306 307 // Set some properties. Failing to specify some of these can cause the MediaCodec 308 // configure() call to throw an unhelpful exception. 309 outputVideoFormat.setInteger( 310 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 311 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 312 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 313 outputVideoFormat.setInteger( 314 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 315 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 316 317 String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 318 if (videoEncoderName == null) { 319 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 320 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 321 return; 322 } 323 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 324 325 MediaFormat outputAudioFormat = 326 MediaFormat.createAudioFormat( 327 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 328 OUTPUT_AUDIO_CHANNEL_COUNT); 329 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 330 // TODO: Bug workaround --- uncomment once fixed. 331 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 332 333 String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 334 if (audioEncoderName == null) { 335 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 336 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 337 return; 338 } 339 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 340 341 MediaExtractor videoExtractor = null; 342 MediaExtractor audioExtractor = null; 343 OutputSurface outputSurface = null; 344 MediaCodec videoDecoder = null; 345 MediaCodec audioDecoder = null; 346 MediaCodec videoEncoder = null; 347 MediaCodec audioEncoder = null; 348 MediaMuxer muxer = null; 349 350 InputSurface inputSurface = null; 351 352 try { 353 if (mCopyVideo) { 354 videoExtractor = createExtractor(); 355 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 356 assertTrue("missing video track in test video", videoInputTrack != -1); 357 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 358 359 // Create a MediaCodec for the desired codec, then configure it as an encoder with 360 // our desired properties. Request a Surface to use for input. 361 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 362 videoEncoder = createVideoEncoder( 363 videoEncoderName, outputVideoFormat, inputSurfaceReference); 364 inputSurface = new InputSurface(inputSurfaceReference.get()); 365 inputSurface.makeCurrent(); 366 // Create a MediaCodec for the decoder, based on the extractor's format. 367 outputSurface = new OutputSurface(); 368 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 369 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 370 } 371 372 if (mCopyAudio) { 373 audioExtractor = createExtractor(); 374 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 375 assertTrue("missing audio track in test video", audioInputTrack != -1); 376 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 377 378 // Create a MediaCodec for the desired codec, then configure it as an encoder with 379 // our desired properties. Request a Surface to use for input. 380 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 381 // Create a MediaCodec for the decoder, based on the extractor's format. 382 audioDecoder = createAudioDecoder(mcl, inputFormat); 383 } 384 385 // Creates a muxer but do not start or add tracks just yet. 386 muxer = createMuxer(); 387 388 doExtractDecodeEditEncodeMux( 389 videoExtractor, 390 audioExtractor, 391 videoDecoder, 392 videoEncoder, 393 audioDecoder, 394 audioEncoder, 395 muxer, 396 inputSurface, 397 outputSurface); 398 } finally { 399 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 400 // Try to release everything we acquired, even if one of the releases fails, in which 401 // case we save the first exception we got and re-throw at the end (unless something 402 // other exception has already been thrown). This guarantees the first exception thrown 403 // is reported as the cause of the error, everything is (attempted) to be released, and 404 // all other exceptions appear in the logs. 405 try { 406 if (videoExtractor != null) { 407 videoExtractor.release(); 408 } 409 } catch(Exception e) { 410 Log.e(TAG, "error while releasing videoExtractor", e); 411 if (exception == null) { 412 exception = e; 413 } 414 } 415 try { 416 if (audioExtractor != null) { 417 audioExtractor.release(); 418 } 419 } catch(Exception e) { 420 Log.e(TAG, "error while releasing audioExtractor", e); 421 if (exception == null) { 422 exception = e; 423 } 424 } 425 try { 426 if (videoDecoder != null) { 427 videoDecoder.stop(); 428 videoDecoder.release(); 429 } 430 } catch(Exception e) { 431 Log.e(TAG, "error while releasing videoDecoder", e); 432 if (exception == null) { 433 exception = e; 434 } 435 } 436 try { 437 if (outputSurface != null) { 438 outputSurface.release(); 439 } 440 } catch(Exception e) { 441 Log.e(TAG, "error while releasing outputSurface", e); 442 if (exception == null) { 443 exception = e; 444 } 445 } 446 try { 447 if (videoEncoder != null) { 448 videoEncoder.stop(); 449 videoEncoder.release(); 450 } 451 } catch(Exception e) { 452 Log.e(TAG, "error while releasing videoEncoder", e); 453 if (exception == null) { 454 exception = e; 455 } 456 } 457 try { 458 if (audioDecoder != null) { 459 audioDecoder.stop(); 460 audioDecoder.release(); 461 } 462 } catch(Exception e) { 463 Log.e(TAG, "error while releasing audioDecoder", e); 464 if (exception == null) { 465 exception = e; 466 } 467 } 468 try { 469 if (audioEncoder != null) { 470 audioEncoder.stop(); 471 audioEncoder.release(); 472 } 473 } catch(Exception e) { 474 Log.e(TAG, "error while releasing audioEncoder", e); 475 if (exception == null) { 476 exception = e; 477 } 478 } 479 try { 480 if (muxer != null) { 481 muxer.stop(); 482 muxer.release(); 483 } 484 } catch(Exception e) { 485 Log.e(TAG, "error while releasing muxer", e); 486 if (exception == null) { 487 exception = e; 488 } 489 } 490 try { 491 if (inputSurface != null) { 492 inputSurface.release(); 493 } 494 } catch(Exception e) { 495 Log.e(TAG, "error while releasing inputSurface", e); 496 if (exception == null) { 497 exception = e; 498 } 499 } 500 } 501 if (exception != null) { 502 throw exception; 503 } 504 505 MediaExtractor mediaExtractor = null; 506 try { 507 mediaExtractor = new MediaExtractor(); 508 mediaExtractor.setDataSource(mOutputFile); 509 510 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 511 mediaExtractor.getTrackCount()); 512 if (mVerifyAudioFormat) { 513 boolean foundAudio = false; 514 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 515 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 516 if (isAudioFormat(trackFormat)) { 517 foundAudio = true; 518 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 519 520 // SBR mode halves the sample rate in the format. 521 if (OUTPUT_AUDIO_AAC_PROFILE == 522 MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 523 expectedSampleRate /= 2; 524 } 525 assertEquals("sample rates should match", expectedSampleRate, 526 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 527 } 528 } 529 530 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 531 } 532 } catch (IOException e) { 533 throw new IllegalStateException("exception verifying output file", e); 534 } finally { 535 if (mediaExtractor != null) { 536 mediaExtractor.release(); 537 } 538 } 539 540 // TODO: Check the generated output file's video format and sample data. 541 542 MediaStubActivity activity = getActivity(); 543 final MediaPlayer mp = new MediaPlayer(); 544 final Exception[] exceptionHolder = { null }; 545 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 546 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 547 @Override 548 public boolean onError(MediaPlayer origin, int what, int extra) { 549 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 550 + " extra=" + extra); 551 // Returning false would trigger onCompletion() so that 552 // playbackEndSignal.await() can stop waiting. 553 return false; 554 } 555 }); 556 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 557 @Override 558 public void onCompletion(MediaPlayer origin) { 559 playbackEndSignal.countDown(); 560 } 561 }); 562 try { 563 mp.setDataSource(mOutputFile); 564 mp.setDisplay(activity.getSurfaceHolder()); 565 mp.prepare(); 566 mp.start(); 567 playbackEndSignal.await(); 568 } catch (Exception e) { 569 exceptionHolder[0] = e; 570 } finally { 571 mp.release(); 572 } 573 574 if (exceptionHolder[0] != null) { 575 throw exceptionHolder[0]; 576 } 577 } 578 579 /** 580 * Creates an extractor that reads its frames from {@link #mSourceResId}. 581 */ createExtractor()582 private MediaExtractor createExtractor() throws IOException { 583 MediaExtractor extractor; 584 Context context = getInstrumentation().getTargetContext(); 585 AssetFileDescriptor srcFd = context.getResources().openRawResourceFd(mSourceResId); 586 extractor = new MediaExtractor(); 587 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 588 srcFd.getLength()); 589 return extractor; 590 } 591 592 /** 593 * Creates a decoder for the given format, which outputs to the given surface. 594 * 595 * @param inputFormat the format of the stream to decode 596 * @param surface into which to decode the frames 597 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)598 private MediaCodec createVideoDecoder( 599 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 600 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 601 decoder.configure(inputFormat, surface, null, 0); 602 decoder.start(); 603 return decoder; 604 } 605 606 /** 607 * Creates an encoder for the given format using the specified codec, taking input from a 608 * surface. 609 * 610 * <p>The surface to use as input is stored in the given reference. 611 * 612 * @param codecInfo of the codec to use 613 * @param format of the stream to be produced 614 * @param surfaceReference to store the surface to use as input 615 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)616 private MediaCodec createVideoEncoder( 617 String codecName, 618 MediaFormat format, 619 AtomicReference<Surface> surfaceReference) 620 throws IOException { 621 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 622 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 623 // Must be called before start() is. 624 surfaceReference.set(encoder.createInputSurface()); 625 encoder.start(); 626 return encoder; 627 } 628 629 /** 630 * Creates a decoder for the given format. 631 * 632 * @param inputFormat the format of the stream to decode 633 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)634 private MediaCodec createAudioDecoder( 635 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 636 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 637 decoder.configure(inputFormat, null, null, 0); 638 decoder.start(); 639 return decoder; 640 } 641 642 /** 643 * Creates an encoder for the given format using the specified codec. 644 * 645 * @param codecInfo of the codec to use 646 * @param format of the stream to be produced 647 */ createAudioEncoder(String codecName, MediaFormat format)648 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 649 throws IOException { 650 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 651 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 652 encoder.start(); 653 return encoder; 654 } 655 656 /** 657 * Creates a muxer to write the encoded frames. 658 * 659 * <p>The muxer is not started as it needs to be started only after all streams have been added. 660 */ createMuxer()661 private MediaMuxer createMuxer() throws IOException { 662 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 663 } 664 getAndSelectVideoTrackIndex(MediaExtractor extractor)665 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 666 for (int index = 0; index < extractor.getTrackCount(); ++index) { 667 if (VERBOSE) { 668 Log.d(TAG, "format for track " + index + " is " 669 + getMimeTypeFor(extractor.getTrackFormat(index))); 670 } 671 if (isVideoFormat(extractor.getTrackFormat(index))) { 672 extractor.selectTrack(index); 673 return index; 674 } 675 } 676 return -1; 677 } 678 getAndSelectAudioTrackIndex(MediaExtractor extractor)679 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 680 for (int index = 0; index < extractor.getTrackCount(); ++index) { 681 if (VERBOSE) { 682 Log.d(TAG, "format for track " + index + " is " 683 + getMimeTypeFor(extractor.getTrackFormat(index))); 684 } 685 if (isAudioFormat(extractor.getTrackFormat(index))) { 686 extractor.selectTrack(index); 687 return index; 688 } 689 } 690 return -1; 691 } 692 693 /** 694 * Does the actual work for extracting, decoding, encoding and muxing. 695 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)696 private void doExtractDecodeEditEncodeMux( 697 MediaExtractor videoExtractor, 698 MediaExtractor audioExtractor, 699 MediaCodec videoDecoder, 700 MediaCodec videoEncoder, 701 MediaCodec audioDecoder, 702 MediaCodec audioEncoder, 703 MediaMuxer muxer, 704 InputSurface inputSurface, 705 OutputSurface outputSurface) { 706 ByteBuffer[] videoDecoderInputBuffers = null; 707 ByteBuffer[] videoDecoderOutputBuffers = null; 708 ByteBuffer[] videoEncoderOutputBuffers = null; 709 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 710 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 711 if (mCopyVideo) { 712 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 713 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 714 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 715 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 716 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 717 } 718 ByteBuffer[] audioDecoderInputBuffers = null; 719 ByteBuffer[] audioDecoderOutputBuffers = null; 720 ByteBuffer[] audioEncoderInputBuffers = null; 721 ByteBuffer[] audioEncoderOutputBuffers = null; 722 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 723 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 724 if (mCopyAudio) { 725 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 726 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 727 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 728 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 729 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 730 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 731 } 732 // We will get these from the decoders when notified of a format change. 733 MediaFormat decoderOutputVideoFormat = null; 734 MediaFormat decoderOutputAudioFormat = null; 735 // We will get these from the encoders when notified of a format change. 736 MediaFormat encoderOutputVideoFormat = null; 737 MediaFormat encoderOutputAudioFormat = null; 738 // We will determine these once we have the output format. 739 int outputVideoTrack = -1; 740 int outputAudioTrack = -1; 741 // Whether things are done on the video side. 742 boolean videoExtractorDone = false; 743 boolean videoDecoderDone = false; 744 boolean videoEncoderDone = false; 745 // Whether things are done on the audio side. 746 boolean audioExtractorDone = false; 747 boolean audioDecoderDone = false; 748 boolean audioEncoderDone = false; 749 // The audio decoder output buffer to process, -1 if none. 750 int pendingAudioDecoderOutputBufferIndex = -1; 751 752 boolean muxing = false; 753 754 int videoExtractedFrameCount = 0; 755 int videoDecodedFrameCount = 0; 756 int videoEncodedFrameCount = 0; 757 758 int audioExtractedFrameCount = 0; 759 int audioDecodedFrameCount = 0; 760 int audioEncodedFrameCount = 0; 761 762 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 763 if (VERBOSE) { 764 Log.d(TAG, String.format( 765 "loop: " 766 767 + "V(%b){" 768 + "extracted:%d(done:%b) " 769 + "decoded:%d(done:%b) " 770 + "encoded:%d(done:%b)} " 771 772 + "A(%b){" 773 + "extracted:%d(done:%b) " 774 + "decoded:%d(done:%b) " 775 + "encoded:%d(done:%b) " 776 + "pending:%d} " 777 778 + "muxing:%b(V:%d,A:%d)", 779 780 mCopyVideo, 781 videoExtractedFrameCount, videoExtractorDone, 782 videoDecodedFrameCount, videoDecoderDone, 783 videoEncodedFrameCount, videoEncoderDone, 784 785 mCopyAudio, 786 audioExtractedFrameCount, audioExtractorDone, 787 audioDecodedFrameCount, audioDecoderDone, 788 audioEncodedFrameCount, audioEncoderDone, 789 pendingAudioDecoderOutputBufferIndex, 790 791 muxing, outputVideoTrack, outputAudioTrack)); 792 } 793 794 // Extract video from file and feed to decoder. 795 // Do not extract video if we have determined the output format but we are not yet 796 // ready to mux the frames. 797 while (mCopyVideo && !videoExtractorDone 798 && (encoderOutputVideoFormat == null || muxing)) { 799 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 800 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 801 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 802 break; 803 } 804 if (VERBOSE) { 805 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 806 } 807 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 808 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 809 long presentationTime = videoExtractor.getSampleTime(); 810 if (VERBOSE) { 811 Log.d(TAG, "video extractor: returned buffer of size " + size); 812 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 813 } 814 if (size >= 0) { 815 videoDecoder.queueInputBuffer( 816 decoderInputBufferIndex, 817 0, 818 size, 819 presentationTime, 820 videoExtractor.getSampleFlags()); 821 } 822 videoExtractorDone = !videoExtractor.advance(); 823 if (videoExtractorDone) { 824 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 825 videoDecoder.queueInputBuffer( 826 decoderInputBufferIndex, 827 0, 828 0, 829 0, 830 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 831 } 832 videoExtractedFrameCount++; 833 // We extracted a frame, let's try something else next. 834 break; 835 } 836 837 // Extract audio from file and feed to decoder. 838 // Do not extract audio if we have determined the output format but we are not yet 839 // ready to mux the frames. 840 while (mCopyAudio && !audioExtractorDone 841 && (encoderOutputAudioFormat == null || muxing)) { 842 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 843 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 844 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 845 break; 846 } 847 if (VERBOSE) { 848 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 849 } 850 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 851 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 852 long presentationTime = audioExtractor.getSampleTime(); 853 if (VERBOSE) { 854 Log.d(TAG, "audio extractor: returned buffer of size " + size); 855 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 856 } 857 if (size >= 0) { 858 audioDecoder.queueInputBuffer( 859 decoderInputBufferIndex, 860 0, 861 size, 862 presentationTime, 863 audioExtractor.getSampleFlags()); 864 } 865 audioExtractorDone = !audioExtractor.advance(); 866 if (audioExtractorDone) { 867 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 868 audioDecoder.queueInputBuffer( 869 decoderInputBufferIndex, 870 0, 871 0, 872 0, 873 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 874 } 875 audioExtractedFrameCount++; 876 // We extracted a frame, let's try something else next. 877 break; 878 } 879 880 // Poll output frames from the video decoder and feed the encoder. 881 while (mCopyVideo && !videoDecoderDone 882 && (encoderOutputVideoFormat == null || muxing)) { 883 int decoderOutputBufferIndex = 884 videoDecoder.dequeueOutputBuffer( 885 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 886 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 887 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 888 break; 889 } 890 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 891 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 892 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 893 break; 894 } 895 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 896 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 897 if (VERBOSE) { 898 Log.d(TAG, "video decoder: output format changed: " 899 + decoderOutputVideoFormat); 900 } 901 break; 902 } 903 if (VERBOSE) { 904 Log.d(TAG, "video decoder: returned output buffer: " 905 + decoderOutputBufferIndex); 906 Log.d(TAG, "video decoder: returned buffer of size " 907 + videoDecoderOutputBufferInfo.size); 908 } 909 ByteBuffer decoderOutputBuffer = 910 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 911 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 912 != 0) { 913 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 914 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 915 break; 916 } 917 if (VERBOSE) { 918 Log.d(TAG, "video decoder: returned buffer for time " 919 + videoDecoderOutputBufferInfo.presentationTimeUs); 920 } 921 boolean render = videoDecoderOutputBufferInfo.size != 0; 922 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 923 if (render) { 924 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 925 outputSurface.awaitNewImage(); 926 // Edit the frame and send it to the encoder. 927 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 928 outputSurface.drawImage(); 929 inputSurface.setPresentationTime( 930 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 931 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 932 inputSurface.swapBuffers(); 933 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 934 } 935 if ((videoDecoderOutputBufferInfo.flags 936 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 937 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 938 videoDecoderDone = true; 939 videoEncoder.signalEndOfInputStream(); 940 } 941 videoDecodedFrameCount++; 942 // We extracted a pending frame, let's try something else next. 943 break; 944 } 945 946 // Poll output frames from the audio decoder. 947 // Do not poll if we already have a pending buffer to feed to the encoder. 948 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 949 && (encoderOutputAudioFormat == null || muxing)) { 950 int decoderOutputBufferIndex = 951 audioDecoder.dequeueOutputBuffer( 952 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 953 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 954 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 955 break; 956 } 957 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 958 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 959 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 960 break; 961 } 962 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 963 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 964 if (VERBOSE) { 965 Log.d(TAG, "audio decoder: output format changed: " 966 + decoderOutputAudioFormat); 967 } 968 break; 969 } 970 if (VERBOSE) { 971 Log.d(TAG, "audio decoder: returned output buffer: " 972 + decoderOutputBufferIndex); 973 } 974 if (VERBOSE) { 975 Log.d(TAG, "audio decoder: returned buffer of size " 976 + audioDecoderOutputBufferInfo.size); 977 } 978 ByteBuffer decoderOutputBuffer = 979 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 980 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 981 != 0) { 982 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 983 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 984 break; 985 } 986 if (VERBOSE) { 987 Log.d(TAG, "audio decoder: returned buffer for time " 988 + audioDecoderOutputBufferInfo.presentationTimeUs); 989 } 990 if (VERBOSE) { 991 Log.d(TAG, "audio decoder: output buffer is now pending: " 992 + pendingAudioDecoderOutputBufferIndex); 993 } 994 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 995 audioDecodedFrameCount++; 996 // We extracted a pending frame, let's try something else next. 997 break; 998 } 999 1000 // Feed the pending decoded audio buffer to the audio encoder. 1001 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1002 if (VERBOSE) { 1003 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1004 + pendingAudioDecoderOutputBufferIndex); 1005 } 1006 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1007 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1008 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1009 break; 1010 } 1011 if (VERBOSE) { 1012 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1013 } 1014 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1015 int size = audioDecoderOutputBufferInfo.size; 1016 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1017 if (VERBOSE) { 1018 Log.d(TAG, "audio decoder: processing pending buffer: " 1019 + pendingAudioDecoderOutputBufferIndex); 1020 } 1021 if (VERBOSE) { 1022 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1023 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1024 } 1025 if (size >= 0) { 1026 ByteBuffer decoderOutputBuffer = 1027 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1028 .duplicate(); 1029 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1030 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1031 encoderInputBuffer.position(0); 1032 encoderInputBuffer.put(decoderOutputBuffer); 1033 1034 audioEncoder.queueInputBuffer( 1035 encoderInputBufferIndex, 1036 0, 1037 size, 1038 presentationTime, 1039 audioDecoderOutputBufferInfo.flags); 1040 } 1041 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1042 pendingAudioDecoderOutputBufferIndex = -1; 1043 if ((audioDecoderOutputBufferInfo.flags 1044 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1045 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1046 audioDecoderDone = true; 1047 } 1048 // We enqueued a pending frame, let's try something else next. 1049 break; 1050 } 1051 1052 // Poll frames from the video encoder and send them to the muxer. 1053 while (mCopyVideo && !videoEncoderDone 1054 && (encoderOutputVideoFormat == null || muxing)) { 1055 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1056 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1057 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1058 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1059 break; 1060 } 1061 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1062 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1063 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1064 break; 1065 } 1066 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1067 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1068 if (outputVideoTrack >= 0) { 1069 fail("video encoder changed its output format again?"); 1070 } 1071 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1072 break; 1073 } 1074 assertTrue("should have added track before processing output", muxing); 1075 if (VERBOSE) { 1076 Log.d(TAG, "video encoder: returned output buffer: " 1077 + encoderOutputBufferIndex); 1078 Log.d(TAG, "video encoder: returned buffer of size " 1079 + videoEncoderOutputBufferInfo.size); 1080 } 1081 ByteBuffer encoderOutputBuffer = 1082 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1083 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1084 != 0) { 1085 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1086 // Simply ignore codec config buffers. 1087 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1088 break; 1089 } 1090 if (VERBOSE) { 1091 Log.d(TAG, "video encoder: returned buffer for time " 1092 + videoEncoderOutputBufferInfo.presentationTimeUs); 1093 } 1094 if (videoEncoderOutputBufferInfo.size != 0) { 1095 muxer.writeSampleData( 1096 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1097 } 1098 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1099 != 0) { 1100 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1101 videoEncoderDone = true; 1102 } 1103 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1104 videoEncodedFrameCount++; 1105 // We enqueued an encoded frame, let's try something else next. 1106 break; 1107 } 1108 1109 // Poll frames from the audio encoder and send them to the muxer. 1110 while (mCopyAudio && !audioEncoderDone 1111 && (encoderOutputAudioFormat == null || muxing)) { 1112 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1113 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1114 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1115 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1116 break; 1117 } 1118 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1119 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1120 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1121 break; 1122 } 1123 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1124 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1125 if (outputAudioTrack >= 0) { 1126 fail("audio encoder changed its output format again?"); 1127 } 1128 1129 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1130 break; 1131 } 1132 assertTrue("should have added track before processing output", muxing); 1133 if (VERBOSE) { 1134 Log.d(TAG, "audio encoder: returned output buffer: " 1135 + encoderOutputBufferIndex); 1136 Log.d(TAG, "audio encoder: returned buffer of size " 1137 + audioEncoderOutputBufferInfo.size); 1138 } 1139 ByteBuffer encoderOutputBuffer = 1140 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1141 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1142 != 0) { 1143 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1144 // Simply ignore codec config buffers. 1145 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1146 break; 1147 } 1148 if (VERBOSE) { 1149 Log.d(TAG, "audio encoder: returned buffer for time " 1150 + audioEncoderOutputBufferInfo.presentationTimeUs); 1151 } 1152 if (audioEncoderOutputBufferInfo.size != 0) { 1153 muxer.writeSampleData( 1154 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1155 } 1156 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1157 != 0) { 1158 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1159 audioEncoderDone = true; 1160 } 1161 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1162 audioEncodedFrameCount++; 1163 // We enqueued an encoded frame, let's try something else next. 1164 break; 1165 } 1166 1167 if (!muxing 1168 && (!mCopyAudio || encoderOutputAudioFormat != null) 1169 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1170 if (mCopyVideo) { 1171 Log.d(TAG, "muxer: adding video track."); 1172 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1173 } 1174 if (mCopyAudio) { 1175 Log.d(TAG, "muxer: adding audio track."); 1176 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1177 } 1178 Log.d(TAG, "muxer: starting"); 1179 muxer.start(); 1180 muxing = true; 1181 } 1182 } 1183 1184 // Basic sanity checks. 1185 if (mCopyVideo) { 1186 assertEquals("encoded and decoded video frame counts should match", 1187 videoDecodedFrameCount, videoEncodedFrameCount); 1188 assertTrue("decoded frame count should be less than extracted frame count", 1189 videoDecodedFrameCount <= videoExtractedFrameCount); 1190 } 1191 if (mCopyAudio) { 1192 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1193 } 1194 } 1195 isVideoFormat(MediaFormat format)1196 private static boolean isVideoFormat(MediaFormat format) { 1197 return getMimeTypeFor(format).startsWith("video/"); 1198 } 1199 isAudioFormat(MediaFormat format)1200 private static boolean isAudioFormat(MediaFormat format) { 1201 return getMimeTypeFor(format).startsWith("audio/"); 1202 } 1203 getMimeTypeFor(MediaFormat format)1204 private static String getMimeTypeFor(MediaFormat format) { 1205 return format.getString(MediaFormat.KEY_MIME); 1206 } 1207 1208 /** 1209 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1210 * found. 1211 */ selectCodec(String mimeType)1212 private static MediaCodecInfo selectCodec(String mimeType) { 1213 int numCodecs = MediaCodecList.getCodecCount(); 1214 for (int i = 0; i < numCodecs; i++) { 1215 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1216 1217 if (!codecInfo.isEncoder()) { 1218 continue; 1219 } 1220 1221 String[] types = codecInfo.getSupportedTypes(); 1222 for (int j = 0; j < types.length; j++) { 1223 if (types[j].equalsIgnoreCase(mimeType)) { 1224 return codecInfo; 1225 } 1226 } 1227 } 1228 return null; 1229 } 1230 } 1231