1 /* 2 * Copyright (C) 2013 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media.cts; 18 19 import android.annotation.TargetApi; 20 import android.content.res.AssetFileDescriptor; 21 import android.content.Context; 22 import android.media.MediaCodec; 23 import android.media.MediaCodecInfo; 24 import android.media.MediaCodecInfo.CodecCapabilities; 25 import android.media.MediaCodecInfo.CodecProfileLevel; 26 import android.media.MediaCodecList; 27 import android.media.MediaExtractor; 28 import android.media.MediaFormat; 29 import android.media.MediaMuxer; 30 import android.media.MediaPlayer; 31 import android.os.Environment; 32 import android.platform.test.annotations.AppModeFull; 33 import android.test.ActivityInstrumentationTestCase2; 34 import android.util.Log; 35 import android.view.Surface; 36 37 import android.media.MediaCodecInfo; 38 import android.media.MediaCodecInfo.CodecCapabilities; 39 import android.media.MediaCodecInfo.CodecProfileLevel; 40 41 import android.media.cts.R; 42 43 import java.io.File; 44 import java.io.IOException; 45 import java.nio.ByteBuffer; 46 import java.util.concurrent.atomic.AtomicReference; 47 import java.util.concurrent.CountDownLatch; 48 49 /** 50 * Test for the integration of MediaMuxer and MediaCodec's encoder. 51 * 52 * <p>It uses MediaExtractor to get frames from a test stream, decodes them to a surface, uses a 53 * shader to edit them, encodes them from the resulting surface, and then uses MediaMuxer to write 54 * them into a file. 55 * 56 * <p>It does not currently check whether the result file is correct, but makes sure that nothing 57 * fails along the way. 58 * 59 * <p>It also tests the way the codec config buffers need to be passed from the MediaCodec to the 60 * MediaMuxer. 61 */ 62 @TargetApi(18) 63 @AppModeFull(reason = "Instant apps cannot access the SD card") 64 public class ExtractDecodeEditEncodeMuxTest 65 extends ActivityInstrumentationTestCase2<MediaStubActivity> { 66 67 private static final String TAG = ExtractDecodeEditEncodeMuxTest.class.getSimpleName(); 68 private static final boolean VERBOSE = false; // lots of logging 69 70 /** How long to wait for the next buffer to become available. */ 71 private static final int TIMEOUT_USEC = 10000; 72 73 /** Where to output the test files. */ 74 private static final File OUTPUT_FILENAME_DIR = Environment.getExternalStorageDirectory(); 75 76 // parameters for the video encoder 77 private static final int OUTPUT_VIDEO_BIT_RATE = 2000000; // 2Mbps 78 private static final int OUTPUT_VIDEO_FRAME_RATE = 15; // 15fps 79 private static final int OUTPUT_VIDEO_IFRAME_INTERVAL = 10; // 10 seconds between I-frames 80 private static final int OUTPUT_VIDEO_COLOR_FORMAT = 81 MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface; 82 83 // parameters for the audio encoder 84 // Advanced Audio Coding 85 private static final String OUTPUT_AUDIO_MIME_TYPE = MediaFormat.MIMETYPE_AUDIO_AAC; 86 private static final int OUTPUT_AUDIO_CHANNEL_COUNT = 2; // Must match the input stream. 87 private static final int OUTPUT_AUDIO_BIT_RATE = 128 * 1024; 88 private static final int OUTPUT_AUDIO_AAC_PROFILE = 89 MediaCodecInfo.CodecProfileLevel.AACObjectHE; 90 private static final int OUTPUT_AUDIO_SAMPLE_RATE_HZ = 44100; // Must match the input stream. 91 92 /** 93 * Used for editing the frames. 94 * 95 * <p>Swaps green and blue channels by storing an RBGA color in an RGBA buffer. 96 */ 97 private static final String FRAGMENT_SHADER = 98 "#extension GL_OES_EGL_image_external : require\n" + 99 "precision mediump float;\n" + 100 "varying vec2 vTextureCoord;\n" + 101 "uniform samplerExternalOES sTexture;\n" + 102 "void main() {\n" + 103 " gl_FragColor = texture2D(sTexture, vTextureCoord).rbga;\n" + 104 "}\n"; 105 106 /** Whether to copy the video from the test video. */ 107 private boolean mCopyVideo; 108 /** Whether to copy the audio from the test video. */ 109 private boolean mCopyAudio; 110 /** Whether to verify the audio format. */ 111 private boolean mVerifyAudioFormat; 112 /** Width of the output frames. */ 113 private int mWidth = -1; 114 /** Height of the output frames. */ 115 private int mHeight = -1; 116 117 /** The raw resource used as the input file. */ 118 private int mSourceResId; 119 120 /** The destination file for the encoded output. */ 121 private String mOutputFile; 122 123 private String mOutputVideoMimeType; 124 ExtractDecodeEditEncodeMuxTest()125 public ExtractDecodeEditEncodeMuxTest() { 126 super(MediaStubActivity.class); 127 } 128 testExtractDecodeEditEncodeMuxQCIF()129 public void testExtractDecodeEditEncodeMuxQCIF() throws Throwable { 130 if(!setSize(176, 144)) return; 131 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 132 setCopyVideo(); 133 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 134 TestWrapper.runTest(this); 135 } 136 testExtractDecodeEditEncodeMuxQVGA()137 public void testExtractDecodeEditEncodeMuxQVGA() throws Throwable { 138 if(!setSize(320, 240)) return; 139 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 140 setCopyVideo(); 141 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 142 TestWrapper.runTest(this); 143 } 144 testExtractDecodeEditEncodeMux720p()145 public void testExtractDecodeEditEncodeMux720p() throws Throwable { 146 if(!setSize(1280, 720)) return; 147 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 148 setCopyVideo(); 149 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_AVC); 150 TestWrapper.runTest(this); 151 } 152 testExtractDecodeEditEncodeMux2160pHevc()153 public void testExtractDecodeEditEncodeMux2160pHevc() throws Throwable { 154 if(!setSize(3840, 2160)) return; 155 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 156 setCopyVideo(); 157 setVideoMimeType(MediaFormat.MIMETYPE_VIDEO_HEVC); 158 TestWrapper.runTest(this); 159 } 160 testExtractDecodeEditEncodeMuxAudio()161 public void testExtractDecodeEditEncodeMuxAudio() throws Throwable { 162 if(!setSize(1280, 720)) return; 163 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 164 setCopyAudio(); 165 setVerifyAudioFormat(); 166 TestWrapper.runTest(this); 167 } 168 testExtractDecodeEditEncodeMuxAudioVideo()169 public void testExtractDecodeEditEncodeMuxAudioVideo() throws Throwable { 170 if(!setSize(1280, 720)) return; 171 setSource(R.raw.video_480x360_mp4_h264_500kbps_30fps_aac_stereo_128kbps_44100hz); 172 setCopyAudio(); 173 setCopyVideo(); 174 setVerifyAudioFormat(); 175 TestWrapper.runTest(this); 176 } 177 178 /** Wraps testExtractDecodeEditEncodeMux() */ 179 private static class TestWrapper implements Runnable { 180 private Throwable mThrowable; 181 private ExtractDecodeEditEncodeMuxTest mTest; 182 TestWrapper(ExtractDecodeEditEncodeMuxTest test)183 private TestWrapper(ExtractDecodeEditEncodeMuxTest test) { 184 mTest = test; 185 } 186 187 @Override run()188 public void run() { 189 try { 190 mTest.extractDecodeEditEncodeMux(); 191 } catch (Throwable th) { 192 mThrowable = th; 193 } 194 } 195 196 /** 197 * Entry point. 198 */ runTest(ExtractDecodeEditEncodeMuxTest test)199 public static void runTest(ExtractDecodeEditEncodeMuxTest test) throws Throwable { 200 test.setOutputFile(); 201 TestWrapper wrapper = new TestWrapper(test); 202 Thread th = new Thread(wrapper, "codec test"); 203 th.start(); 204 th.join(); 205 if (wrapper.mThrowable != null) { 206 throw wrapper.mThrowable; 207 } 208 } 209 } 210 211 /** 212 * Sets the test to copy the video stream. 213 */ setCopyVideo()214 private void setCopyVideo() { 215 mCopyVideo = true; 216 } 217 218 /** 219 * Sets the test to copy the video stream. 220 */ setCopyAudio()221 private void setCopyAudio() { 222 mCopyAudio = true; 223 } 224 225 /** 226 * Sets the test to verify the output audio format. 227 */ setVerifyAudioFormat()228 private void setVerifyAudioFormat() { 229 mVerifyAudioFormat = true; 230 } 231 232 /** 233 * Sets the desired frame size and returns whether the given resolution is 234 * supported. 235 * 236 * <p>If decoding/encoding using AVC as the codec, checks that the resolution 237 * is supported. For other codecs, always return {@code true}. 238 */ setSize(int width, int height)239 private boolean setSize(int width, int height) { 240 if ((width % 16) != 0 || (height % 16) != 0) { 241 Log.w(TAG, "WARNING: width or height not multiple of 16"); 242 } 243 mWidth = width; 244 mHeight = height; 245 246 // TODO: remove this logic in setSize as it is now handled when configuring codecs 247 return true; 248 } 249 250 /** 251 * Sets the raw resource used as the source video. 252 */ setSource(int resId)253 private void setSource(int resId) { 254 mSourceResId = resId; 255 } 256 257 /** 258 * Sets the name of the output file based on the other parameters. 259 * 260 * <p>Must be called after {@link #setSize(int, int)} and {@link #setSource(int)}. 261 */ setOutputFile()262 private void setOutputFile() { 263 StringBuilder sb = new StringBuilder(); 264 sb.append(OUTPUT_FILENAME_DIR.getAbsolutePath()); 265 sb.append("/cts-media-"); 266 sb.append(getClass().getSimpleName()); 267 assertTrue("should have called setSource() first", mSourceResId != -1); 268 sb.append('-'); 269 sb.append(mSourceResId); 270 if (mCopyVideo) { 271 assertTrue("should have called setSize() first", mWidth != -1); 272 assertTrue("should have called setSize() first", mHeight != -1); 273 sb.append('-'); 274 sb.append("video"); 275 sb.append('-'); 276 sb.append(mWidth); 277 sb.append('x'); 278 sb.append(mHeight); 279 } 280 if (mCopyAudio) { 281 sb.append('-'); 282 sb.append("audio"); 283 } 284 sb.append(".mp4"); 285 mOutputFile = sb.toString(); 286 } 287 setVideoMimeType(String mimeType)288 private void setVideoMimeType(String mimeType) { 289 mOutputVideoMimeType = mimeType; 290 } 291 292 /** 293 * Tests encoding and subsequently decoding video from frames generated into a buffer. 294 * <p> 295 * We encode several frames of a video test pattern using MediaCodec, then decode the output 296 * with MediaCodec and do some simple checks. 297 */ extractDecodeEditEncodeMux()298 private void extractDecodeEditEncodeMux() throws Exception { 299 // Exception that may be thrown during release. 300 Exception exception = null; 301 302 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 303 304 // We avoid the device-specific limitations on width and height by using values 305 // that are multiples of 16, which all tested devices seem to be able to handle. 306 MediaFormat outputVideoFormat = 307 MediaFormat.createVideoFormat(mOutputVideoMimeType, mWidth, mHeight); 308 309 // Set some properties. Failing to specify some of these can cause the MediaCodec 310 // configure() call to throw an unhelpful exception. 311 outputVideoFormat.setInteger( 312 MediaFormat.KEY_COLOR_FORMAT, OUTPUT_VIDEO_COLOR_FORMAT); 313 outputVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_VIDEO_BIT_RATE); 314 outputVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, OUTPUT_VIDEO_FRAME_RATE); 315 outputVideoFormat.setInteger( 316 MediaFormat.KEY_I_FRAME_INTERVAL, OUTPUT_VIDEO_IFRAME_INTERVAL); 317 if (VERBOSE) Log.d(TAG, "video format: " + outputVideoFormat); 318 319 String videoEncoderName = mcl.findEncoderForFormat(outputVideoFormat); 320 if (videoEncoderName == null) { 321 // Don't fail CTS if they don't have an AVC codec (not here, anyway). 322 Log.e(TAG, "Unable to find an appropriate codec for " + outputVideoFormat); 323 return; 324 } 325 if (VERBOSE) Log.d(TAG, "video found codec: " + videoEncoderName); 326 327 MediaFormat outputAudioFormat = 328 MediaFormat.createAudioFormat( 329 OUTPUT_AUDIO_MIME_TYPE, OUTPUT_AUDIO_SAMPLE_RATE_HZ, 330 OUTPUT_AUDIO_CHANNEL_COUNT); 331 outputAudioFormat.setInteger(MediaFormat.KEY_BIT_RATE, OUTPUT_AUDIO_BIT_RATE); 332 // TODO: Bug workaround --- uncomment once fixed. 333 // outputAudioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, OUTPUT_AUDIO_AAC_PROFILE); 334 335 String audioEncoderName = mcl.findEncoderForFormat(outputAudioFormat); 336 if (audioEncoderName == null) { 337 // Don't fail CTS if they don't have an AAC codec (not here, anyway). 338 Log.e(TAG, "Unable to find an appropriate codec for " + outputAudioFormat); 339 return; 340 } 341 if (VERBOSE) Log.d(TAG, "audio found codec: " + audioEncoderName); 342 343 MediaExtractor videoExtractor = null; 344 MediaExtractor audioExtractor = null; 345 OutputSurface outputSurface = null; 346 MediaCodec videoDecoder = null; 347 MediaCodec audioDecoder = null; 348 MediaCodec videoEncoder = null; 349 MediaCodec audioEncoder = null; 350 MediaMuxer muxer = null; 351 352 InputSurface inputSurface = null; 353 354 try { 355 if (mCopyVideo) { 356 videoExtractor = createExtractor(); 357 int videoInputTrack = getAndSelectVideoTrackIndex(videoExtractor); 358 assertTrue("missing video track in test video", videoInputTrack != -1); 359 MediaFormat inputFormat = videoExtractor.getTrackFormat(videoInputTrack); 360 361 // Create a MediaCodec for the desired codec, then configure it as an encoder with 362 // our desired properties. Request a Surface to use for input. 363 AtomicReference<Surface> inputSurfaceReference = new AtomicReference<Surface>(); 364 videoEncoder = createVideoEncoder( 365 videoEncoderName, outputVideoFormat, inputSurfaceReference); 366 inputSurface = new InputSurface(inputSurfaceReference.get()); 367 inputSurface.makeCurrent(); 368 // Create a MediaCodec for the decoder, based on the extractor's format. 369 outputSurface = new OutputSurface(); 370 outputSurface.changeFragmentShader(FRAGMENT_SHADER); 371 videoDecoder = createVideoDecoder(mcl, inputFormat, outputSurface.getSurface()); 372 } 373 374 if (mCopyAudio) { 375 audioExtractor = createExtractor(); 376 int audioInputTrack = getAndSelectAudioTrackIndex(audioExtractor); 377 assertTrue("missing audio track in test video", audioInputTrack != -1); 378 MediaFormat inputFormat = audioExtractor.getTrackFormat(audioInputTrack); 379 380 // Create a MediaCodec for the desired codec, then configure it as an encoder with 381 // our desired properties. Request a Surface to use for input. 382 audioEncoder = createAudioEncoder(audioEncoderName, outputAudioFormat); 383 // Create a MediaCodec for the decoder, based on the extractor's format. 384 audioDecoder = createAudioDecoder(mcl, inputFormat); 385 } 386 387 // Creates a muxer but do not start or add tracks just yet. 388 muxer = createMuxer(); 389 390 doExtractDecodeEditEncodeMux( 391 videoExtractor, 392 audioExtractor, 393 videoDecoder, 394 videoEncoder, 395 audioDecoder, 396 audioEncoder, 397 muxer, 398 inputSurface, 399 outputSurface); 400 } finally { 401 if (VERBOSE) Log.d(TAG, "releasing extractor, decoder, encoder, and muxer"); 402 // Try to release everything we acquired, even if one of the releases fails, in which 403 // case we save the first exception we got and re-throw at the end (unless something 404 // other exception has already been thrown). This guarantees the first exception thrown 405 // is reported as the cause of the error, everything is (attempted) to be released, and 406 // all other exceptions appear in the logs. 407 try { 408 if (videoExtractor != null) { 409 videoExtractor.release(); 410 } 411 } catch(Exception e) { 412 Log.e(TAG, "error while releasing videoExtractor", e); 413 if (exception == null) { 414 exception = e; 415 } 416 } 417 try { 418 if (audioExtractor != null) { 419 audioExtractor.release(); 420 } 421 } catch(Exception e) { 422 Log.e(TAG, "error while releasing audioExtractor", e); 423 if (exception == null) { 424 exception = e; 425 } 426 } 427 try { 428 if (videoDecoder != null) { 429 videoDecoder.stop(); 430 videoDecoder.release(); 431 } 432 } catch(Exception e) { 433 Log.e(TAG, "error while releasing videoDecoder", e); 434 if (exception == null) { 435 exception = e; 436 } 437 } 438 try { 439 if (outputSurface != null) { 440 outputSurface.release(); 441 } 442 } catch(Exception e) { 443 Log.e(TAG, "error while releasing outputSurface", e); 444 if (exception == null) { 445 exception = e; 446 } 447 } 448 try { 449 if (videoEncoder != null) { 450 videoEncoder.stop(); 451 videoEncoder.release(); 452 } 453 } catch(Exception e) { 454 Log.e(TAG, "error while releasing videoEncoder", e); 455 if (exception == null) { 456 exception = e; 457 } 458 } 459 try { 460 if (audioDecoder != null) { 461 audioDecoder.stop(); 462 audioDecoder.release(); 463 } 464 } catch(Exception e) { 465 Log.e(TAG, "error while releasing audioDecoder", e); 466 if (exception == null) { 467 exception = e; 468 } 469 } 470 try { 471 if (audioEncoder != null) { 472 audioEncoder.stop(); 473 audioEncoder.release(); 474 } 475 } catch(Exception e) { 476 Log.e(TAG, "error while releasing audioEncoder", e); 477 if (exception == null) { 478 exception = e; 479 } 480 } 481 try { 482 if (muxer != null) { 483 muxer.stop(); 484 muxer.release(); 485 } 486 } catch(Exception e) { 487 Log.e(TAG, "error while releasing muxer", e); 488 if (exception == null) { 489 exception = e; 490 } 491 } 492 try { 493 if (inputSurface != null) { 494 inputSurface.release(); 495 } 496 } catch(Exception e) { 497 Log.e(TAG, "error while releasing inputSurface", e); 498 if (exception == null) { 499 exception = e; 500 } 501 } 502 } 503 if (exception != null) { 504 throw exception; 505 } 506 507 MediaExtractor mediaExtractor = null; 508 try { 509 mediaExtractor = new MediaExtractor(); 510 mediaExtractor.setDataSource(mOutputFile); 511 512 assertEquals("incorrect number of tracks", (mCopyAudio ? 1 : 0) + (mCopyVideo ? 1 : 0), 513 mediaExtractor.getTrackCount()); 514 if (mVerifyAudioFormat) { 515 boolean foundAudio = false; 516 for (int i = 0; i < mediaExtractor.getTrackCount(); i++) { 517 MediaFormat trackFormat = mediaExtractor.getTrackFormat(i); 518 if (isAudioFormat(trackFormat)) { 519 foundAudio = true; 520 int expectedSampleRate = OUTPUT_AUDIO_SAMPLE_RATE_HZ; 521 522 // SBR mode halves the sample rate in the format. 523 if (OUTPUT_AUDIO_AAC_PROFILE == 524 MediaCodecInfo.CodecProfileLevel.AACObjectHE) { 525 expectedSampleRate /= 2; 526 } 527 assertEquals("sample rates should match", expectedSampleRate, 528 trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); 529 } 530 } 531 532 assertTrue("output should have an audio track", foundAudio || !mCopyAudio); 533 } 534 } catch (IOException e) { 535 throw new IllegalStateException("exception verifying output file", e); 536 } finally { 537 if (mediaExtractor != null) { 538 mediaExtractor.release(); 539 } 540 } 541 542 // TODO: Check the generated output file's video format and sample data. 543 544 MediaStubActivity activity = getActivity(); 545 final MediaPlayer mp = new MediaPlayer(); 546 final Exception[] exceptionHolder = { null }; 547 final CountDownLatch playbackEndSignal = new CountDownLatch(1); 548 mp.setOnErrorListener(new MediaPlayer.OnErrorListener() { 549 @Override 550 public boolean onError(MediaPlayer origin, int what, int extra) { 551 exceptionHolder[0] = new RuntimeException("error playing output file: what=" + what 552 + " extra=" + extra); 553 // Returning false would trigger onCompletion() so that 554 // playbackEndSignal.await() can stop waiting. 555 return false; 556 } 557 }); 558 mp.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { 559 @Override 560 public void onCompletion(MediaPlayer origin) { 561 playbackEndSignal.countDown(); 562 } 563 }); 564 try { 565 mp.setDataSource(mOutputFile); 566 mp.setDisplay(activity.getSurfaceHolder()); 567 mp.prepare(); 568 mp.start(); 569 playbackEndSignal.await(); 570 } catch (Exception e) { 571 exceptionHolder[0] = e; 572 } finally { 573 mp.release(); 574 } 575 576 if (exceptionHolder[0] != null) { 577 throw exceptionHolder[0]; 578 } 579 } 580 581 /** 582 * Creates an extractor that reads its frames from {@link #mSourceResId}. 583 */ createExtractor()584 private MediaExtractor createExtractor() throws IOException { 585 MediaExtractor extractor; 586 Context context = getInstrumentation().getTargetContext(); 587 AssetFileDescriptor srcFd = context.getResources().openRawResourceFd(mSourceResId); 588 extractor = new MediaExtractor(); 589 extractor.setDataSource(srcFd.getFileDescriptor(), srcFd.getStartOffset(), 590 srcFd.getLength()); 591 return extractor; 592 } 593 594 /** 595 * Creates a decoder for the given format, which outputs to the given surface. 596 * 597 * @param inputFormat the format of the stream to decode 598 * @param surface into which to decode the frames 599 */ createVideoDecoder( MediaCodecList mcl, MediaFormat inputFormat, Surface surface)600 private MediaCodec createVideoDecoder( 601 MediaCodecList mcl, MediaFormat inputFormat, Surface surface) throws IOException { 602 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 603 decoder.configure(inputFormat, surface, null, 0); 604 decoder.start(); 605 return decoder; 606 } 607 608 /** 609 * Creates an encoder for the given format using the specified codec, taking input from a 610 * surface. 611 * 612 * <p>The surface to use as input is stored in the given reference. 613 * 614 * @param codecInfo of the codec to use 615 * @param format of the stream to be produced 616 * @param surfaceReference to store the surface to use as input 617 */ createVideoEncoder( String codecName, MediaFormat format, AtomicReference<Surface> surfaceReference)618 private MediaCodec createVideoEncoder( 619 String codecName, 620 MediaFormat format, 621 AtomicReference<Surface> surfaceReference) 622 throws IOException { 623 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 624 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 625 // Must be called before start() is. 626 surfaceReference.set(encoder.createInputSurface()); 627 encoder.start(); 628 return encoder; 629 } 630 631 /** 632 * Creates a decoder for the given format. 633 * 634 * @param inputFormat the format of the stream to decode 635 */ createAudioDecoder( MediaCodecList mcl, MediaFormat inputFormat)636 private MediaCodec createAudioDecoder( 637 MediaCodecList mcl, MediaFormat inputFormat) throws IOException { 638 MediaCodec decoder = MediaCodec.createByCodecName(mcl.findDecoderForFormat(inputFormat)); 639 decoder.configure(inputFormat, null, null, 0); 640 decoder.start(); 641 return decoder; 642 } 643 644 /** 645 * Creates an encoder for the given format using the specified codec. 646 * 647 * @param codecInfo of the codec to use 648 * @param format of the stream to be produced 649 */ createAudioEncoder(String codecName, MediaFormat format)650 private MediaCodec createAudioEncoder(String codecName, MediaFormat format) 651 throws IOException { 652 MediaCodec encoder = MediaCodec.createByCodecName(codecName); 653 encoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); 654 encoder.start(); 655 return encoder; 656 } 657 658 /** 659 * Creates a muxer to write the encoded frames. 660 * 661 * <p>The muxer is not started as it needs to be started only after all streams have been added. 662 */ createMuxer()663 private MediaMuxer createMuxer() throws IOException { 664 return new MediaMuxer(mOutputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 665 } 666 getAndSelectVideoTrackIndex(MediaExtractor extractor)667 private int getAndSelectVideoTrackIndex(MediaExtractor extractor) { 668 for (int index = 0; index < extractor.getTrackCount(); ++index) { 669 if (VERBOSE) { 670 Log.d(TAG, "format for track " + index + " is " 671 + getMimeTypeFor(extractor.getTrackFormat(index))); 672 } 673 if (isVideoFormat(extractor.getTrackFormat(index))) { 674 extractor.selectTrack(index); 675 return index; 676 } 677 } 678 return -1; 679 } 680 getAndSelectAudioTrackIndex(MediaExtractor extractor)681 private int getAndSelectAudioTrackIndex(MediaExtractor extractor) { 682 for (int index = 0; index < extractor.getTrackCount(); ++index) { 683 if (VERBOSE) { 684 Log.d(TAG, "format for track " + index + " is " 685 + getMimeTypeFor(extractor.getTrackFormat(index))); 686 } 687 if (isAudioFormat(extractor.getTrackFormat(index))) { 688 extractor.selectTrack(index); 689 return index; 690 } 691 } 692 return -1; 693 } 694 695 /** 696 * Does the actual work for extracting, decoding, encoding and muxing. 697 */ doExtractDecodeEditEncodeMux( MediaExtractor videoExtractor, MediaExtractor audioExtractor, MediaCodec videoDecoder, MediaCodec videoEncoder, MediaCodec audioDecoder, MediaCodec audioEncoder, MediaMuxer muxer, InputSurface inputSurface, OutputSurface outputSurface)698 private void doExtractDecodeEditEncodeMux( 699 MediaExtractor videoExtractor, 700 MediaExtractor audioExtractor, 701 MediaCodec videoDecoder, 702 MediaCodec videoEncoder, 703 MediaCodec audioDecoder, 704 MediaCodec audioEncoder, 705 MediaMuxer muxer, 706 InputSurface inputSurface, 707 OutputSurface outputSurface) { 708 ByteBuffer[] videoDecoderInputBuffers = null; 709 ByteBuffer[] videoDecoderOutputBuffers = null; 710 ByteBuffer[] videoEncoderOutputBuffers = null; 711 MediaCodec.BufferInfo videoDecoderOutputBufferInfo = null; 712 MediaCodec.BufferInfo videoEncoderOutputBufferInfo = null; 713 if (mCopyVideo) { 714 videoDecoderInputBuffers = videoDecoder.getInputBuffers(); 715 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 716 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 717 videoDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 718 videoEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 719 } 720 ByteBuffer[] audioDecoderInputBuffers = null; 721 ByteBuffer[] audioDecoderOutputBuffers = null; 722 ByteBuffer[] audioEncoderInputBuffers = null; 723 ByteBuffer[] audioEncoderOutputBuffers = null; 724 MediaCodec.BufferInfo audioDecoderOutputBufferInfo = null; 725 MediaCodec.BufferInfo audioEncoderOutputBufferInfo = null; 726 if (mCopyAudio) { 727 audioDecoderInputBuffers = audioDecoder.getInputBuffers(); 728 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 729 audioEncoderInputBuffers = audioEncoder.getInputBuffers(); 730 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 731 audioDecoderOutputBufferInfo = new MediaCodec.BufferInfo(); 732 audioEncoderOutputBufferInfo = new MediaCodec.BufferInfo(); 733 } 734 // We will get these from the decoders when notified of a format change. 735 MediaFormat decoderOutputVideoFormat = null; 736 MediaFormat decoderOutputAudioFormat = null; 737 // We will get these from the encoders when notified of a format change. 738 MediaFormat encoderOutputVideoFormat = null; 739 MediaFormat encoderOutputAudioFormat = null; 740 // We will determine these once we have the output format. 741 int outputVideoTrack = -1; 742 int outputAudioTrack = -1; 743 // Whether things are done on the video side. 744 boolean videoExtractorDone = false; 745 boolean videoDecoderDone = false; 746 boolean videoEncoderDone = false; 747 // Whether things are done on the audio side. 748 boolean audioExtractorDone = false; 749 boolean audioDecoderDone = false; 750 boolean audioEncoderDone = false; 751 // The audio decoder output buffer to process, -1 if none. 752 int pendingAudioDecoderOutputBufferIndex = -1; 753 754 boolean muxing = false; 755 756 int videoExtractedFrameCount = 0; 757 int videoDecodedFrameCount = 0; 758 int videoEncodedFrameCount = 0; 759 760 int audioExtractedFrameCount = 0; 761 int audioDecodedFrameCount = 0; 762 int audioEncodedFrameCount = 0; 763 764 while ((mCopyVideo && !videoEncoderDone) || (mCopyAudio && !audioEncoderDone)) { 765 if (VERBOSE) { 766 Log.d(TAG, String.format( 767 "loop: " 768 769 + "V(%b){" 770 + "extracted:%d(done:%b) " 771 + "decoded:%d(done:%b) " 772 + "encoded:%d(done:%b)} " 773 774 + "A(%b){" 775 + "extracted:%d(done:%b) " 776 + "decoded:%d(done:%b) " 777 + "encoded:%d(done:%b) " 778 + "pending:%d} " 779 780 + "muxing:%b(V:%d,A:%d)", 781 782 mCopyVideo, 783 videoExtractedFrameCount, videoExtractorDone, 784 videoDecodedFrameCount, videoDecoderDone, 785 videoEncodedFrameCount, videoEncoderDone, 786 787 mCopyAudio, 788 audioExtractedFrameCount, audioExtractorDone, 789 audioDecodedFrameCount, audioDecoderDone, 790 audioEncodedFrameCount, audioEncoderDone, 791 pendingAudioDecoderOutputBufferIndex, 792 793 muxing, outputVideoTrack, outputAudioTrack)); 794 } 795 796 // Extract video from file and feed to decoder. 797 // Do not extract video if we have determined the output format but we are not yet 798 // ready to mux the frames. 799 while (mCopyVideo && !videoExtractorDone 800 && (encoderOutputVideoFormat == null || muxing)) { 801 int decoderInputBufferIndex = videoDecoder.dequeueInputBuffer(TIMEOUT_USEC); 802 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 803 if (VERBOSE) Log.d(TAG, "no video decoder input buffer"); 804 break; 805 } 806 if (VERBOSE) { 807 Log.d(TAG, "video decoder: returned input buffer: " + decoderInputBufferIndex); 808 } 809 ByteBuffer decoderInputBuffer = videoDecoderInputBuffers[decoderInputBufferIndex]; 810 int size = videoExtractor.readSampleData(decoderInputBuffer, 0); 811 long presentationTime = videoExtractor.getSampleTime(); 812 if (VERBOSE) { 813 Log.d(TAG, "video extractor: returned buffer of size " + size); 814 Log.d(TAG, "video extractor: returned buffer for time " + presentationTime); 815 } 816 if (size >= 0) { 817 videoDecoder.queueInputBuffer( 818 decoderInputBufferIndex, 819 0, 820 size, 821 presentationTime, 822 videoExtractor.getSampleFlags()); 823 } 824 videoExtractorDone = !videoExtractor.advance(); 825 if (videoExtractorDone) { 826 if (VERBOSE) Log.d(TAG, "video extractor: EOS"); 827 videoDecoder.queueInputBuffer( 828 decoderInputBufferIndex, 829 0, 830 0, 831 0, 832 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 833 } 834 videoExtractedFrameCount++; 835 // We extracted a frame, let's try something else next. 836 break; 837 } 838 839 // Extract audio from file and feed to decoder. 840 // Do not extract audio if we have determined the output format but we are not yet 841 // ready to mux the frames. 842 while (mCopyAudio && !audioExtractorDone 843 && (encoderOutputAudioFormat == null || muxing)) { 844 int decoderInputBufferIndex = audioDecoder.dequeueInputBuffer(TIMEOUT_USEC); 845 if (decoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 846 if (VERBOSE) Log.d(TAG, "no audio decoder input buffer"); 847 break; 848 } 849 if (VERBOSE) { 850 Log.d(TAG, "audio decoder: returned input buffer: " + decoderInputBufferIndex); 851 } 852 ByteBuffer decoderInputBuffer = audioDecoderInputBuffers[decoderInputBufferIndex]; 853 int size = audioExtractor.readSampleData(decoderInputBuffer, 0); 854 long presentationTime = audioExtractor.getSampleTime(); 855 if (VERBOSE) { 856 Log.d(TAG, "audio extractor: returned buffer of size " + size); 857 Log.d(TAG, "audio extractor: returned buffer for time " + presentationTime); 858 } 859 if (size >= 0) { 860 audioDecoder.queueInputBuffer( 861 decoderInputBufferIndex, 862 0, 863 size, 864 presentationTime, 865 audioExtractor.getSampleFlags()); 866 } 867 audioExtractorDone = !audioExtractor.advance(); 868 if (audioExtractorDone) { 869 if (VERBOSE) Log.d(TAG, "audio extractor: EOS"); 870 audioDecoder.queueInputBuffer( 871 decoderInputBufferIndex, 872 0, 873 0, 874 0, 875 MediaCodec.BUFFER_FLAG_END_OF_STREAM); 876 } 877 audioExtractedFrameCount++; 878 // We extracted a frame, let's try something else next. 879 break; 880 } 881 882 // Poll output frames from the video decoder and feed the encoder. 883 while (mCopyVideo && !videoDecoderDone 884 && (encoderOutputVideoFormat == null || muxing)) { 885 int decoderOutputBufferIndex = 886 videoDecoder.dequeueOutputBuffer( 887 videoDecoderOutputBufferInfo, TIMEOUT_USEC); 888 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 889 if (VERBOSE) Log.d(TAG, "no video decoder output buffer"); 890 break; 891 } 892 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 893 if (VERBOSE) Log.d(TAG, "video decoder: output buffers changed"); 894 videoDecoderOutputBuffers = videoDecoder.getOutputBuffers(); 895 break; 896 } 897 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 898 decoderOutputVideoFormat = videoDecoder.getOutputFormat(); 899 if (VERBOSE) { 900 Log.d(TAG, "video decoder: output format changed: " 901 + decoderOutputVideoFormat); 902 } 903 break; 904 } 905 if (VERBOSE) { 906 Log.d(TAG, "video decoder: returned output buffer: " 907 + decoderOutputBufferIndex); 908 Log.d(TAG, "video decoder: returned buffer of size " 909 + videoDecoderOutputBufferInfo.size); 910 } 911 ByteBuffer decoderOutputBuffer = 912 videoDecoderOutputBuffers[decoderOutputBufferIndex]; 913 if ((videoDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 914 != 0) { 915 if (VERBOSE) Log.d(TAG, "video decoder: codec config buffer"); 916 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 917 break; 918 } 919 if (VERBOSE) { 920 Log.d(TAG, "video decoder: returned buffer for time " 921 + videoDecoderOutputBufferInfo.presentationTimeUs); 922 } 923 boolean render = videoDecoderOutputBufferInfo.size != 0; 924 videoDecoder.releaseOutputBuffer(decoderOutputBufferIndex, render); 925 if (render) { 926 if (VERBOSE) Log.d(TAG, "output surface: await new image"); 927 outputSurface.awaitNewImage(); 928 // Edit the frame and send it to the encoder. 929 if (VERBOSE) Log.d(TAG, "output surface: draw image"); 930 outputSurface.drawImage(); 931 inputSurface.setPresentationTime( 932 videoDecoderOutputBufferInfo.presentationTimeUs * 1000); 933 if (VERBOSE) Log.d(TAG, "input surface: swap buffers"); 934 inputSurface.swapBuffers(); 935 if (VERBOSE) Log.d(TAG, "video encoder: notified of new frame"); 936 } 937 if ((videoDecoderOutputBufferInfo.flags 938 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 939 if (VERBOSE) Log.d(TAG, "video decoder: EOS"); 940 videoDecoderDone = true; 941 videoEncoder.signalEndOfInputStream(); 942 } 943 videoDecodedFrameCount++; 944 // We extracted a pending frame, let's try something else next. 945 break; 946 } 947 948 // Poll output frames from the audio decoder. 949 // Do not poll if we already have a pending buffer to feed to the encoder. 950 while (mCopyAudio && !audioDecoderDone && pendingAudioDecoderOutputBufferIndex == -1 951 && (encoderOutputAudioFormat == null || muxing)) { 952 int decoderOutputBufferIndex = 953 audioDecoder.dequeueOutputBuffer( 954 audioDecoderOutputBufferInfo, TIMEOUT_USEC); 955 if (decoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 956 if (VERBOSE) Log.d(TAG, "no audio decoder output buffer"); 957 break; 958 } 959 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 960 if (VERBOSE) Log.d(TAG, "audio decoder: output buffers changed"); 961 audioDecoderOutputBuffers = audioDecoder.getOutputBuffers(); 962 break; 963 } 964 if (decoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 965 decoderOutputAudioFormat = audioDecoder.getOutputFormat(); 966 if (VERBOSE) { 967 Log.d(TAG, "audio decoder: output format changed: " 968 + decoderOutputAudioFormat); 969 } 970 break; 971 } 972 if (VERBOSE) { 973 Log.d(TAG, "audio decoder: returned output buffer: " 974 + decoderOutputBufferIndex); 975 } 976 if (VERBOSE) { 977 Log.d(TAG, "audio decoder: returned buffer of size " 978 + audioDecoderOutputBufferInfo.size); 979 } 980 ByteBuffer decoderOutputBuffer = 981 audioDecoderOutputBuffers[decoderOutputBufferIndex]; 982 if ((audioDecoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 983 != 0) { 984 if (VERBOSE) Log.d(TAG, "audio decoder: codec config buffer"); 985 audioDecoder.releaseOutputBuffer(decoderOutputBufferIndex, false); 986 break; 987 } 988 if (VERBOSE) { 989 Log.d(TAG, "audio decoder: returned buffer for time " 990 + audioDecoderOutputBufferInfo.presentationTimeUs); 991 } 992 if (VERBOSE) { 993 Log.d(TAG, "audio decoder: output buffer is now pending: " 994 + pendingAudioDecoderOutputBufferIndex); 995 } 996 pendingAudioDecoderOutputBufferIndex = decoderOutputBufferIndex; 997 audioDecodedFrameCount++; 998 // We extracted a pending frame, let's try something else next. 999 break; 1000 } 1001 1002 // Feed the pending decoded audio buffer to the audio encoder. 1003 while (mCopyAudio && pendingAudioDecoderOutputBufferIndex != -1) { 1004 if (VERBOSE) { 1005 Log.d(TAG, "audio decoder: attempting to process pending buffer: " 1006 + pendingAudioDecoderOutputBufferIndex); 1007 } 1008 int encoderInputBufferIndex = audioEncoder.dequeueInputBuffer(TIMEOUT_USEC); 1009 if (encoderInputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1010 if (VERBOSE) Log.d(TAG, "no audio encoder input buffer"); 1011 break; 1012 } 1013 if (VERBOSE) { 1014 Log.d(TAG, "audio encoder: returned input buffer: " + encoderInputBufferIndex); 1015 } 1016 ByteBuffer encoderInputBuffer = audioEncoderInputBuffers[encoderInputBufferIndex]; 1017 int size = audioDecoderOutputBufferInfo.size; 1018 long presentationTime = audioDecoderOutputBufferInfo.presentationTimeUs; 1019 if (VERBOSE) { 1020 Log.d(TAG, "audio decoder: processing pending buffer: " 1021 + pendingAudioDecoderOutputBufferIndex); 1022 } 1023 if (VERBOSE) { 1024 Log.d(TAG, "audio decoder: pending buffer of size " + size); 1025 Log.d(TAG, "audio decoder: pending buffer for time " + presentationTime); 1026 } 1027 if (size >= 0) { 1028 ByteBuffer decoderOutputBuffer = 1029 audioDecoderOutputBuffers[pendingAudioDecoderOutputBufferIndex] 1030 .duplicate(); 1031 decoderOutputBuffer.position(audioDecoderOutputBufferInfo.offset); 1032 decoderOutputBuffer.limit(audioDecoderOutputBufferInfo.offset + size); 1033 encoderInputBuffer.position(0); 1034 encoderInputBuffer.put(decoderOutputBuffer); 1035 1036 audioEncoder.queueInputBuffer( 1037 encoderInputBufferIndex, 1038 0, 1039 size, 1040 presentationTime, 1041 audioDecoderOutputBufferInfo.flags); 1042 } 1043 audioDecoder.releaseOutputBuffer(pendingAudioDecoderOutputBufferIndex, false); 1044 pendingAudioDecoderOutputBufferIndex = -1; 1045 if ((audioDecoderOutputBufferInfo.flags 1046 & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { 1047 if (VERBOSE) Log.d(TAG, "audio decoder: EOS"); 1048 audioDecoderDone = true; 1049 } 1050 // We enqueued a pending frame, let's try something else next. 1051 break; 1052 } 1053 1054 // Poll frames from the video encoder and send them to the muxer. 1055 while (mCopyVideo && !videoEncoderDone 1056 && (encoderOutputVideoFormat == null || muxing)) { 1057 int encoderOutputBufferIndex = videoEncoder.dequeueOutputBuffer( 1058 videoEncoderOutputBufferInfo, TIMEOUT_USEC); 1059 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1060 if (VERBOSE) Log.d(TAG, "no video encoder output buffer"); 1061 break; 1062 } 1063 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1064 if (VERBOSE) Log.d(TAG, "video encoder: output buffers changed"); 1065 videoEncoderOutputBuffers = videoEncoder.getOutputBuffers(); 1066 break; 1067 } 1068 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1069 if (VERBOSE) Log.d(TAG, "video encoder: output format changed"); 1070 if (outputVideoTrack >= 0) { 1071 fail("video encoder changed its output format again?"); 1072 } 1073 encoderOutputVideoFormat = videoEncoder.getOutputFormat(); 1074 break; 1075 } 1076 assertTrue("should have added track before processing output", muxing); 1077 if (VERBOSE) { 1078 Log.d(TAG, "video encoder: returned output buffer: " 1079 + encoderOutputBufferIndex); 1080 Log.d(TAG, "video encoder: returned buffer of size " 1081 + videoEncoderOutputBufferInfo.size); 1082 } 1083 ByteBuffer encoderOutputBuffer = 1084 videoEncoderOutputBuffers[encoderOutputBufferIndex]; 1085 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1086 != 0) { 1087 if (VERBOSE) Log.d(TAG, "video encoder: codec config buffer"); 1088 // Simply ignore codec config buffers. 1089 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1090 break; 1091 } 1092 if (VERBOSE) { 1093 Log.d(TAG, "video encoder: returned buffer for time " 1094 + videoEncoderOutputBufferInfo.presentationTimeUs); 1095 } 1096 if (videoEncoderOutputBufferInfo.size != 0) { 1097 muxer.writeSampleData( 1098 outputVideoTrack, encoderOutputBuffer, videoEncoderOutputBufferInfo); 1099 } 1100 if ((videoEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1101 != 0) { 1102 if (VERBOSE) Log.d(TAG, "video encoder: EOS"); 1103 videoEncoderDone = true; 1104 } 1105 videoEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1106 videoEncodedFrameCount++; 1107 // We enqueued an encoded frame, let's try something else next. 1108 break; 1109 } 1110 1111 // Poll frames from the audio encoder and send them to the muxer. 1112 while (mCopyAudio && !audioEncoderDone 1113 && (encoderOutputAudioFormat == null || muxing)) { 1114 int encoderOutputBufferIndex = audioEncoder.dequeueOutputBuffer( 1115 audioEncoderOutputBufferInfo, TIMEOUT_USEC); 1116 if (encoderOutputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { 1117 if (VERBOSE) Log.d(TAG, "no audio encoder output buffer"); 1118 break; 1119 } 1120 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 1121 if (VERBOSE) Log.d(TAG, "audio encoder: output buffers changed"); 1122 audioEncoderOutputBuffers = audioEncoder.getOutputBuffers(); 1123 break; 1124 } 1125 if (encoderOutputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 1126 if (VERBOSE) Log.d(TAG, "audio encoder: output format changed"); 1127 if (outputAudioTrack >= 0) { 1128 fail("audio encoder changed its output format again?"); 1129 } 1130 1131 encoderOutputAudioFormat = audioEncoder.getOutputFormat(); 1132 break; 1133 } 1134 assertTrue("should have added track before processing output", muxing); 1135 if (VERBOSE) { 1136 Log.d(TAG, "audio encoder: returned output buffer: " 1137 + encoderOutputBufferIndex); 1138 Log.d(TAG, "audio encoder: returned buffer of size " 1139 + audioEncoderOutputBufferInfo.size); 1140 } 1141 ByteBuffer encoderOutputBuffer = 1142 audioEncoderOutputBuffers[encoderOutputBufferIndex]; 1143 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) 1144 != 0) { 1145 if (VERBOSE) Log.d(TAG, "audio encoder: codec config buffer"); 1146 // Simply ignore codec config buffers. 1147 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1148 break; 1149 } 1150 if (VERBOSE) { 1151 Log.d(TAG, "audio encoder: returned buffer for time " 1152 + audioEncoderOutputBufferInfo.presentationTimeUs); 1153 } 1154 if (audioEncoderOutputBufferInfo.size != 0) { 1155 muxer.writeSampleData( 1156 outputAudioTrack, encoderOutputBuffer, audioEncoderOutputBufferInfo); 1157 } 1158 if ((audioEncoderOutputBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) 1159 != 0) { 1160 if (VERBOSE) Log.d(TAG, "audio encoder: EOS"); 1161 audioEncoderDone = true; 1162 } 1163 audioEncoder.releaseOutputBuffer(encoderOutputBufferIndex, false); 1164 audioEncodedFrameCount++; 1165 // We enqueued an encoded frame, let's try something else next. 1166 break; 1167 } 1168 1169 if (!muxing 1170 && (!mCopyAudio || encoderOutputAudioFormat != null) 1171 && (!mCopyVideo || encoderOutputVideoFormat != null)) { 1172 if (mCopyVideo) { 1173 Log.d(TAG, "muxer: adding video track."); 1174 outputVideoTrack = muxer.addTrack(encoderOutputVideoFormat); 1175 } 1176 if (mCopyAudio) { 1177 Log.d(TAG, "muxer: adding audio track."); 1178 outputAudioTrack = muxer.addTrack(encoderOutputAudioFormat); 1179 } 1180 Log.d(TAG, "muxer: starting"); 1181 muxer.start(); 1182 muxing = true; 1183 } 1184 } 1185 1186 // Basic sanity checks. 1187 if (mCopyVideo) { 1188 assertEquals("encoded and decoded video frame counts should match", 1189 videoDecodedFrameCount, videoEncodedFrameCount); 1190 assertTrue("decoded frame count should be less than extracted frame count", 1191 videoDecodedFrameCount <= videoExtractedFrameCount); 1192 } 1193 if (mCopyAudio) { 1194 assertEquals("no frame should be pending", -1, pendingAudioDecoderOutputBufferIndex); 1195 } 1196 } 1197 isVideoFormat(MediaFormat format)1198 private static boolean isVideoFormat(MediaFormat format) { 1199 return getMimeTypeFor(format).startsWith("video/"); 1200 } 1201 isAudioFormat(MediaFormat format)1202 private static boolean isAudioFormat(MediaFormat format) { 1203 return getMimeTypeFor(format).startsWith("audio/"); 1204 } 1205 getMimeTypeFor(MediaFormat format)1206 private static String getMimeTypeFor(MediaFormat format) { 1207 return format.getString(MediaFormat.KEY_MIME); 1208 } 1209 1210 /** 1211 * Returns the first codec capable of encoding the specified MIME type, or null if no match was 1212 * found. 1213 */ selectCodec(String mimeType)1214 private static MediaCodecInfo selectCodec(String mimeType) { 1215 int numCodecs = MediaCodecList.getCodecCount(); 1216 for (int i = 0; i < numCodecs; i++) { 1217 MediaCodecInfo codecInfo = MediaCodecList.getCodecInfoAt(i); 1218 1219 if (!codecInfo.isEncoder()) { 1220 continue; 1221 } 1222 1223 String[] types = codecInfo.getSupportedTypes(); 1224 for (int j = 0; j < types.length; j++) { 1225 if (types[j].equalsIgnoreCase(mimeType)) { 1226 return codecInfo; 1227 } 1228 } 1229 } 1230 return null; 1231 } 1232 } 1233