1 /* 2 * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache 3 * License, Version 2.0 (the "License"); you may not use this file except in 4 * compliance with the License. You may obtain a copy of the License at 5 * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 * or agreed to in writing, software distributed under the License is 7 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 * KIND, either express or implied. See the License for the specific language 9 * governing permissions and limitations under the License. 10 */ 11 12 package android.hardware.camera2.cts; 13 14 import static android.hardware.camera2.cts.CameraTestUtils.*; 15 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*; 16 17 import android.cts.util.MediaUtils; 18 import android.graphics.ImageFormat; 19 import android.hardware.camera2.CameraCharacteristics; 20 import android.hardware.camera2.CameraCaptureSession; 21 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; 22 import android.hardware.camera2.CameraDevice; 23 import android.hardware.camera2.CaptureRequest; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.params.StreamConfigurationMap; 26 import android.util.Size; 27 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase; 28 import android.media.CamcorderProfile; 29 import android.media.MediaCodec; 30 import android.media.MediaCodecInfo; 31 import android.media.MediaCodecInfo.CodecCapabilities; 32 import android.media.MediaCodecInfo.CodecProfileLevel; 33 import android.media.Image; 34 import android.media.ImageReader; 35 import android.media.MediaCodecList; 36 import android.media.MediaExtractor; 37 import android.media.MediaFormat; 38 import android.media.MediaRecorder; 39 import android.os.Environment; 40 import android.os.SystemClock; 41 import android.test.suitebuilder.annotation.LargeTest; 42 import android.util.Log; 43 import android.util.Range; 44 import android.view.Surface; 45 46 import com.android.ex.camera2.blocking.BlockingSessionCallback; 47 48 import junit.framework.AssertionFailedError; 49 50 import java.io.File; 51 import java.util.ArrayList; 52 import java.util.Arrays; 53 import java.util.List; 54 import java.util.HashMap; 55 56 /** 57 * CameraDevice video recording use case tests by using MediaRecorder and 58 * MediaCodec. 59 */ 60 @LargeTest 61 public class RecordingTest extends Camera2SurfaceViewTestCase { 62 private static final String TAG = "RecordingTest"; 63 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 64 private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG); 65 private static final int RECORDING_DURATION_MS = 3000; 66 private static final float DURATION_MARGIN = 0.2f; 67 private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0; 68 private static final int BIT_RATE_1080P = 16000000; 69 private static final int BIT_RATE_MIN = 64000; 70 private static final int BIT_RATE_MAX = 40000000; 71 private static final int VIDEO_FRAME_RATE = 30; 72 private final String VIDEO_FILE_PATH = Environment.getExternalStorageDirectory().getPath(); 73 private static final int[] mCamcorderProfileList = { 74 CamcorderProfile.QUALITY_HIGH, 75 CamcorderProfile.QUALITY_2160P, 76 CamcorderProfile.QUALITY_1080P, 77 CamcorderProfile.QUALITY_720P, 78 CamcorderProfile.QUALITY_480P, 79 CamcorderProfile.QUALITY_CIF, 80 CamcorderProfile.QUALITY_QCIF, 81 CamcorderProfile.QUALITY_QVGA, 82 CamcorderProfile.QUALITY_LOW, 83 }; 84 private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5; 85 private static final int BURST_VIDEO_SNAPSHOT_NUM = 3; 86 private static final int SLOWMO_SLOW_FACTOR = 4; 87 private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4; 88 private List<Size> mSupportedVideoSizes; 89 private Surface mRecordingSurface; 90 private Surface mPersistentSurface; 91 private MediaRecorder mMediaRecorder; 92 private String mOutMediaFileName; 93 private int mVideoFrameRate; 94 private Size mVideoSize; 95 private long mRecordingStartTime; 96 97 @Override setUp()98 protected void setUp() throws Exception { 99 super.setUp(); 100 } 101 102 @Override tearDown()103 protected void tearDown() throws Exception { 104 super.tearDown(); 105 } 106 doBasicRecording(boolean useVideoStab)107 private void doBasicRecording(boolean useVideoStab) throws Exception { 108 for (int i = 0; i < mCameraIds.length; i++) { 109 try { 110 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 111 // Re-use the MediaRecorder object for the same camera device. 112 mMediaRecorder = new MediaRecorder(); 113 openDevice(mCameraIds[i]); 114 if (!mStaticInfo.isColorOutputSupported()) { 115 Log.i(TAG, "Camera " + mCameraIds[i] + 116 " does not support color outputs, skipping"); 117 continue; 118 } 119 120 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 121 Log.i(TAG, "Camera " + mCameraIds[i] + 122 " does not support video stabilization, skipping the stabilization" 123 + " test"); 124 continue; 125 } 126 127 initSupportedVideoSize(mCameraIds[i]); 128 129 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab); 130 } finally { 131 closeDevice(); 132 releaseRecorder(); 133 } 134 } 135 } 136 137 /** 138 * <p> 139 * Test basic video stabilitzation camera recording. 140 * </p> 141 * <p> 142 * This test covers the typical basic use case of camera recording with video 143 * stabilization is enabled, if video stabilization is supported. 144 * MediaRecorder is used to record the audio and video, CamcorderProfile is 145 * used to configure the MediaRecorder. It goes through the pre-defined 146 * CamcorderProfile list, test each profile configuration and validate the 147 * recorded video. Preview is set to the video size. 148 * </p> 149 */ testBasicVideoStabilizationRecording()150 public void testBasicVideoStabilizationRecording() throws Exception { 151 doBasicRecording(/*useVideoStab*/true); 152 } 153 154 /** 155 * <p> 156 * Test basic camera recording. 157 * </p> 158 * <p> 159 * This test covers the typical basic use case of camera recording. 160 * MediaRecorder is used to record the audio and video, CamcorderProfile is 161 * used to configure the MediaRecorder. It goes through the pre-defined 162 * CamcorderProfile list, test each profile configuration and validate the 163 * recorded video. Preview is set to the video size. 164 * </p> 165 */ testBasicRecording()166 public void testBasicRecording() throws Exception { 167 doBasicRecording(/*useVideoStab*/false); 168 } 169 170 /** 171 * <p> 172 * Test basic camera recording from a persistent input surface. 173 * </p> 174 * <p> 175 * This test is similar to testBasicRecording except that MediaRecorder records 176 * from a persistent input surface that's used across multiple recording sessions. 177 * </p> 178 */ testRecordingFromPersistentSurface()179 public void testRecordingFromPersistentSurface() throws Exception { 180 if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) { 181 return; // skipped 182 } 183 mPersistentSurface = MediaCodec.createPersistentInputSurface(); 184 assertNotNull("Failed to create persistent input surface!", mPersistentSurface); 185 186 try { 187 doBasicRecording(/*useVideoStab*/false); 188 } finally { 189 mPersistentSurface.release(); 190 mPersistentSurface = null; 191 } 192 } 193 194 /** 195 * <p> 196 * Test camera recording for all supported sizes by using MediaRecorder. 197 * </p> 198 * <p> 199 * This test covers camera recording for all supported sizes by camera. MediaRecorder 200 * is used to encode the video. Preview is set to the video size. Recorded videos are 201 * validated according to the recording configuration. 202 * </p> 203 */ testSupportedVideoSizes()204 public void testSupportedVideoSizes() throws Exception { 205 for (int i = 0; i < mCameraIds.length; i++) { 206 try { 207 Log.i(TAG, "Testing supported video size recording for camera " + mCameraIds[i]); 208 // Re-use the MediaRecorder object for the same camera device. 209 mMediaRecorder = new MediaRecorder(); 210 openDevice(mCameraIds[i]); 211 if (!mStaticInfo.isColorOutputSupported()) { 212 Log.i(TAG, "Camera " + mCameraIds[i] + 213 " does not support color outputs, skipping"); 214 continue; 215 } 216 initSupportedVideoSize(mCameraIds[i]); 217 218 recordingSizeTestByCamera(); 219 } finally { 220 closeDevice(); 221 releaseRecorder(); 222 } 223 } 224 } 225 226 /** 227 * Test different start/stop orders of Camera and Recorder. 228 * 229 * <p>The recording should be working fine for any kind of start/stop orders.</p> 230 */ testCameraRecorderOrdering()231 public void testCameraRecorderOrdering() { 232 // TODO: need implement 233 } 234 235 /** 236 * <p> 237 * Test camera recording for all supported sizes by using MediaCodec. 238 * </p> 239 * <p> 240 * This test covers video only recording for all supported sizes (camera and 241 * encoder). MediaCodec is used to encode the video. The recorded videos are 242 * validated according to the recording configuration. 243 * </p> 244 */ testMediaCodecRecording()245 public void testMediaCodecRecording() throws Exception { 246 // TODO. Need implement. 247 } 248 249 /** 250 * <p> 251 * Test video snapshot for each camera. 252 * </p> 253 * <p> 254 * This test covers video snapshot typical use case. The MediaRecorder is used to record the 255 * video for each available video size. The largest still capture size is selected to 256 * capture the JPEG image. The still capture images are validated according to the capture 257 * configuration. The timestamp of capture result before and after video snapshot is also 258 * checked to make sure no frame drop caused by video snapshot. 259 * </p> 260 */ testVideoSnapshot()261 public void testVideoSnapshot() throws Exception { 262 videoSnapshotHelper(/*burstTest*/false); 263 } 264 265 /** 266 * <p> 267 * Test burst video snapshot for each camera. 268 * </p> 269 * <p> 270 * This test covers burst video snapshot capture. The MediaRecorder is used to record the 271 * video for each available video size. The largest still capture size is selected to 272 * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be 273 * sent during the test. The still capture images are validated according to the capture 274 * configuration. 275 * </p> 276 */ testBurstVideoSnapshot()277 public void testBurstVideoSnapshot() throws Exception { 278 videoSnapshotHelper(/*burstTest*/true); 279 } 280 281 /** 282 * Test timelapse recording, where capture rate is slower than video (playback) frame rate. 283 */ testTimelapseRecording()284 public void testTimelapseRecording() throws Exception { 285 // TODO. Need implement. 286 } 287 testSlowMotionRecording()288 public void testSlowMotionRecording() throws Exception { 289 slowMotionRecording(); 290 } 291 testConstrainedHighSpeedRecording()292 public void testConstrainedHighSpeedRecording() throws Exception { 293 constrainedHighSpeedRecording(); 294 } 295 296 /** 297 * <p> 298 * Test recording framerate accuracy when switching from low FPS to high FPS. 299 * </p> 300 * <p> 301 * This test first record a video with profile of lowest framerate then record a video with 302 * profile of highest framerate. Make sure that the video framerate are still accurate. 303 * </p> 304 */ testRecordingFramerateLowToHigh()305 public void testRecordingFramerateLowToHigh() throws Exception { 306 for (int i = 0; i < mCameraIds.length; i++) { 307 try { 308 Log.i(TAG, "Testing basic recording for camera " + mCameraIds[i]); 309 // Re-use the MediaRecorder object for the same camera device. 310 mMediaRecorder = new MediaRecorder(); 311 openDevice(mCameraIds[i]); 312 if (!mStaticInfo.isColorOutputSupported()) { 313 Log.i(TAG, "Camera " + mCameraIds[i] + 314 " does not support color outputs, skipping"); 315 continue; 316 } 317 initSupportedVideoSize(mCameraIds[i]); 318 319 int minFpsProfileId = -1, minFps = 1000; 320 int maxFpsProfileId = -1, maxFps = 0; 321 int cameraId = Integer.valueOf(mCamera.getId()); 322 323 for (int profileId : mCamcorderProfileList) { 324 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 325 continue; 326 } 327 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 328 if (profile.videoFrameRate < minFps) { 329 minFpsProfileId = profileId; 330 minFps = profile.videoFrameRate; 331 } 332 if (profile.videoFrameRate > maxFps) { 333 maxFpsProfileId = profileId; 334 maxFps = profile.videoFrameRate; 335 } 336 } 337 338 int camcorderProfileList[] = new int[] {minFpsProfileId, maxFpsProfileId}; 339 basicRecordingTestByCamera(camcorderProfileList, /*useVideoStab*/false); 340 } finally { 341 closeDevice(); 342 releaseRecorder(); 343 } 344 } 345 } 346 347 /** 348 * Test slow motion recording where capture rate (camera output) is different with 349 * video (playback) frame rate for each camera if high speed recording is supported 350 * by both camera and encoder. 351 * 352 * <p> 353 * Normal recording use cases make the capture rate (camera output frame 354 * rate) the same as the video (playback) frame rate. This guarantees that 355 * the motions in the scene play at the normal speed. If the capture rate is 356 * faster than video frame rate, for a given time duration, more number of 357 * frames are captured than it can be played in the same time duration. This 358 * generates "slow motion" effect during playback. 359 * </p> 360 */ slowMotionRecording()361 private void slowMotionRecording() throws Exception { 362 for (String id : mCameraIds) { 363 try { 364 Log.i(TAG, "Testing slow motion recording for camera " + id); 365 // Re-use the MediaRecorder object for the same camera device. 366 mMediaRecorder = new MediaRecorder(); 367 openDevice(id); 368 if (!mStaticInfo.isColorOutputSupported()) { 369 Log.i(TAG, "Camera " + id + 370 " does not support color outputs, skipping"); 371 continue; 372 } 373 if (!mStaticInfo.isHighSpeedVideoSupported()) { 374 continue; 375 } 376 377 StreamConfigurationMap config = 378 mStaticInfo.getValueFromKeyNonNull( 379 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 380 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 381 for (Size size : highSpeedVideoSizes) { 382 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size); 383 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " + 384 "size " + size, fpsRange); 385 if (fpsRange == null) { 386 continue; 387 } 388 389 int captureRate = fpsRange.getLower(); 390 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR; 391 // Skip the test if the highest recording FPS supported by CamcorderProfile 392 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 393 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 394 + " is not supported by CamcorderProfile"); 395 continue; 396 } 397 398 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video.mp4"; 399 if (DEBUG_DUMP) { 400 mOutMediaFileName = VIDEO_FILE_PATH + "/test_slowMo_video_" + id + "_" 401 + size.toString() + ".mp4"; 402 } 403 404 prepareRecording(size, videoFramerate, captureRate); 405 406 // prepare preview surface by using video size. 407 updatePreviewSurfaceWithVideo(size, captureRate); 408 409 // Start recording 410 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 411 startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate, 412 fpsRange, resultListener, /*useHighSpeedSession*/false); 413 414 // Record certain duration. 415 SystemClock.sleep(RECORDING_DURATION_MS); 416 417 // Stop recording and preview 418 stopRecording(/*useMediaRecorder*/true); 419 // Convert number of frames camera produced into the duration in unit of ms. 420 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 421 videoFramerate); 422 423 // Validation. 424 validateRecording(size, durationMs); 425 } 426 427 } finally { 428 closeDevice(); 429 releaseRecorder(); 430 } 431 } 432 } 433 constrainedHighSpeedRecording()434 private void constrainedHighSpeedRecording() throws Exception { 435 for (String id : mCameraIds) { 436 try { 437 Log.i(TAG, "Testing constrained high speed recording for camera " + id); 438 // Re-use the MediaRecorder object for the same camera device. 439 mMediaRecorder = new MediaRecorder(); 440 openDevice(id); 441 442 if (!mStaticInfo.isConstrainedHighSpeedVideoSupported()) { 443 Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping."); 444 continue; 445 } 446 447 StreamConfigurationMap config = 448 mStaticInfo.getValueFromKeyNonNull( 449 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 450 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 451 for (Size size : highSpeedVideoSizes) { 452 List<Range<Integer>> fixedFpsRanges = 453 getHighSpeedFixedFpsRangeForSize(config, size); 454 mCollector.expectTrue("Unable to find the fixed frame rate fps range for " + 455 "size " + size, fixedFpsRanges.size() > 0); 456 // Test recording for each FPS range 457 for (Range<Integer> fpsRange : fixedFpsRanges) { 458 int captureRate = fpsRange.getLower(); 459 final int VIDEO_FRAME_RATE = 30; 460 // Skip the test if the highest recording FPS supported by CamcorderProfile 461 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 462 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 463 + " is not supported by CamcorderProfile"); 464 continue; 465 } 466 467 mOutMediaFileName = VIDEO_FILE_PATH + "/test_cslowMo_video_" + captureRate + 468 "fps_" + id + "_" + size.toString() + ".mp4"; 469 470 prepareRecording(size, VIDEO_FRAME_RATE, captureRate); 471 472 // prepare preview surface by using video size. 473 updatePreviewSurfaceWithVideo(size, captureRate); 474 475 // Start recording 476 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 477 startSlowMotionRecording(/*useMediaRecorder*/true, VIDEO_FRAME_RATE, 478 captureRate, fpsRange, resultListener, 479 /*useHighSpeedSession*/true); 480 481 // Record certain duration. 482 SystemClock.sleep(RECORDING_DURATION_MS); 483 484 // Stop recording and preview 485 stopRecording(/*useMediaRecorder*/true); 486 // Convert number of frames camera produced into the duration in unit of ms. 487 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 488 VIDEO_FRAME_RATE); 489 490 // Validation. 491 validateRecording(size, durationMs); 492 } 493 } 494 495 } finally { 496 closeDevice(); 497 releaseRecorder(); 498 } 499 } 500 } 501 502 /** 503 * Get high speed FPS from CamcorderProfiles for a given size. 504 * 505 * @param size The size used to search the CamcorderProfiles for the FPS. 506 * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles. 507 */ getFpsFromHighSpeedProfileForSize(Size size)508 private int getFpsFromHighSpeedProfileForSize(Size size) { 509 for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P; 510 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) { 511 if (CamcorderProfile.hasProfile(quality)) { 512 CamcorderProfile profile = CamcorderProfile.get(quality); 513 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){ 514 return profile.videoFrameRate; 515 } 516 } 517 } 518 519 return 0; 520 } 521 getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)522 private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 523 Size size) { 524 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 525 Range<Integer> maxRange = availableFpsRanges[0]; 526 boolean foundRange = false; 527 for (Range<Integer> range : availableFpsRanges) { 528 if (range.getLower().equals(range.getUpper()) && range.getLower() >= maxRange.getLower()) { 529 foundRange = true; 530 maxRange = range; 531 } 532 } 533 534 if (!foundRange) { 535 return null; 536 } 537 return maxRange; 538 } 539 getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)540 private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 541 Size size) { 542 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 543 List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>(); 544 for (Range<Integer> range : availableFpsRanges) { 545 if (range.getLower().equals(range.getUpper())) { 546 fixedRanges.add(range); 547 } 548 } 549 return fixedRanges; 550 } 551 startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, int captureRate, Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession)552 private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, 553 int captureRate, Range<Integer> fpsRange, 554 CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession) throws Exception { 555 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 556 assertTrue("Both preview and recording surfaces should be valid", 557 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 558 outputSurfaces.add(mPreviewSurface); 559 outputSurfaces.add(mRecordingSurface); 560 // Video snapshot surface 561 if (mReaderSurface != null) { 562 outputSurfaces.add(mReaderSurface); 563 } 564 mSessionListener = new BlockingSessionCallback(); 565 mSession = configureCameraSession(mCamera, outputSurfaces, useHighSpeedSession, 566 mSessionListener, mHandler); 567 568 // Create slow motion request list 569 List<CaptureRequest> slowMoRequests = null; 570 if (useHighSpeedSession) { 571 CaptureRequest.Builder requestBuilder = 572 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 573 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 574 requestBuilder.addTarget(mPreviewSurface); 575 requestBuilder.addTarget(mRecordingSurface); 576 slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession). 577 createHighSpeedRequestList(requestBuilder.build()); 578 } else { 579 CaptureRequest.Builder recordingRequestBuilder = 580 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 581 recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE, 582 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 583 recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 584 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 585 586 CaptureRequest.Builder recordingOnlyBuilder = 587 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 588 recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE, 589 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 590 recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 591 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 592 int slowMotionFactor = captureRate / videoFrameRate; 593 594 // Make sure camera output frame rate is set to correct value. 595 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 596 recordingRequestBuilder.addTarget(mRecordingSurface); 597 recordingRequestBuilder.addTarget(mPreviewSurface); 598 recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 599 recordingOnlyBuilder.addTarget(mRecordingSurface); 600 601 slowMoRequests = new ArrayList<CaptureRequest>(); 602 slowMoRequests.add(recordingRequestBuilder.build());// Preview + recording. 603 604 for (int i = 0; i < slowMotionFactor - 1; i++) { 605 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only. 606 } 607 } 608 609 mSession.setRepeatingBurst(slowMoRequests, listener, mHandler); 610 611 if (useMediaRecorder) { 612 mMediaRecorder.start(); 613 } else { 614 // TODO: need implement MediaCodec path. 615 } 616 617 } 618 619 /** 620 * Test camera recording by using each available CamcorderProfile for a 621 * given camera. preview size is set to the video size. 622 */ basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)623 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab) 624 throws Exception { 625 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 626 List<Range<Integer> > fpsRanges = Arrays.asList( 627 mStaticInfo.getAeAvailableTargetFpsRangesChecked()); 628 int cameraId = Integer.valueOf(mCamera.getId()); 629 int maxVideoFrameRate = -1; 630 for (int profileId : camcorderProfileList) { 631 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 632 continue; 633 } 634 635 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 636 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 637 Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate); 638 if (maxVideoFrameRate < profile.videoFrameRate) { 639 maxVideoFrameRate = profile.videoFrameRate; 640 } 641 642 if (allowedUnsupported(cameraId, profileId)) { 643 continue; 644 } 645 646 if (mStaticInfo.isHardwareLevelLegacy() && 647 (videoSz.getWidth() > maxPreviewSize.getWidth() || 648 videoSz.getHeight() > maxPreviewSize.getHeight())) { 649 // Skip. Legacy mode can only do recording up to max preview size 650 continue; 651 } 652 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 653 " must be one of the camera device supported video size!", 654 mSupportedVideoSizes.contains(videoSz)); 655 assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId + 656 ") must be one of the camera device available FPS range!", 657 fpsRanges.contains(fpsRange)); 658 659 if (VERBOSE) { 660 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 661 } 662 663 // Configure preview and recording surfaces. 664 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 665 if (DEBUG_DUMP) { 666 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 667 + videoSz.toString() + ".mp4"; 668 } 669 670 prepareRecordingWithProfile(profile); 671 672 // prepare preview surface by using video size. 673 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 674 675 // Start recording 676 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 677 startRecording(/* useMediaRecorder */true, resultListener, useVideoStab); 678 679 // Record certain duration. 680 SystemClock.sleep(RECORDING_DURATION_MS); 681 682 // Stop recording and preview 683 stopRecording(/* useMediaRecorder */true); 684 // Convert number of frames camera produced into the duration in unit of ms. 685 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 686 profile.videoFrameRate); 687 688 if (VERBOSE) { 689 Log.v(TAG, "video frame rate: " + profile.videoFrameRate + 690 ", num of frames produced: " + resultListener.getTotalNumFrames()); 691 } 692 693 // Validation. 694 validateRecording(videoSz, durationMs); 695 } 696 if (maxVideoFrameRate != -1) { 697 // At least one CamcorderProfile is present, check FPS 698 assertTrue("At least one CamcorderProfile must support >= 24 FPS", 699 maxVideoFrameRate >= 24); 700 } 701 } 702 703 /** 704 * Test camera recording for each supported video size by camera, preview 705 * size is set to the video size. 706 */ recordingSizeTestByCamera()707 private void recordingSizeTestByCamera() throws Exception { 708 for (Size sz : mSupportedVideoSizes) { 709 if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) { 710 continue; 711 } 712 713 if (VERBOSE) { 714 Log.v(TAG, "Testing camera recording with video size " + sz.toString()); 715 } 716 717 // Configure preview and recording surfaces. 718 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 719 if (DEBUG_DUMP) { 720 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + mCamera.getId() + "_" 721 + sz.toString() + ".mp4"; 722 } 723 724 // Use AVC and AAC a/v compression format. 725 prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE); 726 727 // prepare preview surface by using video size. 728 updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE); 729 730 // Start recording 731 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 732 startRecording(/* useMediaRecorder */true, resultListener, /*useVideoStab*/false); 733 734 // Record certain duration. 735 SystemClock.sleep(RECORDING_DURATION_MS); 736 737 // Stop recording and preview 738 stopRecording(/* useMediaRecorder */true); 739 // Convert number of frames camera produced into the duration in unit of ms. 740 int durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 741 VIDEO_FRAME_RATE); 742 743 // Validation. 744 validateRecording(sz, durationMs); 745 } 746 } 747 748 /** 749 * Initialize the supported video sizes. 750 */ initSupportedVideoSize(String cameraId)751 private void initSupportedVideoSize(String cameraId) throws Exception { 752 Size maxVideoSize = SIZE_BOUND_1080P; 753 if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2160P)) { 754 maxVideoSize = SIZE_BOUND_2160P; 755 } 756 mSupportedVideoSizes = 757 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize); 758 } 759 760 /** 761 * Simple wrapper to wrap normal/burst video snapshot tests 762 */ videoSnapshotHelper(boolean burstTest)763 private void videoSnapshotHelper(boolean burstTest) throws Exception { 764 for (String id : mCameraIds) { 765 try { 766 Log.i(TAG, "Testing video snapshot for camera " + id); 767 // Re-use the MediaRecorder object for the same camera device. 768 mMediaRecorder = new MediaRecorder(); 769 770 openDevice(id); 771 772 if (!mStaticInfo.isColorOutputSupported()) { 773 Log.i(TAG, "Camera " + id + 774 " does not support color outputs, skipping"); 775 continue; 776 } 777 778 initSupportedVideoSize(id); 779 780 videoSnapshotTestByCamera(burstTest); 781 } finally { 782 closeDevice(); 783 releaseRecorder(); 784 } 785 } 786 } 787 788 /** 789 * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported. 790 * 791 * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p> 792 * 793 * @param profileId a {@link CamcorderProfile} ID to check. 794 * @return {@code true} if supported. 795 */ allowedUnsupported(int cameraId, int profileId)796 private boolean allowedUnsupported(int cameraId, int profileId) { 797 if (!mStaticInfo.isHardwareLevelLegacy()) { 798 return false; 799 } 800 801 switch(profileId) { 802 case CamcorderProfile.QUALITY_2160P: 803 case CamcorderProfile.QUALITY_1080P: 804 case CamcorderProfile.QUALITY_HIGH: 805 return !CamcorderProfile.hasProfile(cameraId, profileId) || 806 CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080; 807 } 808 return false; 809 } 810 811 /** 812 * Test video snapshot for each available CamcorderProfile for a given camera. 813 * 814 * <p> 815 * Preview size is set to the video size. For the burst test, frame drop and jittering 816 * is not checked. 817 * </p> 818 * 819 * @param burstTest Perform burst capture or single capture. For burst capture 820 * {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent. 821 */ videoSnapshotTestByCamera(boolean burstTest)822 private void videoSnapshotTestByCamera(boolean burstTest) 823 throws Exception { 824 final int NUM_SINGLE_SHOT_TEST = 5; 825 final int FRAMEDROP_TOLERANCE = 8; 826 final int FRAME_SIZE_15M = 15000000; 827 final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f; 828 int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE; 829 830 for (int profileId : mCamcorderProfileList) { 831 int cameraId = Integer.valueOf(mCamera.getId()); 832 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 833 allowedUnsupported(cameraId, profileId)) { 834 continue; 835 } 836 837 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 838 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 839 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 840 841 if (mStaticInfo.isHardwareLevelLegacy() && 842 (videoSz.getWidth() > maxPreviewSize.getWidth() || 843 videoSz.getHeight() > maxPreviewSize.getHeight())) { 844 // Skip. Legacy mode can only do recording up to max preview size 845 continue; 846 } 847 848 if (!mSupportedVideoSizes.contains(videoSz)) { 849 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " + 850 profileId + " must be one of the camera device supported video size!"); 851 continue; 852 } 853 854 // For LEGACY, find closest supported smaller or equal JPEG size to the current video 855 // size; if no size is smaller than the video, pick the smallest JPEG size. The assert 856 // for video size above guarantees that for LIMITED or FULL, we select videoSz here. 857 // Also check for minFrameDuration here to make sure jpeg stream won't slow down 858 // video capture 859 Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1); 860 // Allow a bit tolerance so we don't fail for a few nano seconds of difference 861 final float FRAME_DURATION_TOLERANCE = 0.01f; 862 long videoFrameDuration = (long) (1e9 / profile.videoFrameRate * 863 (1.0 + FRAME_DURATION_TOLERANCE)); 864 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 865 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG); 866 for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) { 867 Size candidateSize = mOrderedStillSizes.get(i); 868 if (mStaticInfo.isHardwareLevelLegacy()) { 869 // Legacy level doesn't report min frame duration 870 if (candidateSize.getWidth() <= videoSz.getWidth() && 871 candidateSize.getHeight() <= videoSz.getHeight()) { 872 videoSnapshotSz = candidateSize; 873 } 874 } else { 875 Long jpegFrameDuration = minFrameDurationMap.get(candidateSize); 876 assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize, 877 jpegFrameDuration != null); 878 if (candidateSize.getWidth() <= videoSz.getWidth() && 879 candidateSize.getHeight() <= videoSz.getHeight() && 880 jpegFrameDuration <= videoFrameDuration) { 881 videoSnapshotSz = candidateSize; 882 } 883 } 884 } 885 Size defaultvideoSnapshotSz = videoSnapshotSz; 886 887 /** 888 * Only test full res snapshot when below conditions are all true. 889 * 1. Camera is at least a LIMITED device. 890 * 2. video size is up to max preview size, which will be bounded by 1080p. 891 * 3. Full resolution jpeg stream can keep up to video stream speed. 892 * When full res jpeg stream cannot keep up to video stream speed, search 893 * the largest jpeg size that can susptain video speed instead. 894 */ 895 if (mStaticInfo.isHardwareLevelAtLeastLimited() && 896 videoSz.getWidth() <= maxPreviewSize.getWidth() && 897 videoSz.getHeight() <= maxPreviewSize.getHeight()) { 898 for (Size jpegSize : mOrderedStillSizes) { 899 Long jpegFrameDuration = minFrameDurationMap.get(jpegSize); 900 assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize, 901 jpegFrameDuration != null); 902 if (jpegFrameDuration <= videoFrameDuration) { 903 videoSnapshotSz = jpegSize; 904 break; 905 } 906 if (jpegSize.equals(videoSz)) { 907 throw new AssertionFailedError( 908 "Cannot find adequate video snapshot size for video size" + 909 videoSz); 910 } 911 } 912 } 913 914 Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz + 915 " for video size " + videoSz); 916 if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M) 917 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR); 918 919 createImageReader( 920 videoSnapshotSz, ImageFormat.JPEG, 921 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 922 923 // Full or better devices should support whatever video snapshot size calculated above. 924 // Limited devices may only be able to support the default one. 925 if (mStaticInfo.isHardwareLevelLimited()) { 926 List<Surface> outputs = new ArrayList<Surface>(); 927 outputs.add(mPreviewSurface); 928 outputs.add(mRecordingSurface); 929 outputs.add(mReaderSurface); 930 boolean isSupported = isStreamConfigurationSupported( 931 mCamera, outputs, mSessionListener, mHandler); 932 if (!isSupported) { 933 videoSnapshotSz = defaultvideoSnapshotSz; 934 createImageReader( 935 videoSnapshotSz, ImageFormat.JPEG, 936 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 937 } 938 } 939 940 if (VERBOSE) { 941 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 942 } 943 944 // Configure preview and recording surfaces. 945 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video.mp4"; 946 if (DEBUG_DUMP) { 947 mOutMediaFileName = VIDEO_FILE_PATH + "/test_video_" + cameraId + "_" 948 + videoSz.toString() + ".mp4"; 949 } 950 951 int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST; 952 int totalDroppedFrames = 0; 953 954 for (int numTested = 0; numTested < numTestIterations; numTested++) { 955 prepareRecordingWithProfile(profile); 956 957 // prepare video snapshot 958 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 959 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(); 960 CaptureRequest.Builder videoSnapshotRequestBuilder = 961 mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ? 962 CameraDevice.TEMPLATE_RECORD : 963 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT); 964 965 // prepare preview surface by using video size. 966 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 967 968 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener); 969 CaptureRequest request = videoSnapshotRequestBuilder.build(); 970 971 // Start recording 972 startRecording(/* useMediaRecorder */true, resultListener, 973 /*useVideoStab*/mStaticInfo.isVideoStabilizationSupported()); 974 long startTime = SystemClock.elapsedRealtime(); 975 976 // Record certain duration. 977 SystemClock.sleep(RECORDING_DURATION_MS / 2); 978 979 // take video snapshot 980 if (burstTest) { 981 List<CaptureRequest> requests = 982 new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM); 983 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 984 requests.add(request); 985 } 986 mSession.captureBurst(requests, resultListener, mHandler); 987 } else { 988 mSession.capture(request, resultListener, mHandler); 989 } 990 991 // make sure recording is still going after video snapshot 992 SystemClock.sleep(RECORDING_DURATION_MS / 2); 993 994 // Stop recording and preview 995 int durationMs = stopRecording(/* useMediaRecorder */true); 996 // For non-burst test, use number of frames to also double check video frame rate. 997 // Burst video snapshot is allowed to cause frame rate drop, so do not use number 998 // of frames to estimate duration 999 if (!burstTest) { 1000 durationMs = (int) (resultListener.getTotalNumFrames() * 1000.0f / 1001 profile.videoFrameRate); 1002 } 1003 1004 // Validation recorded video 1005 validateRecording(videoSz, durationMs); 1006 1007 if (burstTest) { 1008 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 1009 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 1010 validateVideoSnapshotCapture(image, videoSnapshotSz); 1011 image.close(); 1012 } 1013 } else { 1014 // validate video snapshot image 1015 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 1016 validateVideoSnapshotCapture(image, videoSnapshotSz); 1017 1018 // validate if there is framedrop around video snapshot 1019 totalDroppedFrames += validateFrameDropAroundVideoSnapshot( 1020 resultListener, image.getTimestamp()); 1021 1022 //TODO: validate jittering. Should move to PTS 1023 //validateJittering(resultListener); 1024 1025 image.close(); 1026 } 1027 } 1028 1029 if (!burstTest) { 1030 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " + 1031 "detected in %d trials is %d frames.", cameraId, videoSz.toString(), 1032 numTestIterations, totalDroppedFrames)); 1033 mCollector.expectLessOrEqual( 1034 String.format( 1035 "Camera %d Video size %s: Number of dropped frames %d must not" 1036 + " be larger than %d", 1037 cameraId, videoSz.toString(), totalDroppedFrames, 1038 kFrameDrop_Tolerence), 1039 kFrameDrop_Tolerence, totalDroppedFrames); 1040 } 1041 closeImageReader(); 1042 } 1043 } 1044 1045 /** 1046 * Configure video snapshot request according to the still capture size 1047 */ prepareVideoSnapshot( CaptureRequest.Builder requestBuilder, ImageReader.OnImageAvailableListener imageListener)1048 private void prepareVideoSnapshot( 1049 CaptureRequest.Builder requestBuilder, 1050 ImageReader.OnImageAvailableListener imageListener) 1051 throws Exception { 1052 mReader.setOnImageAvailableListener(imageListener, mHandler); 1053 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1054 requestBuilder.addTarget(mRecordingSurface); 1055 assertNotNull("Preview surface must be non-null!", mPreviewSurface); 1056 requestBuilder.addTarget(mPreviewSurface); 1057 assertNotNull("Reader surface must be non-null!", mReaderSurface); 1058 requestBuilder.addTarget(mReaderSurface); 1059 } 1060 1061 /** 1062 * Update preview size with video size. 1063 * 1064 * <p>Preview size will be capped with max preview size.</p> 1065 * 1066 * @param videoSize The video size used for preview. 1067 * @param videoFrameRate The video frame rate 1068 * 1069 */ updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate)1070 private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) { 1071 if (mOrderedPreviewSizes == null) { 1072 throw new IllegalStateException("supported preview size list is not initialized yet"); 1073 } 1074 final float FRAME_DURATION_TOLERANCE = 0.01f; 1075 long videoFrameDuration = (long) (1e9 / videoFrameRate * 1076 (1.0 + FRAME_DURATION_TOLERANCE)); 1077 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 1078 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE); 1079 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 1080 Size previewSize = null; 1081 if (videoSize.getWidth() > maxPreviewSize.getWidth() || 1082 videoSize.getHeight() > maxPreviewSize.getHeight()) { 1083 for (Size s : mOrderedPreviewSizes) { 1084 Long frameDuration = minFrameDurationMap.get(s); 1085 if (mStaticInfo.isHardwareLevelLegacy()) { 1086 // Legacy doesn't report min frame duration 1087 frameDuration = new Long(0); 1088 } 1089 assertTrue("Cannot find minimum frame duration for private size" + s, 1090 frameDuration != null); 1091 if (frameDuration <= videoFrameDuration && 1092 s.getWidth() <= videoSize.getWidth() && 1093 s.getHeight() <= videoSize.getHeight()) { 1094 Log.w(TAG, "Overwrite preview size from " + videoSize.toString() + 1095 " to " + s.toString()); 1096 previewSize = s; 1097 break; 1098 // If all preview size doesn't work then we fallback to video size 1099 } 1100 } 1101 } 1102 if (previewSize == null) { 1103 previewSize = videoSize; 1104 } 1105 updatePreviewSurface(previewSize); 1106 } 1107 1108 /** 1109 * Configure MediaRecorder recording session with CamcorderProfile, prepare 1110 * the recording surface. 1111 */ prepareRecordingWithProfile(CamcorderProfile profile)1112 private void prepareRecordingWithProfile(CamcorderProfile profile) 1113 throws Exception { 1114 // Prepare MediaRecorder. 1115 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 1116 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 1117 mMediaRecorder.setProfile(profile); 1118 mMediaRecorder.setOutputFile(mOutMediaFileName); 1119 if (mPersistentSurface != null) { 1120 mMediaRecorder.setInputSurface(mPersistentSurface); 1121 mRecordingSurface = mPersistentSurface; 1122 } 1123 mMediaRecorder.prepare(); 1124 if (mPersistentSurface == null) { 1125 mRecordingSurface = mMediaRecorder.getSurface(); 1126 } 1127 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1128 mVideoFrameRate = profile.videoFrameRate; 1129 mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 1130 } 1131 1132 /** 1133 * Configure MediaRecorder recording session with CamcorderProfile, prepare 1134 * the recording surface. Use AVC for video compression, AAC for audio compression. 1135 * Both are required for android devices by android CDD. 1136 */ prepareRecording(Size sz, int videoFrameRate, int captureRate)1137 private void prepareRecording(Size sz, int videoFrameRate, int captureRate) 1138 throws Exception { 1139 // Prepare MediaRecorder. 1140 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 1141 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 1142 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); 1143 mMediaRecorder.setOutputFile(mOutMediaFileName); 1144 mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz)); 1145 mMediaRecorder.setVideoFrameRate(videoFrameRate); 1146 mMediaRecorder.setCaptureRate(captureRate); 1147 mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight()); 1148 mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264); 1149 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); 1150 if (mPersistentSurface != null) { 1151 mMediaRecorder.setInputSurface(mPersistentSurface); 1152 mRecordingSurface = mPersistentSurface; 1153 } 1154 mMediaRecorder.prepare(); 1155 if (mPersistentSurface == null) { 1156 mRecordingSurface = mMediaRecorder.getSurface(); 1157 } 1158 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 1159 mVideoFrameRate = videoFrameRate; 1160 mVideoSize = sz; 1161 } 1162 startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab)1163 private void startRecording(boolean useMediaRecorder, 1164 CameraCaptureSession.CaptureCallback listener, boolean useVideoStab) throws Exception { 1165 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 1166 throw new IllegalArgumentException("Video stabilization is not supported"); 1167 } 1168 1169 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 1170 assertTrue("Both preview and recording surfaces should be valid", 1171 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 1172 outputSurfaces.add(mPreviewSurface); 1173 outputSurfaces.add(mRecordingSurface); 1174 // Video snapshot surface 1175 if (mReaderSurface != null) { 1176 outputSurfaces.add(mReaderSurface); 1177 } 1178 mSessionListener = new BlockingSessionCallback(); 1179 mSession = configureCameraSession(mCamera, outputSurfaces, mSessionListener, mHandler); 1180 1181 CaptureRequest.Builder recordingRequestBuilder = 1182 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1183 // Make sure camera output frame rate is set to correct value. 1184 Range<Integer> fpsRange = Range.create(mVideoFrameRate, mVideoFrameRate); 1185 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1186 if (useVideoStab) { 1187 recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 1188 CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON); 1189 } 1190 recordingRequestBuilder.addTarget(mRecordingSurface); 1191 recordingRequestBuilder.addTarget(mPreviewSurface); 1192 mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler); 1193 1194 if (useMediaRecorder) { 1195 mMediaRecorder.start(); 1196 } else { 1197 // TODO: need implement MediaCodec path. 1198 } 1199 mRecordingStartTime = SystemClock.elapsedRealtime(); 1200 } 1201 stopCameraStreaming()1202 private void stopCameraStreaming() throws Exception { 1203 if (VERBOSE) { 1204 Log.v(TAG, "Stopping camera streaming and waiting for idle"); 1205 } 1206 // Stop repeating, wait for captures to complete, and disconnect from 1207 // surfaces 1208 mSession.close(); 1209 mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 1210 } 1211 1212 // Stop recording and return the estimated video duration in milliseconds. stopRecording(boolean useMediaRecorder)1213 private int stopRecording(boolean useMediaRecorder) throws Exception { 1214 long stopRecordingTime = SystemClock.elapsedRealtime(); 1215 if (useMediaRecorder) { 1216 stopCameraStreaming(); 1217 1218 mMediaRecorder.stop(); 1219 // Can reuse the MediaRecorder object after reset. 1220 mMediaRecorder.reset(); 1221 } else { 1222 // TODO: need implement MediaCodec path. 1223 } 1224 if (mPersistentSurface == null && mRecordingSurface != null) { 1225 mRecordingSurface.release(); 1226 mRecordingSurface = null; 1227 } 1228 return (int) (stopRecordingTime - mRecordingStartTime); 1229 } 1230 releaseRecorder()1231 private void releaseRecorder() { 1232 if (mMediaRecorder != null) { 1233 mMediaRecorder.release(); 1234 mMediaRecorder = null; 1235 } 1236 } 1237 validateRecording(Size sz, int expectedDurationMs)1238 private void validateRecording(Size sz, int expectedDurationMs) throws Exception { 1239 File outFile = new File(mOutMediaFileName); 1240 assertTrue("No video is recorded", outFile.exists()); 1241 1242 MediaExtractor extractor = new MediaExtractor(); 1243 try { 1244 extractor.setDataSource(mOutMediaFileName); 1245 long durationUs = 0; 1246 int width = -1, height = -1; 1247 int numTracks = extractor.getTrackCount(); 1248 final String VIDEO_MIME_TYPE = "video"; 1249 for (int i = 0; i < numTracks; i++) { 1250 MediaFormat format = extractor.getTrackFormat(i); 1251 String mime = format.getString(MediaFormat.KEY_MIME); 1252 if (mime.contains(VIDEO_MIME_TYPE)) { 1253 Log.i(TAG, "video format is: " + format.toString()); 1254 durationUs = format.getLong(MediaFormat.KEY_DURATION); 1255 width = format.getInteger(MediaFormat.KEY_WIDTH); 1256 height = format.getInteger(MediaFormat.KEY_HEIGHT); 1257 break; 1258 } 1259 } 1260 Size videoSz = new Size(width, height); 1261 assertTrue("Video size doesn't match, expected " + sz.toString() + 1262 " got " + videoSz.toString(), videoSz.equals(sz)); 1263 int duration = (int) (durationUs / 1000); 1264 if (VERBOSE) { 1265 Log.v(TAG, String.format("Video duration: recorded %dms, expected %dms", 1266 duration, expectedDurationMs)); 1267 } 1268 1269 // TODO: Don't skip this for video snapshot 1270 if (!mStaticInfo.isHardwareLevelLegacy()) { 1271 assertTrue(String.format( 1272 "Camera %s: Video duration doesn't match: recorded %dms, expected %dms.", 1273 mCamera.getId(), duration, expectedDurationMs), 1274 Math.abs(duration - expectedDurationMs) < 1275 DURATION_MARGIN * expectedDurationMs); 1276 } 1277 } finally { 1278 extractor.release(); 1279 if (!DEBUG_DUMP) { 1280 outFile.delete(); 1281 } 1282 } 1283 } 1284 1285 /** 1286 * Validate video snapshot capture image object sanity and test. 1287 * 1288 * <p> Check for size, format and jpeg decoding</p> 1289 * 1290 * @param image The JPEG image to be verified. 1291 * @param size The JPEG capture size to be verified against. 1292 */ 1293 private void validateVideoSnapshotCapture(Image image, Size size) { 1294 CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(), 1295 ImageFormat.JPEG, /*filePath*/null); 1296 } 1297 1298 /** 1299 * Validate if video snapshot causes frame drop. 1300 * Here frame drop is defined as frame duration >= 2 * expected frame duration. 1301 * Return the estimated number of frames dropped during video snapshot 1302 */ 1303 private int validateFrameDropAroundVideoSnapshot( 1304 SimpleCaptureCallback resultListener, long imageTimeStamp) { 1305 double expectedDurationMs = 1000.0 / mVideoFrameRate; 1306 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1307 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 1308 while (resultListener.hasMoreResults()) { 1309 CaptureResult currentResult = 1310 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1311 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 1312 if (currentTS == imageTimeStamp) { 1313 // validate the timestamp before and after, then return 1314 CaptureResult nextResult = 1315 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1316 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP); 1317 double durationMs = (currentTS - prevTS) / 1000000.0; 1318 int totalFramesDropped = 0; 1319 1320 // Snapshots in legacy mode pause the preview briefly. Skip the duration 1321 // requirements for legacy mode unless this is fixed. 1322 if (!mStaticInfo.isHardwareLevelLegacy()) { 1323 mCollector.expectTrue( 1324 String.format( 1325 "Video %dx%d Frame drop detected before video snapshot: " + 1326 "duration %.2fms (expected %.2fms)", 1327 mVideoSize.getWidth(), mVideoSize.getHeight(), 1328 durationMs, expectedDurationMs 1329 ), 1330 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1331 ); 1332 // Log a warning is there is any frame drop detected. 1333 if (durationMs >= expectedDurationMs * 2) { 1334 Log.w(TAG, String.format( 1335 "Video %dx%d Frame drop detected before video snapshot: " + 1336 "duration %.2fms (expected %.2fms)", 1337 mVideoSize.getWidth(), mVideoSize.getHeight(), 1338 durationMs, expectedDurationMs 1339 )); 1340 } 1341 1342 durationMs = (nextTS - currentTS) / 1000000.0; 1343 mCollector.expectTrue( 1344 String.format( 1345 "Video %dx%d Frame drop detected after video snapshot: " + 1346 "duration %.2fms (expected %.2fms)", 1347 mVideoSize.getWidth(), mVideoSize.getHeight(), 1348 durationMs, expectedDurationMs 1349 ), 1350 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 1351 ); 1352 // Log a warning is there is any frame drop detected. 1353 if (durationMs >= expectedDurationMs * 2) { 1354 Log.w(TAG, String.format( 1355 "Video %dx%d Frame drop detected after video snapshot: " + 1356 "duration %fms (expected %fms)", 1357 mVideoSize.getWidth(), mVideoSize.getHeight(), 1358 durationMs, expectedDurationMs 1359 )); 1360 } 1361 1362 double totalDurationMs = (nextTS - prevTS) / 1000000.0; 1363 // Minus 2 for the expected 2 frames interval 1364 totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2; 1365 if (totalFramesDropped < 0) { 1366 Log.w(TAG, "totalFrameDropped is " + totalFramesDropped + 1367 ". Video frame rate might be too fast."); 1368 } 1369 totalFramesDropped = Math.max(0, totalFramesDropped); 1370 } 1371 return totalFramesDropped; 1372 } 1373 prevTS = currentTS; 1374 } 1375 throw new AssertionFailedError( 1376 "Video snapshot timestamp does not match any of capture results!"); 1377 } 1378 1379 /** 1380 * Validate frame jittering from the input simple listener's buffered results 1381 */ 1382 private void validateJittering(SimpleCaptureCallback resultListener) { 1383 double expectedDurationMs = 1000.0 / mVideoFrameRate; 1384 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1385 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 1386 while (resultListener.hasMoreResults()) { 1387 CaptureResult currentResult = 1388 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 1389 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 1390 double durationMs = (currentTS - prevTS) / 1000000.0; 1391 double durationError = Math.abs(durationMs - expectedDurationMs); 1392 long frameNumber = currentResult.getFrameNumber(); 1393 mCollector.expectTrue( 1394 String.format( 1395 "Resolution %dx%d Frame %d: jittering (%.2fms) exceeds bound [%.2fms,%.2fms]", 1396 mVideoSize.getWidth(), mVideoSize.getHeight(), 1397 frameNumber, durationMs, 1398 expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS, 1399 expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS), 1400 durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS); 1401 prevTS = currentTS; 1402 } 1403 } 1404 1405 /** 1406 * Calculate a video bit rate based on the size. The bit rate is scaled 1407 * based on ratio of video size to 1080p size. 1408 */ 1409 private int getVideoBitRate(Size sz) { 1410 int rate = BIT_RATE_1080P; 1411 float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080); 1412 rate = (int)(rate * scaleFactor); 1413 1414 // Clamp to the MIN, MAX range. 1415 return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate)); 1416 } 1417 1418 /** 1419 * Check if the encoder and camera are able to support this size and frame rate. 1420 * Assume the video compression format is AVC. 1421 */ 1422 private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception { 1423 // Check camera capability. 1424 if (!isSupportedByCamera(sz, captureRate)) { 1425 return false; 1426 } 1427 1428 // Check encode capability. 1429 if (!isSupportedByAVCEncoder(sz, encodingRate)){ 1430 return false; 1431 } 1432 1433 if(VERBOSE) { 1434 Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@" 1435 + getVideoBitRate(sz) / 1000 + "Kbps"); 1436 } 1437 1438 return true; 1439 } 1440 1441 private boolean isSupportedByCamera(Size sz, int frameRate) { 1442 // Check if camera can support this sz and frame rate combination. 1443 StreamConfigurationMap config = mStaticInfo. 1444 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1445 1446 long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz); 1447 if (minDuration == 0) { 1448 return false; 1449 } 1450 1451 int maxFrameRate = (int) (1e9f / minDuration); 1452 return maxFrameRate >= frameRate; 1453 } 1454 1455 /** 1456 * Check if encoder can support this size and frame rate combination by querying 1457 * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate 1458 * as the bit rates targeted in this test are well below the bit rate max value specified 1459 * by AVC specification for certain level. 1460 */ 1461 private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) { 1462 MediaFormat format = MediaFormat.createVideoFormat( 1463 MediaFormat.MIMETYPE_VIDEO_AVC, sz.getWidth(), sz.getHeight()); 1464 format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); 1465 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 1466 return mcl.findEncoderForFormat(format) != null; 1467 } 1468 } 1469