1 /* 2 * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache 3 * License, Version 2.0 (the "License"); you may not use this file except in 4 * compliance with the License. You may obtain a copy of the License at 5 * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law 6 * or agreed to in writing, software distributed under the License is 7 * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 8 * KIND, either express or implied. See the License for the specific language 9 * governing permissions and limitations under the License. 10 */ 11 12 package android.hardware.camera2.cts; 13 14 import static android.hardware.camera2.cts.CameraTestUtils.*; 15 import static android.media.MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10; 16 import static android.media.MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10; 17 import static android.media.MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10Plus; 18 import static android.media.MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10; 19 import static android.media.MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10; 20 import static android.media.MediaCodecInfo.CodecProfileLevel.HEVCProfileMain10HDR10Plus; 21 22 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*; 23 24 import static org.mockito.Matchers.eq; 25 import static org.mockito.Mockito.mock; 26 import static org.mockito.Mockito.timeout; 27 import static org.mockito.Mockito.verify; 28 29 import android.graphics.ImageFormat; 30 import android.graphics.SurfaceTexture; 31 import android.hardware.HardwareBuffer; 32 import android.hardware.camera2.CameraCaptureSession; 33 import android.hardware.camera2.CameraCharacteristics; 34 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession; 35 import android.hardware.camera2.CameraDevice; 36 import android.hardware.camera2.CaptureRequest; 37 import android.hardware.camera2.CaptureResult; 38 import android.hardware.camera2.cts.helpers.StaticMetadata; 39 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase; 40 import android.hardware.camera2.params.DynamicRangeProfiles; 41 import android.hardware.camera2.params.OutputConfiguration; 42 import android.hardware.camera2.params.SessionConfiguration; 43 import android.hardware.camera2.params.StreamConfigurationMap; 44 import android.media.CamcorderProfile; 45 import android.media.EncoderProfiles; 46 import android.media.Image; 47 import android.media.ImageReader; 48 import android.media.ImageWriter; 49 import android.media.MediaCodec; 50 import android.media.MediaCodecInfo.CodecCapabilities; 51 import android.media.MediaCodecList; 52 import android.media.MediaExtractor; 53 import android.media.MediaFormat; 54 import android.media.MediaMuxer; 55 import android.media.MediaRecorder; 56 import android.os.Handler; 57 import android.os.HandlerThread; 58 import android.os.SystemClock; 59 import android.util.Log; 60 import android.util.Range; 61 import android.util.Size; 62 import android.view.Surface; 63 64 import androidx.test.filters.LargeTest; 65 66 import com.android.compatibility.common.util.MediaUtils; 67 import com.android.ex.camera2.blocking.BlockingSessionCallback; 68 69 import junit.framework.AssertionFailedError; 70 71 import org.junit.Test; 72 import org.junit.runner.RunWith; 73 import org.junit.runners.Parameterized; 74 75 import java.io.File; 76 import java.util.ArrayList; 77 import java.util.Arrays; 78 import java.util.Collections; 79 import java.util.HashMap; 80 import java.util.List; 81 import java.util.Objects; 82 import java.util.Optional; 83 import java.util.Set; 84 85 /** 86 * CameraDevice video recording use case tests by using MediaRecorder and 87 * MediaCodec. 88 */ 89 @LargeTest 90 @RunWith(Parameterized.class) 91 public class RecordingTest extends Camera2SurfaceViewTestCase { 92 private static final String TAG = "RecordingTest"; 93 private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE); 94 private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG); 95 private static final int RECORDING_DURATION_MS = 3000; 96 private static final int PREVIEW_DURATION_MS = 3000; 97 private static final float DURATION_MARGIN = 0.2f; 98 private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0; 99 private static final float FRAMEDURATION_MARGIN = 0.2f; 100 private static final float VID_SNPSHT_FRMDRP_RATE_TOLERANCE = 10.0f; 101 private static final float FRMDRP_RATE_TOLERANCE = 5.0f; 102 private static final int BIT_RATE_1080P = 16000000; 103 private static final int BIT_RATE_MIN = 64000; 104 private static final int BIT_RATE_MAX = 40000000; 105 private static final int VIDEO_FRAME_RATE = 30; 106 private static final int[] mCamcorderProfileList = { 107 CamcorderProfile.QUALITY_HIGH, 108 CamcorderProfile.QUALITY_2160P, 109 CamcorderProfile.QUALITY_1080P, 110 CamcorderProfile.QUALITY_720P, 111 CamcorderProfile.QUALITY_480P, 112 CamcorderProfile.QUALITY_CIF, 113 CamcorderProfile.QUALITY_QCIF, 114 CamcorderProfile.QUALITY_QVGA, 115 CamcorderProfile.QUALITY_LOW, 116 }; 117 118 private static final int[] mTenBitHEVCCodecProfileList = { 119 HEVCProfileMain10, 120 HEVCProfileMain10HDR10, 121 HEVCProfileMain10HDR10Plus, 122 //todo(b/215396395): DolbyVision 123 }; 124 private static final int[] mTenBitAV1CodecProfileList = { 125 AV1ProfileMain10, 126 AV1ProfileMain10HDR10, 127 AV1ProfileMain10HDR10Plus, 128 //todo(b/215396395): DolbyVision 129 }; 130 private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5; 131 private static final int BURST_VIDEO_SNAPSHOT_NUM = 3; 132 private static final int SLOWMO_SLOW_FACTOR = 4; 133 private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4; 134 private List<Size> mSupportedVideoSizes; 135 private Surface mRecordingSurface; 136 private Surface mPersistentSurface; 137 private MediaRecorder mMediaRecorder; 138 private String mOutMediaFileName; 139 private int mVideoFrameRate; 140 private Size mVideoSize; 141 private long mRecordingStartTime; 142 143 private Surface mIntermediateSurface; 144 private ImageReader mIntermediateReader; 145 private ImageWriter mIntermediateWriter; 146 private ImageWriterQueuer mQueuer; 147 private HandlerThread mIntermediateThread; 148 private Handler mIntermediateHandler; 149 150 @Override setUp()151 public void setUp() throws Exception { 152 super.setUp(); 153 } 154 155 @Override tearDown()156 public void tearDown() throws Exception { 157 super.tearDown(); 158 } 159 doBasicRecording(boolean useVideoStab)160 private void doBasicRecording(boolean useVideoStab) throws Exception { 161 doBasicRecording(useVideoStab, false); 162 } 163 doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface)164 private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface) 165 throws Exception { 166 doBasicRecording(useVideoStab, useIntermediateSurface, false); 167 } 168 doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)169 private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface, 170 boolean useEncoderProfiles) throws Exception { 171 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 172 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 173 try { 174 Log.i(TAG, "Testing basic recording for camera " + cameraIdsUnderTest[i]); 175 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 176 if (!staticInfo.isColorOutputSupported()) { 177 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 178 " does not support color outputs, skipping"); 179 continue; 180 } 181 182 // External camera doesn't support CamcorderProfile recording 183 if (staticInfo.isExternalCamera()) { 184 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 185 " does not support CamcorderProfile, skipping"); 186 continue; 187 } 188 189 if (!staticInfo.isVideoStabilizationSupported() && useVideoStab) { 190 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 191 " does not support video stabilization, skipping the stabilization" 192 + " test"); 193 continue; 194 } 195 196 // Re-use the MediaRecorder object for the same camera device. 197 mMediaRecorder = new MediaRecorder(); 198 openDevice(cameraIdsUnderTest[i]); 199 initSupportedVideoSize(cameraIdsUnderTest[i]); 200 201 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab, 202 useIntermediateSurface, useEncoderProfiles); 203 } finally { 204 closeDevice(); 205 releaseRecorder(); 206 } 207 } 208 } 209 210 /** 211 * <p> 212 * Test basic video stabilization camera recording. 213 * </p> 214 * <p> 215 * This test covers the typical basic use case of camera recording with video 216 * stabilization is enabled, if video stabilization is supported. 217 * MediaRecorder is used to record the audio and video, CamcorderProfile is 218 * used to configure the MediaRecorder. It goes through the pre-defined 219 * CamcorderProfile list, test each profile configuration and validate the 220 * recorded video. Preview is set to the video size. 221 * </p> 222 */ 223 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBasicVideoStabilizationRecording()224 public void testBasicVideoStabilizationRecording() throws Exception { 225 doBasicRecording(/*useVideoStab*/true); 226 } 227 228 /** 229 * <p> 230 * Test basic camera recording. 231 * </p> 232 * <p> 233 * This test covers the typical basic use case of camera recording. 234 * MediaRecorder is used to record the audio and video, CamcorderProfile is 235 * used to configure the MediaRecorder. It goes through the pre-defined 236 * CamcorderProfile list, test each profile configuration and validate the 237 * recorded video. Preview is set to the video size. 238 * </p> 239 */ 240 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBasicRecording()241 public void testBasicRecording() throws Exception { 242 doBasicRecording(/*useVideoStab*/false); 243 } 244 245 /** 246 * <p> 247 * Test camera recording with intermediate surface. 248 * </p> 249 * <p> 250 * This test is similar to testBasicRecording with a tweak where an intermediate 251 * surface is setup between camera and MediaRecorder, giving application a chance 252 * to decide whether to send a frame to recorder or not. 253 * </p> 254 */ 255 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testIntermediateSurfaceRecording()256 public void testIntermediateSurfaceRecording() throws Exception { 257 doBasicRecording(/*useVideoStab*/false, /*useIntermediateSurface*/true); 258 } 259 260 /** 261 * <p> 262 * Test basic camera recording using encoder profiles. 263 * </p> 264 * <p> 265 * This test covers the typical basic use case of camera recording. 266 * MediaRecorder is used to record the audio and video, 267 * EncoderProfiles are used to configure the MediaRecorder. It 268 * goes through the pre-defined CamcorderProfile list, test each 269 * encoder profile combination and validate the recorded video. 270 * Preview is set to the video size. 271 * </p> 272 */ 273 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBasicEncoderProfilesRecording()274 public void testBasicEncoderProfilesRecording() throws Exception { 275 doBasicRecording(/*useVideoStab*/false, /*useIntermediateSurface*/false, 276 /*useEncoderProfiles*/true); 277 } 278 279 /** 280 * <p> 281 * Test basic camera recording from a persistent input surface. 282 * </p> 283 * <p> 284 * This test is similar to testBasicRecording except that MediaRecorder records 285 * from a persistent input surface that's used across multiple recording sessions. 286 * </p> 287 */ 288 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testRecordingFromPersistentSurface()289 public void testRecordingFromPersistentSurface() throws Exception { 290 if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) { 291 return; // skipped 292 } 293 mPersistentSurface = MediaCodec.createPersistentInputSurface(); 294 assertNotNull("Failed to create persistent input surface!", mPersistentSurface); 295 296 try { 297 doBasicRecording(/*useVideoStab*/false); 298 } finally { 299 mPersistentSurface.release(); 300 mPersistentSurface = null; 301 } 302 } 303 304 /** 305 * <p> 306 * Test camera recording for all supported sizes by using MediaRecorder. 307 * </p> 308 * <p> 309 * This test covers camera recording for all supported sizes by camera. MediaRecorder 310 * is used to encode the video. Preview is set to the video size. Recorded videos are 311 * validated according to the recording configuration. 312 * </p> 313 */ 314 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testSupportedVideoSizes()315 public void testSupportedVideoSizes() throws Exception { 316 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 317 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 318 try { 319 Log.i(TAG, "Testing supported video size recording for camera " + cameraIdsUnderTest[i]); 320 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 321 if (!staticInfo.isColorOutputSupported()) { 322 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 323 " does not support color outputs, skipping"); 324 continue; 325 } 326 if (staticInfo.isExternalCamera()) { 327 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 328 " does not support CamcorderProfile, skipping"); 329 continue; 330 } 331 // Re-use the MediaRecorder object for the same camera device. 332 mMediaRecorder = new MediaRecorder(); 333 openDevice(cameraIdsUnderTest[i]); 334 335 initSupportedVideoSize(cameraIdsUnderTest[i]); 336 337 recordingSizeTestByCamera(); 338 } finally { 339 closeDevice(); 340 releaseRecorder(); 341 } 342 } 343 } 344 345 /** 346 * Test different start/stop orders of Camera and Recorder. 347 * 348 * <p>The recording should be working fine for any kind of start/stop orders.</p> 349 */ 350 @Test testCameraRecorderOrdering()351 public void testCameraRecorderOrdering() { 352 // TODO: need implement 353 } 354 355 /** 356 * <p> 357 * Test camera recording for all supported sizes by using MediaCodec. 358 * </p> 359 * <p> 360 * This test covers video only recording for all supported sizes (camera and 361 * encoder). MediaCodec is used to encode the video. The recorded videos are 362 * validated according to the recording configuration. 363 * </p> 364 */ 365 @Test testMediaCodecRecording()366 public void testMediaCodecRecording() throws Exception { 367 // TODO. Need implement. 368 } 369 370 private class MediaCodecListener extends MediaCodec.Callback { 371 private final MediaMuxer mMediaMuxer; 372 private final Object mCondition; 373 private int mTrackId = -1; 374 private boolean mEndOfStream = false; 375 MediaCodecListener(MediaMuxer mediaMuxer, Object condition)376 private MediaCodecListener(MediaMuxer mediaMuxer, Object condition) { 377 mMediaMuxer = mediaMuxer; 378 mCondition = condition; 379 } 380 381 @Override onInputBufferAvailable(MediaCodec codec, int index)382 public void onInputBufferAvailable(MediaCodec codec, int index) { 383 fail("Unexpected input buffer available callback!"); 384 } 385 386 @Override onOutputBufferAvailable(MediaCodec codec, int index, MediaCodec.BufferInfo info)387 public void onOutputBufferAvailable(MediaCodec codec, int index, 388 MediaCodec.BufferInfo info) { 389 synchronized (mCondition) { 390 if (mTrackId < 0) { 391 return; 392 } 393 mMediaMuxer.writeSampleData(mTrackId, codec.getOutputBuffer(index), info); 394 if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 395 MediaCodec.BUFFER_FLAG_END_OF_STREAM) { 396 mEndOfStream = true; 397 mCondition.notifyAll(); 398 } 399 400 if (!mEndOfStream) { 401 codec.releaseOutputBuffer(index, false); 402 } 403 } 404 } 405 406 @Override onError(MediaCodec codec, MediaCodec.CodecException e)407 public void onError(MediaCodec codec, MediaCodec.CodecException e) { 408 fail("Codec error: " + e.getDiagnosticInfo()); 409 } 410 411 @Override onOutputFormatChanged(MediaCodec codec, MediaFormat format)412 public void onOutputFormatChanged(MediaCodec codec, MediaFormat format) { 413 synchronized (mCondition) { 414 mTrackId = mMediaMuxer.addTrack(format); 415 mMediaMuxer.start(); 416 } 417 } 418 } 419 getDynamicRangeProfile(int codecProfile, StaticMetadata staticMeta)420 private static long getDynamicRangeProfile(int codecProfile, StaticMetadata staticMeta) { 421 if (!staticMeta.isCapabilitySupported( 422 CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT)) { 423 return DynamicRangeProfiles.PUBLIC_MAX; 424 } 425 426 DynamicRangeProfiles profiles = staticMeta.getCharacteristics().get( 427 CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES); 428 Set<Long> availableProfiles = profiles.getSupportedProfiles(); 429 switch (codecProfile) { 430 case HEVCProfileMain10: 431 return availableProfiles.contains(DynamicRangeProfiles.HLG10) ? 432 DynamicRangeProfiles.HLG10 : DynamicRangeProfiles.PUBLIC_MAX; 433 case HEVCProfileMain10HDR10: 434 return availableProfiles.contains(DynamicRangeProfiles.HDR10) ? 435 DynamicRangeProfiles.HDR10 : DynamicRangeProfiles.PUBLIC_MAX; 436 case HEVCProfileMain10HDR10Plus: 437 return availableProfiles.contains(DynamicRangeProfiles.HDR10_PLUS) ? 438 DynamicRangeProfiles.HDR10_PLUS : DynamicRangeProfiles.PUBLIC_MAX; 439 //todo(b/215396395): DolbyVision 440 default: 441 return DynamicRangeProfiles.PUBLIC_MAX; 442 } 443 } 444 getTransferFunction(int codecProfile)445 private static int getTransferFunction(int codecProfile) { 446 switch (codecProfile) { 447 case HEVCProfileMain10: 448 return MediaFormat.COLOR_TRANSFER_HLG; 449 case HEVCProfileMain10HDR10: 450 case HEVCProfileMain10HDR10Plus: 451 //todo(b/215396395): DolbyVision 452 return MediaFormat.COLOR_TRANSFER_ST2084; 453 default: 454 return MediaFormat.COLOR_TRANSFER_SDR_VIDEO; 455 } 456 } 457 458 /** 459 * <p> 460 * Test basic camera 10-bit recording for HEVC codec. 461 * </p> 462 * <p> 463 * This test covers the typical basic use case of camera recording. 464 * MediaCodec is used to record 10-bit video, CamcorderProfile and codec profiles 465 * are used to configure the MediaCodec. It goes through the pre-defined 466 * CamcorderProfile and 10-bit codec profile lists, tests each configuration and 467 * validates the recorded video. Preview is set to the video size. 468 * </p> 469 */ 470 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBasic10BitRecordingHEVC()471 public void testBasic10BitRecordingHEVC() throws Exception { 472 testBasic10BitRecording(mTenBitHEVCCodecProfileList, MediaFormat.MIMETYPE_VIDEO_HEVC); 473 } 474 475 /** 476 * <p> 477 * Test basic camera 10-bit recording for AV1 codec. 478 * </p> 479 * <p> 480 * This test covers the typical basic use case of camera recording. 481 * MediaCodec is used to record 10-bit video, CamcorderProfile and codec profiles 482 * are used to configure the MediaCodec. It goes through the pre-defined 483 * CamcorderProfile and 10-bit codec profile lists, tests each configuration and 484 * validates the recorded video. Preview is set to the video size. 485 * </p> 486 */ 487 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBasic10BitRecordingAV1()488 public void testBasic10BitRecordingAV1() throws Exception { 489 testBasic10BitRecording(mTenBitAV1CodecProfileList, MediaFormat.MIMETYPE_VIDEO_AV1); 490 } 491 testBasic10BitRecording(int[] tenBitCodecProfileList, String mimetype)492 private void testBasic10BitRecording(int[] tenBitCodecProfileList, String mimetype) throws Exception { 493 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 494 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 495 try { 496 Log.i(TAG, "Testing 10-bit recording " + cameraIdsUnderTest[i]); 497 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 498 if (!staticInfo.isColorOutputSupported()) { 499 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 500 " does not support color outputs, skipping"); 501 continue; 502 } 503 if (staticInfo.isExternalCamera()) { 504 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 505 " does not support CamcorderProfile, skipping"); 506 continue; 507 } 508 509 int cameraId; 510 try { 511 cameraId = Integer.valueOf(cameraIdsUnderTest[i]); 512 } catch (NumberFormatException e) { 513 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + " cannot be parsed" 514 + " to an integer camera id, skipping"); 515 continue; 516 } 517 518 for (int camcorderProfile : mCamcorderProfileList) { 519 if (!CamcorderProfile.hasProfile(cameraId, camcorderProfile)) { 520 continue; 521 } 522 523 for (int codecProfile : tenBitCodecProfileList) { 524 CamcorderProfile profile = CamcorderProfile.get(cameraId, camcorderProfile); 525 526 Size videoSize = new Size(profile.videoFrameWidth, 527 profile.videoFrameHeight); 528 MediaCodecList list = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 529 MediaFormat format = MediaFormat.createVideoFormat( 530 mimetype, videoSize.getWidth(), 531 videoSize.getHeight()); 532 format.setInteger(MediaFormat.KEY_PROFILE, codecProfile); 533 format.setInteger(MediaFormat.KEY_BIT_RATE, profile.videoBitRate); 534 format.setInteger(MediaFormat.KEY_FRAME_RATE, profile.videoFrameRate); 535 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, 536 CodecCapabilities.COLOR_FormatSurface); 537 format.setInteger(MediaFormat.KEY_COLOR_STANDARD, 538 MediaFormat.COLOR_STANDARD_BT2020); 539 format.setInteger(MediaFormat.KEY_COLOR_RANGE, 540 MediaFormat.COLOR_RANGE_FULL); 541 format.setInteger(MediaFormat.KEY_COLOR_TRANSFER, 542 getTransferFunction(codecProfile)); 543 format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); 544 545 String codecName = list.findEncoderForFormat(format); 546 if (codecName == null) { 547 continue; 548 } 549 550 long dynamicRangeProfile = getDynamicRangeProfile(codecProfile, staticInfo); 551 if (dynamicRangeProfile == DynamicRangeProfiles.PUBLIC_MAX) { 552 continue; 553 } 554 555 MediaCodec mediaCodec = null; 556 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 557 MediaMuxer muxer = new MediaMuxer(mOutMediaFileName, 558 MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); 559 SimpleCaptureCallback captureCallback = new SimpleCaptureCallback(); 560 try { 561 mediaCodec = MediaCodec.createByCodecName(codecName); 562 assertNotNull(mediaCodec); 563 if (!mediaCodec.getCodecInfo().isHardwareAccelerated()) { 564 continue; 565 } 566 567 openDevice(cameraIdsUnderTest[i]); 568 569 mediaCodec.configure(format, null, null, 570 MediaCodec.CONFIGURE_FLAG_ENCODE); 571 Object condition = new Object(); 572 mediaCodec.setCallback(new MediaCodecListener(muxer, condition), 573 mHandler); 574 575 updatePreviewSurfaceWithVideo(videoSize, profile.videoFrameRate); 576 577 Surface recordingSurface = mediaCodec.createInputSurface(); 578 assertNotNull(recordingSurface); 579 580 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 581 assertTrue("Both preview and recording surfaces should be valid", 582 mPreviewSurface.isValid() && recordingSurface.isValid()); 583 584 outputSurfaces.add(mPreviewSurface); 585 outputSurfaces.add(recordingSurface); 586 587 CameraCaptureSession.StateCallback mockCallback = mock( 588 CameraCaptureSession.StateCallback.class); 589 BlockingSessionCallback sessionListener = 590 new BlockingSessionCallback(mockCallback); 591 592 CaptureRequest.Builder recordingRequestBuilder = 593 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 594 recordingRequestBuilder.addTarget(recordingSurface); 595 recordingRequestBuilder.addTarget(mPreviewSurface); 596 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, 597 new Range<>(profile.videoFrameRate, profile.videoFrameRate)); 598 CaptureRequest recordingRequest = recordingRequestBuilder.build(); 599 600 List<OutputConfiguration> outConfigurations = 601 new ArrayList<>(outputSurfaces.size()); 602 for (Surface s : outputSurfaces) { 603 OutputConfiguration config = new OutputConfiguration(s); 604 config.setDynamicRangeProfile(dynamicRangeProfile); 605 outConfigurations.add(config); 606 } 607 608 SessionConfiguration sessionConfig = new SessionConfiguration( 609 SessionConfiguration.SESSION_REGULAR, outConfigurations, 610 new HandlerExecutor(mHandler), sessionListener); 611 mCamera.createCaptureSession(sessionConfig); 612 613 CameraCaptureSession session = sessionListener.waitAndGetSession( 614 SESSION_CONFIGURE_TIMEOUT_MS); 615 616 mediaCodec.start(); 617 session.setRepeatingRequest(recordingRequest, captureCallback, 618 mHandler); 619 620 SystemClock.sleep(RECORDING_DURATION_MS); 621 622 session.stopRepeating(); 623 session.close(); 624 verify(mockCallback, timeout(SESSION_CLOSE_TIMEOUT_MS). 625 times(1)).onClosed(eq(session)); 626 627 mediaCodec.signalEndOfInputStream(); 628 synchronized (condition) { 629 condition.wait(SESSION_CLOSE_TIMEOUT_MS); 630 } 631 632 mediaCodec.stop(); 633 muxer.stop(); 634 635 } finally { 636 if (mediaCodec != null) { 637 mediaCodec.release(); 638 } 639 if (muxer != null) { 640 muxer.release(); 641 } 642 } 643 644 // Validation. 645 float frameDurationMinMs = 1000.0f / profile.videoFrameRate; 646 float durationMinMs = 647 captureCallback.getTotalNumFrames() * frameDurationMinMs; 648 float durationMaxMs = durationMinMs; 649 float frameDurationMaxMs = 0.f; 650 651 validateRecording(videoSize, durationMinMs, durationMaxMs, 652 frameDurationMinMs, frameDurationMaxMs, 653 FRMDRP_RATE_TOLERANCE); 654 } 655 } 656 } finally { 657 closeDevice(); 658 } 659 } 660 } 661 662 /** 663 * <p> 664 * Test video snapshot for each camera. 665 * </p> 666 * <p> 667 * This test covers video snapshot typical use case. The MediaRecorder is used to record the 668 * video for each available video size. The largest still capture size is selected to 669 * capture the JPEG image. The still capture images are validated according to the capture 670 * configuration. The timestamp of capture result before and after video snapshot is also 671 * checked to make sure no frame drop caused by video snapshot. 672 * </p> 673 */ 674 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testVideoSnapshot()675 public void testVideoSnapshot() throws Exception { 676 videoSnapshotHelper(/*burstTest*/false); 677 } 678 679 /** 680 * <p> 681 * Test burst video snapshot for each camera. 682 * </p> 683 * <p> 684 * This test covers burst video snapshot capture. The MediaRecorder is used to record the 685 * video for each available video size. The largest still capture size is selected to 686 * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be 687 * sent during the test. The still capture images are validated according to the capture 688 * configuration. 689 * </p> 690 */ 691 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testBurstVideoSnapshot()692 public void testBurstVideoSnapshot() throws Exception { 693 videoSnapshotHelper(/*burstTest*/true); 694 } 695 696 /** 697 * Test timelapse recording, where capture rate is slower than video (playback) frame rate. 698 */ 699 @Test testTimelapseRecording()700 public void testTimelapseRecording() throws Exception { 701 // TODO. Need implement. 702 } 703 704 @Test testSlowMotionRecording()705 public void testSlowMotionRecording() throws Exception { 706 slowMotionRecording(); 707 } 708 709 @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests testConstrainedHighSpeedRecording()710 public void testConstrainedHighSpeedRecording() throws Exception { 711 constrainedHighSpeedRecording(); 712 } 713 714 @Test testAbandonedHighSpeedRequest()715 public void testAbandonedHighSpeedRequest() throws Exception { 716 for (String id : getCameraIdsUnderTest()) { 717 try { 718 Log.i(TAG, "Testing bad suface for createHighSpeedRequestList for camera " + id); 719 StaticMetadata staticInfo = mAllStaticInfo.get(id); 720 if (!staticInfo.isColorOutputSupported()) { 721 Log.i(TAG, "Camera " + id + 722 " does not support color outputs, skipping"); 723 continue; 724 } 725 if (!staticInfo.isConstrainedHighSpeedVideoSupported()) { 726 Log.i(TAG, "Camera " + id + 727 " does not support constrained high speed video, skipping"); 728 continue; 729 } 730 731 // Re-use the MediaRecorder object for the same camera device. 732 mMediaRecorder = new MediaRecorder(); 733 openDevice(id); 734 735 StreamConfigurationMap config = 736 mStaticInfo.getValueFromKeyNonNull( 737 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 738 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 739 Size size = highSpeedVideoSizes[0]; 740 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size); 741 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " + 742 "size " + size, fpsRange); 743 if (fpsRange == null) { 744 continue; 745 } 746 747 int captureRate = fpsRange.getLower(); 748 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR; 749 // Skip the test if the highest recording FPS supported by CamcorderProfile 750 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 751 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 752 + " is not supported by CamcorderProfile"); 753 continue; 754 } 755 756 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 757 prepareRecording(size, videoFramerate, captureRate); 758 updatePreviewSurfaceWithVideo(size, captureRate); 759 760 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 761 assertTrue("Both preview and recording surfaces should be valid", 762 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 763 764 outputSurfaces.add(mPreviewSurface); 765 outputSurfaces.add(mRecordingSurface); 766 767 mSessionListener = new BlockingSessionCallback(); 768 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true, 769 mSessionListener, mHandler); 770 771 CaptureRequest.Builder requestBuilder = 772 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 773 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 774 requestBuilder.addTarget(mPreviewSurface); 775 requestBuilder.addTarget(mRecordingSurface); 776 777 // 1. Test abandoned MediaRecorder 778 releaseRecorder(); 779 try { 780 List<CaptureRequest> slowMoRequests = 781 ((CameraConstrainedHighSpeedCaptureSession) mSession). 782 createHighSpeedRequestList(requestBuilder.build()); 783 fail("Create high speed request on abandoned surface must fail!"); 784 } catch (IllegalArgumentException e) { 785 Log.i(TAG, "Release recording surface test passed"); 786 // expected 787 } 788 789 // 2. Test abandoned preview surface 790 mMediaRecorder = new MediaRecorder(); 791 SurfaceTexture preview = new SurfaceTexture(/*random int*/ 1); 792 Surface previewSurface = new Surface(preview); 793 preview.setDefaultBufferSize(size.getWidth(), size.getHeight()); 794 795 outputSurfaces = new ArrayList<Surface>(); 796 outputSurfaces.add(previewSurface); 797 798 prepareRecording(size, videoFramerate, captureRate); 799 updatePreviewSurfaceWithVideo(size, captureRate); 800 801 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true, 802 mSessionListener, mHandler); 803 804 requestBuilder = mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 805 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 806 requestBuilder.addTarget(previewSurface); 807 808 // Abandon preview surface. 809 previewSurface.release(); 810 811 try { 812 List<CaptureRequest> slowMoRequests = 813 ((CameraConstrainedHighSpeedCaptureSession) mSession). 814 createHighSpeedRequestList(requestBuilder.build()); 815 fail("Create high speed request on abandoned preview surface must fail!"); 816 } catch (IllegalArgumentException e) { 817 Log.i(TAG, "Release preview surface test passed"); 818 // expected 819 } 820 } finally { 821 closeDevice(); 822 releaseRecorder(); 823 } 824 } 825 } 826 827 /** 828 * <p> 829 * Test recording framerate accuracy when switching from low FPS to high FPS. 830 * </p> 831 * <p> 832 * This test first record a video with profile of lowest framerate then record a video with 833 * profile of highest framerate. Make sure that the video framerate are still accurate. 834 * </p> 835 */ 836 @Test testRecordingFramerateLowToHigh()837 public void testRecordingFramerateLowToHigh() throws Exception { 838 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 839 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 840 try { 841 Log.i(TAG, "Testing recording framerate low to high for camera " + cameraIdsUnderTest[i]); 842 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 843 if (!staticInfo.isColorOutputSupported()) { 844 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 845 " does not support color outputs, skipping"); 846 continue; 847 } 848 if (staticInfo.isExternalCamera()) { 849 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 850 " does not support CamcorderProfile, skipping"); 851 continue; 852 } 853 // Re-use the MediaRecorder object for the same camera device. 854 mMediaRecorder = new MediaRecorder(); 855 openDevice(cameraIdsUnderTest[i]); 856 857 initSupportedVideoSize(cameraIdsUnderTest[i]); 858 859 int minFpsProfileId = -1, minFps = 1000; 860 int maxFpsProfileId = -1, maxFps = 0; 861 int cameraId = Integer.valueOf(mCamera.getId()); 862 863 for (int profileId : mCamcorderProfileList) { 864 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 865 continue; 866 } 867 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 868 if (profile.videoFrameRate < minFps) { 869 minFpsProfileId = profileId; 870 minFps = profile.videoFrameRate; 871 } 872 if (profile.videoFrameRate > maxFps) { 873 maxFpsProfileId = profileId; 874 maxFps = profile.videoFrameRate; 875 } 876 } 877 878 int camcorderProfileList[] = new int[] {minFpsProfileId, maxFpsProfileId}; 879 basicRecordingTestByCamera(camcorderProfileList, /*useVideoStab*/false); 880 } finally { 881 closeDevice(); 882 releaseRecorder(); 883 } 884 } 885 } 886 887 /** 888 * <p> 889 * Test preview and video surfaces sharing the same camera stream. 890 * </p> 891 */ 892 @Test testVideoPreviewSurfaceSharing()893 public void testVideoPreviewSurfaceSharing() throws Exception { 894 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 895 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 896 try { 897 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 898 if (staticInfo.isHardwareLevelLegacy()) { 899 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + " is legacy, skipping"); 900 continue; 901 } 902 if (!staticInfo.isColorOutputSupported()) { 903 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 904 " does not support color outputs, skipping"); 905 continue; 906 } 907 if (staticInfo.isExternalCamera()) { 908 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 909 " does not support CamcorderProfile, skipping"); 910 continue; 911 } 912 // Re-use the MediaRecorder object for the same camera device. 913 mMediaRecorder = new MediaRecorder(); 914 openDevice(cameraIdsUnderTest[i]); 915 916 initSupportedVideoSize(cameraIdsUnderTest[i]); 917 918 videoPreviewSurfaceSharingTestByCamera(); 919 } finally { 920 closeDevice(); 921 releaseRecorder(); 922 } 923 } 924 } 925 926 /** 927 * <p> 928 * Test recording with same recording surface and different preview surfaces. 929 * </p> 930 * <p> 931 * This test maintains persistent video surface while changing preview surface. 932 * This exercises format/dataspace override behavior of the camera device. 933 * </p> 934 */ 935 @Test testRecordingWithDifferentPreviewSizes()936 public void testRecordingWithDifferentPreviewSizes() throws Exception { 937 if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) { 938 return; // skipped 939 } 940 mPersistentSurface = MediaCodec.createPersistentInputSurface(); 941 assertNotNull("Failed to create persistent input surface!", mPersistentSurface); 942 943 try { 944 doRecordingWithDifferentPreviewSizes(); 945 } finally { 946 mPersistentSurface.release(); 947 mPersistentSurface = null; 948 } 949 } 950 doRecordingWithDifferentPreviewSizes()951 public void doRecordingWithDifferentPreviewSizes() throws Exception { 952 String[] cameraIdsUnderTest = getCameraIdsUnderTest(); 953 for (int i = 0; i < cameraIdsUnderTest.length; i++) { 954 try { 955 Log.i(TAG, "Testing recording with different preview sizes for camera " + 956 cameraIdsUnderTest[i]); 957 StaticMetadata staticInfo = mAllStaticInfo.get(cameraIdsUnderTest[i]); 958 if (!staticInfo.isColorOutputSupported()) { 959 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 960 " does not support color outputs, skipping"); 961 continue; 962 } 963 if (staticInfo.isExternalCamera()) { 964 Log.i(TAG, "Camera " + cameraIdsUnderTest[i] + 965 " does not support CamcorderProfile, skipping"); 966 continue; 967 } 968 // Re-use the MediaRecorder object for the same camera device. 969 mMediaRecorder = new MediaRecorder(); 970 openDevice(cameraIdsUnderTest[i]); 971 972 initSupportedVideoSize(cameraIdsUnderTest[i]); 973 974 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 975 List<Range<Integer> > fpsRanges = Arrays.asList( 976 mStaticInfo.getAeAvailableTargetFpsRangesChecked()); 977 int cameraId = Integer.valueOf(mCamera.getId()); 978 int maxVideoFrameRate = -1; 979 for (int profileId : mCamcorderProfileList) { 980 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 981 continue; 982 } 983 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 984 985 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 986 Range<Integer> fpsRange = new Range( 987 profile.videoFrameRate, profile.videoFrameRate); 988 if (maxVideoFrameRate < profile.videoFrameRate) { 989 maxVideoFrameRate = profile.videoFrameRate; 990 } 991 992 if (allowedUnsupported(cameraId, profileId)) { 993 continue; 994 } 995 996 if (mStaticInfo.isHardwareLevelLegacy() && 997 (videoSz.getWidth() > maxPreviewSize.getWidth() || 998 videoSz.getHeight() > maxPreviewSize.getHeight())) { 999 // Skip. Legacy mode can only do recording up to max preview size 1000 continue; 1001 } 1002 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 1003 " must be one of the camera device supported video size!", 1004 mSupportedVideoSizes.contains(videoSz)); 1005 assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId + 1006 ") must be one of the camera device available FPS range!", 1007 fpsRanges.contains(fpsRange)); 1008 1009 // Configure preview and recording surfaces. 1010 mOutMediaFileName = mDebugFileNameBase + "/test_video_surface_reconfig.mp4"; 1011 1012 // prepare preview surface by using video size. 1013 List<Size> previewSizes = getPreviewSizesForVideo(videoSz, 1014 profile.videoFrameRate); 1015 if (previewSizes.size() <= 1) { 1016 continue; 1017 } 1018 1019 // 1. Do video recording using largest compatbile preview sizes 1020 prepareRecordingWithProfile(profile); 1021 updatePreviewSurface(previewSizes.get(0)); 1022 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1023 startRecording( 1024 /* useMediaRecorder */true, resultListener, 1025 /*useVideoStab*/false, fpsRange, false); 1026 SystemClock.sleep(RECORDING_DURATION_MS); 1027 stopRecording(/* useMediaRecorder */true, /* useIntermediateSurface */false, 1028 /* stopStreaming */false); 1029 1030 // 2. Reconfigure with the same recording surface, but switch to a smaller 1031 // preview size. 1032 prepareRecordingWithProfile(profile); 1033 updatePreviewSurface(previewSizes.get(1)); 1034 SimpleCaptureCallback resultListener2 = new SimpleCaptureCallback(); 1035 startRecording( 1036 /* useMediaRecorder */true, resultListener2, 1037 /*useVideoStab*/false, fpsRange, false); 1038 SystemClock.sleep(RECORDING_DURATION_MS); 1039 stopRecording(/* useMediaRecorder */true); 1040 break; 1041 } 1042 } finally { 1043 closeDevice(); 1044 releaseRecorder(); 1045 } 1046 } 1047 } 1048 1049 /** 1050 * Test camera preview and video surface sharing for maximum supported size. 1051 */ videoPreviewSurfaceSharingTestByCamera()1052 private void videoPreviewSurfaceSharingTestByCamera() throws Exception { 1053 for (Size sz : mOrderedPreviewSizes) { 1054 if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) { 1055 continue; 1056 } 1057 1058 if (VERBOSE) { 1059 Log.v(TAG, "Testing camera recording with video size " + sz.toString()); 1060 } 1061 1062 // Configure preview and recording surfaces. 1063 mOutMediaFileName = mDebugFileNameBase + "/test_video_share.mp4"; 1064 if (DEBUG_DUMP) { 1065 mOutMediaFileName = mDebugFileNameBase + "/test_video_share_" + mCamera.getId() + 1066 "_" + sz.toString() + ".mp4"; 1067 } 1068 1069 // Allow external camera to use variable fps range 1070 Range<Integer> fpsRange = null; 1071 if (mStaticInfo.isExternalCamera()) { 1072 Range<Integer>[] availableFpsRange = 1073 mStaticInfo.getAeAvailableTargetFpsRangesChecked(); 1074 1075 boolean foundRange = false; 1076 int minFps = 0; 1077 for (int i = 0; i < availableFpsRange.length; i += 1) { 1078 if (minFps < availableFpsRange[i].getLower() 1079 && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) { 1080 minFps = availableFpsRange[i].getLower(); 1081 foundRange = true; 1082 } 1083 } 1084 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange); 1085 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE); 1086 } 1087 1088 // Use AVC and AAC a/v compression format. 1089 prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE); 1090 1091 // prepare preview surface by using video size. 1092 updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE); 1093 1094 // Start recording 1095 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1096 if (!startSharedRecording(/* useMediaRecorder */true, resultListener, 1097 /*useVideoStab*/false, fpsRange)) { 1098 mMediaRecorder.reset(); 1099 continue; 1100 } 1101 1102 // Record certain duration. 1103 SystemClock.sleep(RECORDING_DURATION_MS); 1104 1105 // Stop recording and preview 1106 stopRecording(/* useMediaRecorder */true); 1107 // Convert number of frames camera produced into the duration in unit of ms. 1108 float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE; 1109 float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs; 1110 float durationMaxMs = durationMinMs; 1111 float frameDurationMaxMs = 0.f; 1112 if (fpsRange != null) { 1113 frameDurationMaxMs = 1000.0f / fpsRange.getLower(); 1114 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs; 1115 } 1116 1117 // Validation. 1118 validateRecording(sz, durationMinMs, durationMaxMs, 1119 frameDurationMinMs, frameDurationMaxMs, 1120 FRMDRP_RATE_TOLERANCE); 1121 1122 break; 1123 } 1124 } 1125 1126 /** 1127 * Test slow motion recording where capture rate (camera output) is different with 1128 * video (playback) frame rate for each camera if high speed recording is supported 1129 * by both camera and encoder. 1130 * 1131 * <p> 1132 * Normal recording use cases make the capture rate (camera output frame 1133 * rate) the same as the video (playback) frame rate. This guarantees that 1134 * the motions in the scene play at the normal speed. If the capture rate is 1135 * faster than video frame rate, for a given time duration, more number of 1136 * frames are captured than it can be played in the same time duration. This 1137 * generates "slow motion" effect during playback. 1138 * </p> 1139 */ slowMotionRecording()1140 private void slowMotionRecording() throws Exception { 1141 for (String id : getCameraIdsUnderTest()) { 1142 try { 1143 Log.i(TAG, "Testing slow motion recording for camera " + id); 1144 StaticMetadata staticInfo = mAllStaticInfo.get(id); 1145 if (!staticInfo.isColorOutputSupported()) { 1146 Log.i(TAG, "Camera " + id + 1147 " does not support color outputs, skipping"); 1148 continue; 1149 } 1150 if (!staticInfo.isHighSpeedVideoSupported()) { 1151 continue; 1152 } 1153 1154 // Re-use the MediaRecorder object for the same camera device. 1155 mMediaRecorder = new MediaRecorder(); 1156 openDevice(id); 1157 1158 StreamConfigurationMap config = 1159 mStaticInfo.getValueFromKeyNonNull( 1160 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1161 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 1162 for (Size size : highSpeedVideoSizes) { 1163 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size); 1164 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " + 1165 "size " + size, fpsRange); 1166 if (fpsRange == null) { 1167 continue; 1168 } 1169 1170 int captureRate = fpsRange.getLower(); 1171 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR; 1172 // Skip the test if the highest recording FPS supported by CamcorderProfile 1173 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 1174 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 1175 + " is not supported by CamcorderProfile"); 1176 continue; 1177 } 1178 1179 mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video.mp4"; 1180 if (DEBUG_DUMP) { 1181 mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video_" + id + "_" 1182 + size.toString() + ".mp4"; 1183 } 1184 1185 prepareRecording(size, videoFramerate, captureRate); 1186 1187 // prepare preview surface by using video size. 1188 updatePreviewSurfaceWithVideo(size, captureRate); 1189 1190 // Start recording 1191 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1192 startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate, 1193 fpsRange, resultListener, /*useHighSpeedSession*/false); 1194 1195 // Record certain duration. 1196 SystemClock.sleep(RECORDING_DURATION_MS); 1197 1198 // Stop recording and preview 1199 stopRecording(/*useMediaRecorder*/true); 1200 // Convert number of frames camera produced into the duration in unit of ms. 1201 float frameDurationMs = 1000.0f / videoFramerate; 1202 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 1203 1204 // Validation. 1205 validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 1206 } 1207 1208 } finally { 1209 closeDevice(); 1210 releaseRecorder(); 1211 } 1212 } 1213 } 1214 constrainedHighSpeedRecording()1215 private void constrainedHighSpeedRecording() throws Exception { 1216 for (String id : getCameraIdsUnderTest()) { 1217 try { 1218 Log.i(TAG, "Testing constrained high speed recording for camera " + id); 1219 1220 if (!mAllStaticInfo.get(id).isConstrainedHighSpeedVideoSupported()) { 1221 Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping."); 1222 continue; 1223 } 1224 1225 // Re-use the MediaRecorder object for the same camera device. 1226 mMediaRecorder = new MediaRecorder(); 1227 openDevice(id); 1228 1229 StreamConfigurationMap config = 1230 mStaticInfo.getValueFromKeyNonNull( 1231 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 1232 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes(); 1233 Log.v(TAG, "highSpeedVideoSizes:" + Arrays.toString(highSpeedVideoSizes)); 1234 int previewFrameRate = Integer.MAX_VALUE; 1235 for (Size size : highSpeedVideoSizes) { 1236 List<Range<Integer>> fixedFpsRanges = 1237 getHighSpeedFixedFpsRangeForSize(config, size); 1238 Range<Integer>[] highSpeedFpsRangesForSize = 1239 config.getHighSpeedVideoFpsRangesFor(size); 1240 1241 Log.v(TAG, "highSpeedFpsRangesForSize for size - " + size + " : " + 1242 Arrays.toString(highSpeedFpsRangesForSize)); 1243 // Map to store max_fps and preview fps for each video size 1244 HashMap<Integer, Integer> previewRateMap = new HashMap(); 1245 for (Range<Integer> r : highSpeedFpsRangesForSize ) { 1246 if (!Objects.equals(r.getLower(), r.getUpper())) { 1247 if (previewRateMap.containsKey(r.getUpper())) { 1248 Log.w(TAG, "previewFps for max_fps already exists."); 1249 } else { 1250 previewRateMap.put(r.getUpper(), r.getLower()); 1251 } 1252 } 1253 } 1254 1255 mCollector.expectTrue("Unable to find the fixed frame rate fps range for " + 1256 "size " + size, fixedFpsRanges.size() > 0); 1257 // Test recording for each FPS range 1258 for (Range<Integer> fpsRange : fixedFpsRanges) { 1259 int captureRate = fpsRange.getLower(); 1260 previewFrameRate = previewRateMap.get(captureRate); 1261 Log.v(TAG, "previewFrameRate: " + previewFrameRate + " captureRate: " + 1262 captureRate); 1263 1264 Range<Integer> previewfpsRange = 1265 new Range<Integer>(previewFrameRate, captureRate); 1266 1267 // Skip the test if the highest recording FPS supported by CamcorderProfile 1268 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) { 1269 Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps" 1270 + " is not supported by CamcorderProfile"); 1271 continue; 1272 } 1273 1274 SimpleCaptureCallback previewResultListener = new SimpleCaptureCallback(); 1275 1276 // prepare preview surface by using video size. 1277 updatePreviewSurfaceWithVideo(size, captureRate); 1278 1279 startConstrainedPreview(previewfpsRange, previewResultListener); 1280 1281 mOutMediaFileName = mDebugFileNameBase + "/test_cslowMo_video_" + 1282 captureRate + "fps_" + id + "_" + size.toString() + ".mp4"; 1283 1284 // b/239101664 It appears that video frame rates higher than 30 fps may not 1285 // trigger slow motion recording consistently. 1286 int videoFrameRate = previewFrameRate > VIDEO_FRAME_RATE ? 1287 VIDEO_FRAME_RATE : previewFrameRate; 1288 Log.v(TAG, "videoFrameRate:" + videoFrameRate); 1289 1290 int cameraId = Integer.valueOf(mCamera.getId()); 1291 int videoEncoder = MediaRecorder.VideoEncoder.H264; 1292 for (int profileId : mCamcorderProfileList) { 1293 if (CamcorderProfile.hasProfile(cameraId, profileId)) { 1294 CamcorderProfile profile = 1295 CamcorderProfile.get(cameraId, profileId); 1296 1297 if (profile.videoFrameHeight == size.getHeight() && 1298 profile.videoFrameWidth == size.getWidth() && 1299 profile.videoFrameRate == videoFrameRate) { 1300 videoEncoder = profile.videoCodec; 1301 // Since mCamcorderProfileList is a list representing different 1302 // resolutions, we can break when a profile with the same 1303 // dimensions as size is found 1304 break; 1305 } 1306 } 1307 } 1308 1309 prepareRecording(size, videoFrameRate, captureRate, videoEncoder); 1310 1311 SystemClock.sleep(PREVIEW_DURATION_MS); 1312 1313 stopCameraStreaming(); 1314 1315 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1316 // Start recording 1317 startSlowMotionRecording(/*useMediaRecorder*/true, videoFrameRate, 1318 captureRate, fpsRange, resultListener, 1319 /*useHighSpeedSession*/true); 1320 1321 // Record certain duration. 1322 SystemClock.sleep(RECORDING_DURATION_MS); 1323 1324 // Stop recording and preview 1325 stopRecording(/*useMediaRecorder*/true); 1326 1327 startConstrainedPreview(previewfpsRange, previewResultListener); 1328 1329 // Convert number of frames camera produced into the duration in unit of ms. 1330 float frameDurationMs = 1000.0f / videoFrameRate; 1331 float durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 1332 1333 // Validation. 1334 validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 1335 1336 SystemClock.sleep(PREVIEW_DURATION_MS); 1337 1338 stopCameraStreaming(); 1339 } 1340 } 1341 } catch (NumberFormatException e) { 1342 fail("Cannot convert cameraId " + mCamera.getId() + " to int"); 1343 } finally { 1344 closeDevice(); 1345 releaseRecorder(); 1346 } 1347 } 1348 } 1349 1350 /** 1351 * Get high speed FPS from CamcorderProfiles for a given size. 1352 * 1353 * @param size The size used to search the CamcorderProfiles for the FPS. 1354 * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles. 1355 */ getFpsFromHighSpeedProfileForSize(Size size)1356 private int getFpsFromHighSpeedProfileForSize(Size size) { 1357 for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P; 1358 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) { 1359 if (CamcorderProfile.hasProfile(quality)) { 1360 CamcorderProfile profile = CamcorderProfile.get(quality); 1361 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){ 1362 return profile.videoFrameRate; 1363 } 1364 } 1365 } 1366 1367 return 0; 1368 } 1369 getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)1370 private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 1371 Size size) { 1372 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 1373 Range<Integer> maxRange = availableFpsRanges[0]; 1374 boolean foundRange = false; 1375 for (Range<Integer> range : availableFpsRanges) { 1376 if (range.getLower().equals(range.getUpper()) && range.getLower() >= maxRange.getLower()) { 1377 foundRange = true; 1378 maxRange = range; 1379 } 1380 } 1381 1382 if (!foundRange) { 1383 return null; 1384 } 1385 return maxRange; 1386 } 1387 getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)1388 private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, 1389 Size size) { 1390 Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size); 1391 List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>(); 1392 for (Range<Integer> range : availableFpsRanges) { 1393 if (range.getLower().equals(range.getUpper())) { 1394 fixedRanges.add(range); 1395 } 1396 } 1397 return fixedRanges; 1398 } 1399 startConstrainedPreview(Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener)1400 private void startConstrainedPreview(Range<Integer> fpsRange, 1401 CameraCaptureSession.CaptureCallback listener) throws Exception { 1402 List<Surface> outputSurfaces = new ArrayList<Surface>(1); 1403 assertTrue("Preview surface should be valid", mPreviewSurface.isValid()); 1404 outputSurfaces.add(mPreviewSurface); 1405 mSessionListener = new BlockingSessionCallback(); 1406 1407 List<CaptureRequest> slowMoRequests = null; 1408 CaptureRequest.Builder requestBuilder = 1409 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1410 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1411 requestBuilder.addTarget(mPreviewSurface); 1412 CaptureRequest initialRequest = requestBuilder.build(); 1413 CameraTestUtils.checkSessionConfigurationWithSurfaces(mCamera, mHandler, 1414 outputSurfaces, /*inputConfig*/ null, SessionConfiguration.SESSION_HIGH_SPEED, 1415 mCameraManager, /*defaultSupport*/ true, 1416 "Constrained session configuration query failed"); 1417 mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener, 1418 mHandler, initialRequest); 1419 slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession). 1420 createHighSpeedRequestList(initialRequest); 1421 1422 mSession.setRepeatingBurst(slowMoRequests, listener, mHandler); 1423 } 1424 startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, int captureRate, Range<Integer> fpsRange, SimpleCaptureCallback listener, boolean useHighSpeedSession)1425 private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, 1426 int captureRate, Range<Integer> fpsRange, 1427 SimpleCaptureCallback listener, boolean useHighSpeedSession) 1428 throws Exception { 1429 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 1430 assertTrue("Both preview and recording surfaces should be valid", 1431 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 1432 outputSurfaces.add(mPreviewSurface); 1433 outputSurfaces.add(mRecordingSurface); 1434 // Video snapshot surface 1435 if (mReaderSurface != null) { 1436 outputSurfaces.add(mReaderSurface); 1437 } 1438 mSessionListener = new BlockingSessionCallback(); 1439 1440 // Create slow motion request list 1441 List<CaptureRequest> slowMoRequests = null; 1442 if (useHighSpeedSession) { 1443 CaptureRequest.Builder requestBuilder = 1444 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1445 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1446 requestBuilder.addTarget(mPreviewSurface); 1447 requestBuilder.addTarget(mRecordingSurface); 1448 CaptureRequest initialRequest = requestBuilder.build(); 1449 mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener, 1450 mHandler, initialRequest); 1451 slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession). 1452 createHighSpeedRequestList(initialRequest); 1453 } else { 1454 CaptureRequest.Builder recordingRequestBuilder = 1455 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1456 recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE, 1457 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 1458 recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 1459 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 1460 1461 CaptureRequest.Builder recordingOnlyBuilder = 1462 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 1463 recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE, 1464 CaptureRequest.CONTROL_MODE_USE_SCENE_MODE); 1465 recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE, 1466 CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO); 1467 int slowMotionFactor = captureRate / videoFrameRate; 1468 1469 // Make sure camera output frame rate is set to correct value. 1470 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1471 recordingRequestBuilder.addTarget(mRecordingSurface); 1472 recordingRequestBuilder.addTarget(mPreviewSurface); 1473 recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 1474 recordingOnlyBuilder.addTarget(mRecordingSurface); 1475 1476 CaptureRequest initialRequest = recordingRequestBuilder.build(); 1477 mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces, 1478 mSessionListener, mHandler, initialRequest); 1479 1480 slowMoRequests = new ArrayList<CaptureRequest>(); 1481 slowMoRequests.add(initialRequest);// Preview + recording. 1482 1483 for (int i = 0; i < slowMotionFactor - 1; i++) { 1484 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only. 1485 } 1486 } 1487 1488 mSession.setRepeatingBurst(slowMoRequests, listener, mHandler); 1489 1490 if (useMediaRecorder) { 1491 // Wait for the first capture start before starting mediaRecorder 1492 listener.getCaptureStartTimestamps(1); 1493 mMediaRecorder.start(); 1494 } else { 1495 // TODO: need implement MediaCodec path. 1496 } 1497 1498 } 1499 basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)1500 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab) 1501 throws Exception { 1502 basicRecordingTestByCamera(camcorderProfileList, useVideoStab, false); 1503 } 1504 basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface)1505 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, 1506 boolean useIntermediateSurface) throws Exception { 1507 basicRecordingTestByCamera(camcorderProfileList, useVideoStab, 1508 useIntermediateSurface, false); 1509 } 1510 1511 /** 1512 * Test camera recording by using each available CamcorderProfile for a 1513 * given camera. preview size is set to the video size. 1514 */ basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)1515 private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, 1516 boolean useIntermediateSurface, boolean useEncoderProfiles) throws Exception { 1517 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 1518 List<Range<Integer> > fpsRanges = Arrays.asList( 1519 mStaticInfo.getAeAvailableTargetFpsRangesChecked()); 1520 int cameraId = Integer.valueOf(mCamera.getId()); 1521 int maxVideoFrameRate = -1; 1522 1523 // only validate recording for non-perf measurement runs 1524 boolean validateRecording = !isPerfMeasure(); 1525 for (int profileId : camcorderProfileList) { 1526 if (!CamcorderProfile.hasProfile(cameraId, profileId)) { 1527 continue; 1528 } 1529 1530 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 1531 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 1532 1533 Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate); 1534 if (maxVideoFrameRate < profile.videoFrameRate) { 1535 maxVideoFrameRate = profile.videoFrameRate; 1536 } 1537 1538 if (allowedUnsupported(cameraId, profileId)) { 1539 continue; 1540 } 1541 1542 if (mStaticInfo.isHardwareLevelLegacy() && 1543 (videoSz.getWidth() > maxPreviewSize.getWidth() || 1544 videoSz.getHeight() > maxPreviewSize.getHeight())) { 1545 // Skip. Legacy mode can only do recording up to max preview size 1546 continue; 1547 } 1548 assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId + 1549 " must be one of the camera device supported video size!", 1550 mSupportedVideoSizes.contains(videoSz)); 1551 assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId + 1552 ") must be one of the camera device available FPS range!", 1553 fpsRanges.contains(fpsRange)); 1554 1555 1556 if (useEncoderProfiles) { 1557 // Iterate through all video-audio codec combination 1558 EncoderProfiles profiles = CamcorderProfile.getAll(mCamera.getId(), profileId); 1559 for (EncoderProfiles.VideoProfile videoProfile : profiles.getVideoProfiles()) { 1560 boolean hasAudioProfile = false; 1561 for (EncoderProfiles.AudioProfile audioProfile : profiles.getAudioProfiles()) { 1562 hasAudioProfile = true; 1563 doBasicRecordingByProfile(profiles, videoProfile, audioProfile, 1564 useVideoStab, useIntermediateSurface, validateRecording); 1565 // Only measure the default video profile of the largest video 1566 // recording size when measuring perf 1567 if (isPerfMeasure()) { 1568 break; 1569 } 1570 } 1571 // Timelapse profiles do not have audio track 1572 if (!hasAudioProfile) { 1573 doBasicRecordingByProfile(profiles, videoProfile, /* audioProfile */null, 1574 useVideoStab, useIntermediateSurface, validateRecording); 1575 } 1576 } 1577 } else { 1578 doBasicRecordingByProfile( 1579 profile, useVideoStab, useIntermediateSurface, validateRecording); 1580 } 1581 1582 if (isPerfMeasure()) { 1583 // Only measure the largest video recording size when measuring perf 1584 break; 1585 } 1586 } 1587 if (maxVideoFrameRate != -1) { 1588 // At least one CamcorderProfile is present, check FPS 1589 assertTrue("At least one CamcorderProfile must support >= 24 FPS", 1590 maxVideoFrameRate >= 24); 1591 } 1592 } 1593 doBasicRecordingByProfile( CamcorderProfile profile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1594 private void doBasicRecordingByProfile( 1595 CamcorderProfile profile, boolean userVideoStab, 1596 boolean useIntermediateSurface, boolean validate) throws Exception { 1597 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 1598 int frameRate = profile.videoFrameRate; 1599 1600 if (VERBOSE) { 1601 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 1602 } 1603 1604 // Configure preview and recording surfaces. 1605 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 1606 if (DEBUG_DUMP) { 1607 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_" 1608 + videoSz.toString() + ".mp4"; 1609 } 1610 1611 setupMediaRecorder(profile); 1612 completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate); 1613 } 1614 doBasicRecordingByProfile( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1615 private void doBasicRecordingByProfile( 1616 EncoderProfiles profiles, 1617 EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile, 1618 boolean userVideoStab, boolean useIntermediateSurface, boolean validate) 1619 throws Exception { 1620 Size videoSz = new Size(videoProfile.getWidth(), videoProfile.getHeight()); 1621 int frameRate = videoProfile.getFrameRate(); 1622 1623 if (VERBOSE) { 1624 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString() + 1625 ", video codec " + videoProfile.getMediaType() + ", and audio codec " + 1626 (audioProfile == null ? "(null)" : audioProfile.getMediaType())); 1627 } 1628 1629 // Configure preview and recording surfaces. 1630 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 1631 if (DEBUG_DUMP) { 1632 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_" 1633 + videoSz.toString() + "_" + videoProfile.getCodec(); 1634 if (audioProfile != null) { 1635 mOutMediaFileName += "_" + audioProfile.getCodec(); 1636 } 1637 mOutMediaFileName += ".mp4"; 1638 } 1639 1640 setupMediaRecorder(profiles, videoProfile, audioProfile); 1641 completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate); 1642 } 1643 completeBasicRecording( Size videoSz, int frameRate, boolean useVideoStab, boolean useIntermediateSurface, boolean validate)1644 private void completeBasicRecording( 1645 Size videoSz, int frameRate, boolean useVideoStab, 1646 boolean useIntermediateSurface, boolean validate) throws Exception { 1647 prepareRecording(useIntermediateSurface); 1648 1649 // prepare preview surface by using video size. 1650 updatePreviewSurfaceWithVideo(videoSz, frameRate); 1651 1652 // Start recording 1653 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1654 startRecording(/* useMediaRecorder */true, resultListener, useVideoStab, 1655 useIntermediateSurface); 1656 1657 // Record certain duration. 1658 SystemClock.sleep(RECORDING_DURATION_MS); 1659 1660 // Stop recording and preview 1661 stopRecording(/* useMediaRecorder */true, useIntermediateSurface, 1662 /* stopCameraStreaming */true); 1663 // Convert number of frames camera produced into the duration in unit of ms. 1664 float frameDurationMs = 1000.0f / frameRate; 1665 float durationMs = 0.f; 1666 if (useIntermediateSurface) { 1667 durationMs = mQueuer.getQueuedCount() * frameDurationMs; 1668 } else { 1669 durationMs = resultListener.getTotalNumFrames() * frameDurationMs; 1670 } 1671 1672 if (VERBOSE) { 1673 Log.v(TAG, "video frame rate: " + frameRate + 1674 ", num of frames produced: " + resultListener.getTotalNumFrames()); 1675 } 1676 1677 if (validate) { 1678 validateRecording(videoSz, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE); 1679 } 1680 } 1681 1682 /** 1683 * Test camera recording for each supported video size by camera, preview 1684 * size is set to the video size. 1685 */ recordingSizeTestByCamera()1686 private void recordingSizeTestByCamera() throws Exception { 1687 for (Size sz : mSupportedVideoSizes) { 1688 if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) { 1689 continue; 1690 } 1691 1692 if (VERBOSE) { 1693 Log.v(TAG, "Testing camera recording with video size " + sz.toString()); 1694 } 1695 1696 // Configure preview and recording surfaces. 1697 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 1698 if (DEBUG_DUMP) { 1699 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_" 1700 + sz.toString() + ".mp4"; 1701 } 1702 1703 // Allow external camera to use variable fps range 1704 Range<Integer> fpsRange = null; 1705 if (mStaticInfo.isExternalCamera()) { 1706 Range<Integer>[] availableFpsRange = 1707 mStaticInfo.getAeAvailableTargetFpsRangesChecked(); 1708 1709 boolean foundRange = false; 1710 int minFps = 0; 1711 for (int i = 0; i < availableFpsRange.length; i += 1) { 1712 if (minFps < availableFpsRange[i].getLower() 1713 && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) { 1714 minFps = availableFpsRange[i].getLower(); 1715 foundRange = true; 1716 } 1717 } 1718 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange); 1719 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE); 1720 } 1721 1722 // Use AVC and AAC a/v compression format. 1723 prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE); 1724 1725 // prepare preview surface by using video size. 1726 updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE); 1727 1728 // Start recording 1729 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 1730 startRecording( 1731 /* useMediaRecorder */true, resultListener, 1732 /*useVideoStab*/false, fpsRange, false); 1733 1734 // Record certain duration. 1735 SystemClock.sleep(RECORDING_DURATION_MS); 1736 1737 // Stop recording and preview 1738 stopRecording(/* useMediaRecorder */true); 1739 // Convert number of frames camera produced into the duration in unit of ms. 1740 float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE; 1741 float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs; 1742 float durationMaxMs = durationMinMs; 1743 float frameDurationMaxMs = 0.f; 1744 if (fpsRange != null) { 1745 frameDurationMaxMs = 1000.0f / fpsRange.getLower(); 1746 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs; 1747 } 1748 1749 // Validation. 1750 validateRecording(sz, durationMinMs, durationMaxMs, 1751 frameDurationMinMs, frameDurationMaxMs, 1752 FRMDRP_RATE_TOLERANCE); 1753 } 1754 } 1755 1756 /** 1757 * Initialize the supported video sizes. 1758 */ initSupportedVideoSize(String cameraId)1759 private void initSupportedVideoSize(String cameraId) throws Exception { 1760 int id = Integer.valueOf(cameraId); 1761 Size maxVideoSize = SIZE_BOUND_720P; 1762 if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_2160P)) { 1763 maxVideoSize = SIZE_BOUND_2160P; 1764 } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_QHD)) { 1765 maxVideoSize = SIZE_BOUND_QHD; 1766 } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_2K)) { 1767 maxVideoSize = SIZE_BOUND_2K; 1768 } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_1080P)) { 1769 maxVideoSize = SIZE_BOUND_1080P; 1770 } 1771 1772 mSupportedVideoSizes = 1773 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize); 1774 } 1775 1776 /** 1777 * Simple wrapper to wrap normal/burst video snapshot tests 1778 */ videoSnapshotHelper(boolean burstTest)1779 private void videoSnapshotHelper(boolean burstTest) throws Exception { 1780 for (String id : getCameraIdsUnderTest()) { 1781 try { 1782 Log.i(TAG, "Testing video snapshot for camera " + id); 1783 1784 StaticMetadata staticInfo = mAllStaticInfo.get(id); 1785 if (!staticInfo.isColorOutputSupported()) { 1786 Log.i(TAG, "Camera " + id + 1787 " does not support color outputs, skipping"); 1788 continue; 1789 } 1790 1791 if (staticInfo.isExternalCamera()) { 1792 Log.i(TAG, "Camera " + id + 1793 " does not support CamcorderProfile, skipping"); 1794 continue; 1795 } 1796 1797 // Re-use the MediaRecorder object for the same camera device. 1798 mMediaRecorder = new MediaRecorder(); 1799 1800 openDevice(id); 1801 1802 initSupportedVideoSize(id); 1803 1804 videoSnapshotTestByCamera(burstTest); 1805 } finally { 1806 closeDevice(); 1807 releaseRecorder(); 1808 } 1809 } 1810 } 1811 1812 /** 1813 * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported. 1814 * 1815 * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p> 1816 * 1817 * @param profileId a {@link CamcorderProfile} ID to check. 1818 * @return {@code true} if supported. 1819 */ allowedUnsupported(int cameraId, int profileId)1820 private boolean allowedUnsupported(int cameraId, int profileId) { 1821 if (!mStaticInfo.isHardwareLevelLegacy()) { 1822 return false; 1823 } 1824 1825 switch(profileId) { 1826 case CamcorderProfile.QUALITY_2160P: 1827 case CamcorderProfile.QUALITY_1080P: 1828 case CamcorderProfile.QUALITY_HIGH: 1829 return !CamcorderProfile.hasProfile(cameraId, profileId) || 1830 CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080; 1831 } 1832 return false; 1833 } 1834 1835 /** 1836 * Test video snapshot for each available CamcorderProfile for a given camera. 1837 * 1838 * <p> 1839 * Preview size is set to the video size. For the burst test, frame drop and jittering 1840 * is not checked. 1841 * </p> 1842 * 1843 * @param burstTest Perform burst capture or single capture. For burst capture 1844 * {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent. 1845 */ videoSnapshotTestByCamera(boolean burstTest)1846 private void videoSnapshotTestByCamera(boolean burstTest) 1847 throws Exception { 1848 final int NUM_SINGLE_SHOT_TEST = 5; 1849 final int FRAMEDROP_TOLERANCE = 8; 1850 final int FRAME_SIZE_15M = 15000000; 1851 final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f; 1852 int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE; 1853 1854 for (int profileId : mCamcorderProfileList) { 1855 int cameraId = Integer.valueOf(mCamera.getId()); 1856 if (!CamcorderProfile.hasProfile(cameraId, profileId) || 1857 allowedUnsupported(cameraId, profileId)) { 1858 continue; 1859 } 1860 1861 CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId); 1862 Size QCIF = new Size(176, 144); 1863 Size FULL_HD = new Size(1920, 1080); 1864 Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 1865 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 1866 1867 if (mStaticInfo.isHardwareLevelLegacy() && 1868 (videoSz.getWidth() > maxPreviewSize.getWidth() || 1869 videoSz.getHeight() > maxPreviewSize.getHeight())) { 1870 // Skip. Legacy mode can only do recording up to max preview size 1871 continue; 1872 } 1873 1874 if (!mSupportedVideoSizes.contains(videoSz)) { 1875 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " + 1876 profileId + " must be one of the camera device supported video size!"); 1877 continue; 1878 } 1879 1880 // For LEGACY, find closest supported smaller or equal JPEG size to the current video 1881 // size; if no size is smaller than the video, pick the smallest JPEG size. The assert 1882 // for video size above guarantees that for LIMITED or FULL, we select videoSz here. 1883 // Also check for minFrameDuration here to make sure jpeg stream won't slow down 1884 // video capture 1885 Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1); 1886 // Allow a bit tolerance so we don't fail for a few nano seconds of difference 1887 final float FRAME_DURATION_TOLERANCE = 0.01f; 1888 long videoFrameDuration = (long) (1e9 / profile.videoFrameRate * 1889 (1.0 + FRAME_DURATION_TOLERANCE)); 1890 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 1891 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG); 1892 for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) { 1893 Size candidateSize = mOrderedStillSizes.get(i); 1894 if (mStaticInfo.isHardwareLevelLegacy()) { 1895 // Legacy level doesn't report min frame duration 1896 if (candidateSize.getWidth() <= videoSz.getWidth() && 1897 candidateSize.getHeight() <= videoSz.getHeight()) { 1898 videoSnapshotSz = candidateSize; 1899 } 1900 } else { 1901 Long jpegFrameDuration = minFrameDurationMap.get(candidateSize); 1902 assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize, 1903 jpegFrameDuration != null); 1904 if (candidateSize.getWidth() <= videoSz.getWidth() && 1905 candidateSize.getHeight() <= videoSz.getHeight() && 1906 jpegFrameDuration <= videoFrameDuration) { 1907 videoSnapshotSz = candidateSize; 1908 } 1909 } 1910 } 1911 Size defaultvideoSnapshotSz = videoSnapshotSz; 1912 1913 /** 1914 * Only test full res snapshot when below conditions are all true. 1915 * 1. Camera is at least a LIMITED device. 1916 * 2. video size is up to max preview size, which will be bounded by 1080p. 1917 * 3. Full resolution jpeg stream can keep up to video stream speed. 1918 * When full res jpeg stream cannot keep up to video stream speed, search 1919 * the largest jpeg size that can susptain video speed instead. 1920 */ 1921 if (mStaticInfo.isHardwareLevelAtLeastLimited() && 1922 videoSz.getWidth() <= maxPreviewSize.getWidth() && 1923 videoSz.getHeight() <= maxPreviewSize.getHeight()) { 1924 for (Size jpegSize : mOrderedStillSizes) { 1925 Long jpegFrameDuration = minFrameDurationMap.get(jpegSize); 1926 assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize, 1927 jpegFrameDuration != null); 1928 if (jpegFrameDuration <= videoFrameDuration) { 1929 videoSnapshotSz = jpegSize; 1930 break; 1931 } 1932 if (jpegSize.equals(videoSz)) { 1933 throw new AssertionFailedError( 1934 "Cannot find adequate video snapshot size for video size" + 1935 videoSz); 1936 } 1937 } 1938 } 1939 1940 if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M) 1941 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR); 1942 1943 createImageReader( 1944 videoSnapshotSz, ImageFormat.JPEG, 1945 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 1946 1947 // Full or better devices should support whatever video snapshot size calculated above. 1948 // Limited devices may only be able to support the default one. 1949 if (mStaticInfo.isHardwareLevelLimited()) { 1950 List<Surface> outputs = new ArrayList<Surface>(); 1951 outputs.add(mPreviewSurface); 1952 outputs.add(mRecordingSurface); 1953 outputs.add(mReaderSurface); 1954 boolean isSupported = isStreamConfigurationSupported( 1955 mCamera, outputs, mSessionListener, mHandler); 1956 if (!isSupported) { 1957 videoSnapshotSz = defaultvideoSnapshotSz; 1958 createImageReader( 1959 videoSnapshotSz, ImageFormat.JPEG, 1960 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 1961 } 1962 } 1963 1964 if (videoSz.equals(QCIF) && 1965 ((videoSnapshotSz.getWidth() > FULL_HD.getWidth()) || 1966 (videoSnapshotSz.getHeight() > FULL_HD.getHeight()))) { 1967 List<Surface> outputs = new ArrayList<Surface>(); 1968 outputs.add(mPreviewSurface); 1969 outputs.add(mRecordingSurface); 1970 outputs.add(mReaderSurface); 1971 boolean isSupported = isStreamConfigurationSupported( 1972 mCamera, outputs, mSessionListener, mHandler); 1973 if (!isSupported) { 1974 videoSnapshotSz = defaultvideoSnapshotSz; 1975 createImageReader( 1976 videoSnapshotSz, ImageFormat.JPEG, 1977 MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null); 1978 } 1979 } 1980 1981 Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz + 1982 " for video size " + videoSz); 1983 1984 if (VERBOSE) { 1985 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString()); 1986 } 1987 1988 // Configure preview and recording surfaces. 1989 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4"; 1990 if (DEBUG_DUMP) { 1991 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + cameraId + "_" 1992 + videoSz.toString() + ".mp4"; 1993 } 1994 1995 int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST; 1996 int totalDroppedFrames = 0; 1997 1998 for (int numTested = 0; numTested < numTestIterations; numTested++) { 1999 prepareRecordingWithProfile(profile); 2000 2001 // prepare video snapshot 2002 SimpleCaptureCallback resultListener = new SimpleCaptureCallback(); 2003 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(); 2004 CaptureRequest.Builder videoSnapshotRequestBuilder = 2005 mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ? 2006 CameraDevice.TEMPLATE_RECORD : 2007 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT); 2008 2009 // prepare preview surface by using video size. 2010 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate); 2011 2012 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener); 2013 Range<Integer> fpsRange = Range.create(profile.videoFrameRate, 2014 profile.videoFrameRate); 2015 videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, 2016 fpsRange); 2017 boolean videoStabilizationSupported = mStaticInfo.isVideoStabilizationSupported(); 2018 if (videoStabilizationSupported) { 2019 videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 2020 mStaticInfo.getChosenVideoStabilizationMode()); 2021 } 2022 CaptureRequest request = videoSnapshotRequestBuilder.build(); 2023 2024 // Start recording 2025 startRecording(/* useMediaRecorder */true, resultListener, 2026 /*useVideoStab*/videoStabilizationSupported); 2027 long startTime = SystemClock.elapsedRealtime(); 2028 2029 // Record certain duration. 2030 SystemClock.sleep(RECORDING_DURATION_MS / 2); 2031 2032 // take video snapshot 2033 if (burstTest) { 2034 List<CaptureRequest> requests = 2035 new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM); 2036 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 2037 requests.add(request); 2038 } 2039 mSession.captureBurst(requests, resultListener, mHandler); 2040 } else { 2041 mSession.capture(request, resultListener, mHandler); 2042 } 2043 2044 // make sure recording is still going after video snapshot 2045 SystemClock.sleep(RECORDING_DURATION_MS / 2); 2046 2047 // Stop recording and preview 2048 float durationMs = (float) stopRecording(/* useMediaRecorder */true); 2049 // For non-burst test, use number of frames to also double check video frame rate. 2050 // Burst video snapshot is allowed to cause frame rate drop, so do not use number 2051 // of frames to estimate duration 2052 if (!burstTest) { 2053 durationMs = resultListener.getTotalNumFrames() * 1000.0f / 2054 profile.videoFrameRate; 2055 } 2056 2057 float frameDurationMs = 1000.0f / profile.videoFrameRate; 2058 // Validation recorded video 2059 validateRecording(videoSz, durationMs, 2060 frameDurationMs, VID_SNPSHT_FRMDRP_RATE_TOLERANCE); 2061 2062 if (burstTest) { 2063 for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) { 2064 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 2065 validateVideoSnapshotCapture(image, videoSnapshotSz); 2066 image.close(); 2067 } 2068 } else { 2069 // validate video snapshot image 2070 Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS); 2071 validateVideoSnapshotCapture(image, videoSnapshotSz); 2072 2073 // validate if there is framedrop around video snapshot 2074 totalDroppedFrames += validateFrameDropAroundVideoSnapshot( 2075 resultListener, image.getTimestamp()); 2076 2077 //TODO: validate jittering. Should move to PTS 2078 //validateJittering(resultListener); 2079 2080 image.close(); 2081 } 2082 } 2083 2084 if (!burstTest) { 2085 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " + 2086 "detected in %d trials is %d frames.", cameraId, videoSz.toString(), 2087 numTestIterations, totalDroppedFrames)); 2088 mCollector.expectLessOrEqual( 2089 String.format( 2090 "Camera %d Video size %s: Number of dropped frames %d must not" 2091 + " be larger than %d", 2092 cameraId, videoSz.toString(), totalDroppedFrames, 2093 kFrameDrop_Tolerence), 2094 kFrameDrop_Tolerence, totalDroppedFrames); 2095 } 2096 closeImageReader(); 2097 } 2098 } 2099 2100 /** 2101 * Configure video snapshot request according to the still capture size 2102 */ prepareVideoSnapshot( CaptureRequest.Builder requestBuilder, ImageReader.OnImageAvailableListener imageListener)2103 private void prepareVideoSnapshot( 2104 CaptureRequest.Builder requestBuilder, 2105 ImageReader.OnImageAvailableListener imageListener) 2106 throws Exception { 2107 mReader.setOnImageAvailableListener(imageListener, mHandler); 2108 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 2109 requestBuilder.addTarget(mRecordingSurface); 2110 assertNotNull("Preview surface must be non-null!", mPreviewSurface); 2111 requestBuilder.addTarget(mPreviewSurface); 2112 assertNotNull("Reader surface must be non-null!", mReaderSurface); 2113 requestBuilder.addTarget(mReaderSurface); 2114 } 2115 2116 /** 2117 * Find compatible preview sizes for video size and framerate. 2118 * 2119 * <p>Preview size will be capped with max preview size.</p> 2120 * 2121 * @param videoSize The video size used for preview. 2122 * @param videoFrameRate The video frame rate 2123 */ getPreviewSizesForVideo(Size videoSize, int videoFrameRate)2124 private List<Size> getPreviewSizesForVideo(Size videoSize, int videoFrameRate) { 2125 if (mOrderedPreviewSizes == null) { 2126 throw new IllegalStateException("supported preview size list is not initialized yet"); 2127 } 2128 final float FRAME_DURATION_TOLERANCE = 0.01f; 2129 long videoFrameDuration = (long) (1e9 / videoFrameRate * 2130 (1.0 + FRAME_DURATION_TOLERANCE)); 2131 HashMap<Size, Long> minFrameDurationMap = mStaticInfo. 2132 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE); 2133 Size maxPreviewSize = mOrderedPreviewSizes.get(0); 2134 ArrayList<Size> previewSizes = new ArrayList<>(); 2135 if (videoSize.getWidth() > maxPreviewSize.getWidth() || 2136 videoSize.getHeight() > maxPreviewSize.getHeight()) { 2137 for (Size s : mOrderedPreviewSizes) { 2138 Long frameDuration = minFrameDurationMap.get(s); 2139 if (mStaticInfo.isHardwareLevelLegacy()) { 2140 // Legacy doesn't report min frame duration 2141 frameDuration = new Long(0); 2142 } 2143 assertTrue("Cannot find minimum frame duration for private size" + s, 2144 frameDuration != null); 2145 if (frameDuration <= videoFrameDuration && 2146 s.getWidth() <= videoSize.getWidth() && 2147 s.getHeight() <= videoSize.getHeight()) { 2148 Log.v(TAG, "Add preview size " + s.toString() + " for video size " + 2149 videoSize.toString()); 2150 previewSizes.add(s); 2151 } 2152 } 2153 } 2154 2155 if (previewSizes.isEmpty()) { 2156 previewSizes.add(videoSize); 2157 } 2158 2159 return previewSizes; 2160 } 2161 2162 /** 2163 * Update preview size with video size. 2164 * 2165 * <p>Preview size will be capped with max preview size.</p> 2166 * 2167 * @param videoSize The video size used for preview. 2168 * @param videoFrameRate The video frame rate 2169 * 2170 */ updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate)2171 private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) { 2172 List<Size> previewSizes = getPreviewSizesForVideo(videoSize, videoFrameRate); 2173 updatePreviewSurface(previewSizes.get(0)); 2174 } 2175 prepareRecordingWithProfile(CamcorderProfile profile)2176 private void prepareRecordingWithProfile(CamcorderProfile profile) throws Exception { 2177 prepareRecordingWithProfile(profile, false); 2178 } 2179 2180 /** 2181 * Configure MediaRecorder recording session with CamcorderProfile, prepare 2182 * the recording surface. 2183 */ prepareRecordingWithProfile(CamcorderProfile profile, boolean useIntermediateSurface)2184 private void prepareRecordingWithProfile(CamcorderProfile profile, 2185 boolean useIntermediateSurface) throws Exception { 2186 // Prepare MediaRecorder. 2187 setupMediaRecorder(profile); 2188 prepareRecording(useIntermediateSurface); 2189 } 2190 setupMediaRecorder(CamcorderProfile profile)2191 private void setupMediaRecorder(CamcorderProfile profile) throws Exception { 2192 // Set-up MediaRecorder. 2193 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 2194 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 2195 mMediaRecorder.setProfile(profile); 2196 2197 mVideoFrameRate = profile.videoFrameRate; 2198 mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight); 2199 } 2200 setupMediaRecorder( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile)2201 private void setupMediaRecorder( 2202 EncoderProfiles profiles, 2203 EncoderProfiles.VideoProfile videoProfile, 2204 EncoderProfiles.AudioProfile audioProfile) throws Exception { 2205 // Set-up MediaRecorder. 2206 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 2207 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 2208 mMediaRecorder.setOutputFormat(profiles.getRecommendedFileFormat()); 2209 mMediaRecorder.setVideoProfile(videoProfile); 2210 if (audioProfile != null) { 2211 mMediaRecorder.setAudioProfile(audioProfile); 2212 } 2213 2214 mVideoFrameRate = videoProfile.getFrameRate(); 2215 mVideoSize = new Size(videoProfile.getWidth(), videoProfile.getHeight()); 2216 } 2217 prepareRecording(boolean useIntermediateSurface)2218 private void prepareRecording(boolean useIntermediateSurface) throws Exception { 2219 // Continue preparing MediaRecorder 2220 mMediaRecorder.setOutputFile(mOutMediaFileName); 2221 if (mPersistentSurface != null) { 2222 mMediaRecorder.setInputSurface(mPersistentSurface); 2223 mRecordingSurface = mPersistentSurface; 2224 } 2225 mMediaRecorder.prepare(); 2226 if (mPersistentSurface == null) { 2227 mRecordingSurface = mMediaRecorder.getSurface(); 2228 } 2229 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 2230 2231 if (useIntermediateSurface) { 2232 Optional<Long> usage = getSurfaceUsage(mRecordingSurface); 2233 mIntermediateReader = ImageReader.newInstance( 2234 mVideoSize.getWidth(), mVideoSize.getHeight(), 2235 ImageFormat.PRIVATE, /*maxImages*/3, 2236 usage.orElse(HardwareBuffer.USAGE_VIDEO_ENCODE)); 2237 2238 mIntermediateSurface = mIntermediateReader.getSurface(); 2239 mIntermediateWriter = ImageWriter.newInstance(mRecordingSurface, /*maxImages*/3, 2240 ImageFormat.PRIVATE); 2241 mQueuer = new ImageWriterQueuer(mIntermediateWriter); 2242 2243 mIntermediateThread = new HandlerThread(TAG); 2244 mIntermediateThread.start(); 2245 mIntermediateHandler = new Handler(mIntermediateThread.getLooper()); 2246 mIntermediateReader.setOnImageAvailableListener(mQueuer, mIntermediateHandler); 2247 } 2248 } 2249 2250 /** 2251 * Configure MediaRecorder recording session with CamcorderProfile, prepare 2252 * the recording surface. Use AVC for video compression, AAC for audio compression. 2253 * Both are required for android devices by android CDD. 2254 */ prepareRecording(Size sz, int videoFrameRate, int captureRate)2255 private void prepareRecording(Size sz, int videoFrameRate, int captureRate) 2256 throws Exception { 2257 // Prepare MediaRecorder. 2258 prepareRecording(sz, videoFrameRate, captureRate, MediaRecorder.VideoEncoder.H264); 2259 } 2260 2261 /** 2262 * Configure MediaRecorder recording session with CamcorderProfile, prepare 2263 * the recording surface. Use AAC for audio compression as required for 2264 * android devices by android CDD. 2265 */ prepareRecording(Size sz, int videoFrameRate, int captureRate, int videoEncoder)2266 private void prepareRecording(Size sz, int videoFrameRate, int captureRate, 2267 int videoEncoder) throws Exception { 2268 // Prepare MediaRecorder. 2269 mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER); 2270 mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE); 2271 mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); 2272 mMediaRecorder.setOutputFile(mOutMediaFileName); 2273 mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz)); 2274 mMediaRecorder.setVideoFrameRate(videoFrameRate); 2275 mMediaRecorder.setCaptureRate(captureRate); 2276 mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight()); 2277 mMediaRecorder.setVideoEncoder(videoEncoder); 2278 mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC); 2279 if (mPersistentSurface != null) { 2280 mMediaRecorder.setInputSurface(mPersistentSurface); 2281 mRecordingSurface = mPersistentSurface; 2282 } 2283 mMediaRecorder.prepare(); 2284 if (mPersistentSurface == null) { 2285 mRecordingSurface = mMediaRecorder.getSurface(); 2286 } 2287 assertNotNull("Recording surface must be non-null!", mRecordingSurface); 2288 mVideoFrameRate = videoFrameRate; 2289 mVideoSize = sz; 2290 } 2291 startRecording(boolean useMediaRecorder, SimpleCaptureCallback listener, boolean useVideoStab)2292 private void startRecording(boolean useMediaRecorder, 2293 SimpleCaptureCallback listener, boolean useVideoStab) throws Exception { 2294 startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null, 2295 /*useIntermediateSurface*/false); 2296 } 2297 startRecording(boolean useMediaRecorder, SimpleCaptureCallback listener, boolean useVideoStab, boolean useIntermediateSurface)2298 private void startRecording(boolean useMediaRecorder, 2299 SimpleCaptureCallback listener, boolean useVideoStab, 2300 boolean useIntermediateSurface) throws Exception { 2301 startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null, 2302 useIntermediateSurface); 2303 } 2304 startRecording(boolean useMediaRecorder, SimpleCaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange, boolean useIntermediateSurface)2305 private void startRecording(boolean useMediaRecorder, 2306 SimpleCaptureCallback listener, boolean useVideoStab, 2307 Range<Integer> variableFpsRange, boolean useIntermediateSurface) throws Exception { 2308 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 2309 throw new IllegalArgumentException("Video stabilization is not supported"); 2310 } 2311 2312 List<Surface> outputSurfaces = new ArrayList<Surface>(2); 2313 assertTrue("Both preview and recording surfaces should be valid", 2314 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 2315 outputSurfaces.add(mPreviewSurface); 2316 if (useIntermediateSurface) { 2317 outputSurfaces.add(mIntermediateSurface); 2318 } else { 2319 outputSurfaces.add(mRecordingSurface); 2320 } 2321 2322 // Video snapshot surface 2323 if (mReaderSurface != null) { 2324 outputSurfaces.add(mReaderSurface); 2325 } 2326 mSessionListener = new BlockingSessionCallback(); 2327 2328 CaptureRequest.Builder recordingRequestBuilder = 2329 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 2330 // Make sure camera output frame rate is set to correct value. 2331 Range<Integer> fpsRange = (variableFpsRange == null) ? 2332 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange; 2333 2334 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 2335 if (useVideoStab) { 2336 recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 2337 mStaticInfo.getChosenVideoStabilizationMode()); 2338 2339 } 2340 if (useIntermediateSurface) { 2341 recordingRequestBuilder.addTarget(mIntermediateSurface); 2342 if (mQueuer != null) { 2343 mQueuer.resetInvalidSurfaceFlag(); 2344 } 2345 } else { 2346 recordingRequestBuilder.addTarget(mRecordingSurface); 2347 } 2348 recordingRequestBuilder.addTarget(mPreviewSurface); 2349 CaptureRequest recordingRequest = recordingRequestBuilder.build(); 2350 mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces, mSessionListener, 2351 mHandler, recordingRequest); 2352 mSession.setRepeatingRequest(recordingRequest, listener, mHandler); 2353 2354 if (useMediaRecorder) { 2355 // Wait for the first capture start before starting mediaRecorder 2356 listener.getCaptureStartTimestamps(1); 2357 mMediaRecorder.start(); 2358 } else { 2359 // TODO: need implement MediaCodec path. 2360 } 2361 mRecordingStartTime = SystemClock.elapsedRealtime(); 2362 } 2363 2364 /** 2365 * Start video recording with preview and video surfaces sharing the same 2366 * camera stream. 2367 * 2368 * @return true if success, false if sharing is not supported. 2369 */ startSharedRecording(boolean useMediaRecorder, SimpleCaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange)2370 private boolean startSharedRecording(boolean useMediaRecorder, 2371 SimpleCaptureCallback listener, boolean useVideoStab, 2372 Range<Integer> variableFpsRange) throws Exception { 2373 if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) { 2374 throw new IllegalArgumentException("Video stabilization is not supported"); 2375 } 2376 2377 List<OutputConfiguration> outputConfigs = new ArrayList<OutputConfiguration>(2); 2378 assertTrue("Both preview and recording surfaces should be valid", 2379 mPreviewSurface.isValid() && mRecordingSurface.isValid()); 2380 OutputConfiguration sharedConfig = new OutputConfiguration(mPreviewSurface); 2381 sharedConfig.enableSurfaceSharing(); 2382 sharedConfig.addSurface(mRecordingSurface); 2383 outputConfigs.add(sharedConfig); 2384 2385 CaptureRequest.Builder recordingRequestBuilder = 2386 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD); 2387 // Make sure camera output frame rate is set to correct value. 2388 Range<Integer> fpsRange = (variableFpsRange == null) ? 2389 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange; 2390 recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange); 2391 if (useVideoStab) { 2392 recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, 2393 mStaticInfo.getChosenVideoStabilizationMode()); 2394 } 2395 CaptureRequest recordingRequest = recordingRequestBuilder.build(); 2396 2397 mSessionListener = new BlockingSessionCallback(); 2398 mSession = tryConfigureCameraSessionWithConfig(mCamera, outputConfigs, recordingRequest, 2399 mSessionListener, mHandler); 2400 2401 if (mSession == null) { 2402 Log.i(TAG, "Sharing between preview and video is not supported"); 2403 return false; 2404 } 2405 2406 recordingRequestBuilder.addTarget(mRecordingSurface); 2407 recordingRequestBuilder.addTarget(mPreviewSurface); 2408 mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler); 2409 2410 if (useMediaRecorder) { 2411 // Wait for the first capture start before starting mediaRecorder 2412 listener.getCaptureStartTimestamps(1); 2413 mMediaRecorder.start(); 2414 } else { 2415 // TODO: need implement MediaCodec path. 2416 } 2417 mRecordingStartTime = SystemClock.elapsedRealtime(); 2418 return true; 2419 } 2420 2421 stopCameraStreaming()2422 private void stopCameraStreaming() throws Exception { 2423 if (VERBOSE) { 2424 Log.v(TAG, "Stopping camera streaming and waiting for idle"); 2425 } 2426 // Stop repeating, wait for captures to complete, and disconnect from 2427 // surfaces 2428 mSession.close(); 2429 mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS); 2430 } 2431 stopRecording(boolean useMediaRecorder)2432 private int stopRecording(boolean useMediaRecorder) throws Exception { 2433 return stopRecording(useMediaRecorder, /*useIntermediateSurface*/false, 2434 /*stopStreaming*/true); 2435 } 2436 2437 // Stop recording and return the estimated video duration in milliseconds. stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface, boolean stopStreaming)2438 private int stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface, 2439 boolean stopStreaming) throws Exception { 2440 long stopRecordingTime = SystemClock.elapsedRealtime(); 2441 if (useMediaRecorder) { 2442 if (stopStreaming) { 2443 stopCameraStreaming(); 2444 } 2445 if (useIntermediateSurface) { 2446 mIntermediateReader.setOnImageAvailableListener(null, null); 2447 mQueuer.expectInvalidSurface(); 2448 } 2449 2450 mMediaRecorder.stop(); 2451 // Can reuse the MediaRecorder object after reset. 2452 mMediaRecorder.reset(); 2453 } else { 2454 // TODO: need implement MediaCodec path. 2455 } 2456 2457 if (useIntermediateSurface) { 2458 mIntermediateReader.close(); 2459 mQueuer.close(); 2460 mIntermediateWriter.close(); 2461 mIntermediateSurface.release(); 2462 mIntermediateReader = null; 2463 mIntermediateSurface = null; 2464 mIntermediateWriter = null; 2465 mIntermediateThread.quitSafely(); 2466 mIntermediateHandler = null; 2467 } 2468 2469 if (mPersistentSurface == null && mRecordingSurface != null) { 2470 mRecordingSurface.release(); 2471 mRecordingSurface = null; 2472 } 2473 return (int) (stopRecordingTime - mRecordingStartTime); 2474 } 2475 releaseRecorder()2476 private void releaseRecorder() { 2477 if (mMediaRecorder != null) { 2478 mMediaRecorder.release(); 2479 mMediaRecorder = null; 2480 } 2481 } 2482 validateRecording( Size sz, float expectedDurationMs, float expectedFrameDurationMs, float frameDropTolerance)2483 private void validateRecording( 2484 Size sz, float expectedDurationMs, float expectedFrameDurationMs, 2485 float frameDropTolerance) throws Exception { 2486 validateRecording(sz, 2487 expectedDurationMs, /*fixed FPS recording*/0.f, 2488 expectedFrameDurationMs, /*fixed FPS recording*/0.f, 2489 frameDropTolerance); 2490 } 2491 validateRecording( Size sz, float expectedDurationMinMs, float expectedDurationMaxMs, float expectedFrameDurationMinMs, float expectedFrameDurationMaxMs, float frameDropTolerance)2492 private void validateRecording( 2493 Size sz, 2494 float expectedDurationMinMs, // Min duration (maxFps) 2495 float expectedDurationMaxMs, // Max duration (minFps). 0.f for fixed fps recording 2496 float expectedFrameDurationMinMs, // maxFps 2497 float expectedFrameDurationMaxMs, // minFps. 0.f for fixed fps recording 2498 float frameDropTolerance) throws Exception { 2499 File outFile = new File(mOutMediaFileName); 2500 assertTrue("No video is recorded", outFile.exists()); 2501 float maxFrameDuration = expectedFrameDurationMinMs * (1.0f + FRAMEDURATION_MARGIN); 2502 if (expectedFrameDurationMaxMs > 0.f) { 2503 maxFrameDuration = expectedFrameDurationMaxMs * (1.0f + FRAMEDURATION_MARGIN); 2504 } 2505 2506 if (expectedDurationMaxMs == 0.f) { 2507 expectedDurationMaxMs = expectedDurationMinMs; 2508 } 2509 2510 MediaExtractor extractor = new MediaExtractor(); 2511 try { 2512 extractor.setDataSource(mOutMediaFileName); 2513 long durationUs = 0; 2514 int width = -1, height = -1; 2515 int numTracks = extractor.getTrackCount(); 2516 int selectedTrack = -1; 2517 final String VIDEO_MIME_TYPE = "video"; 2518 for (int i = 0; i < numTracks; i++) { 2519 MediaFormat format = extractor.getTrackFormat(i); 2520 String mime = format.getString(MediaFormat.KEY_MIME); 2521 if (mime.contains(VIDEO_MIME_TYPE)) { 2522 Log.i(TAG, "video format is: " + format.toString()); 2523 durationUs = format.getLong(MediaFormat.KEY_DURATION); 2524 width = format.getInteger(MediaFormat.KEY_WIDTH); 2525 height = format.getInteger(MediaFormat.KEY_HEIGHT); 2526 selectedTrack = i; 2527 extractor.selectTrack(i); 2528 break; 2529 } 2530 } 2531 if (selectedTrack < 0) { 2532 throw new AssertionFailedError( 2533 "Cannot find video track!"); 2534 } 2535 2536 Size videoSz = new Size(width, height); 2537 assertTrue("Video size doesn't match, expected " + sz.toString() + 2538 " got " + videoSz.toString(), videoSz.equals(sz)); 2539 float duration = (float) (durationUs / 1000); 2540 if (VERBOSE) { 2541 Log.v(TAG, String.format("Video duration: recorded %fms, expected [%f,%f]ms", 2542 duration, expectedDurationMinMs, expectedDurationMaxMs)); 2543 } 2544 2545 // Do rest of validation only for better-than-LEGACY devices 2546 if (mStaticInfo.isHardwareLevelLegacy()) return; 2547 2548 // TODO: Don't skip this one for video snapshot on LEGACY 2549 assertTrue(String.format( 2550 "Camera %s: Video duration doesn't match: recorded %fms, expected [%f,%f]ms.", 2551 mCamera.getId(), duration, 2552 expectedDurationMinMs * (1.f - DURATION_MARGIN), 2553 expectedDurationMaxMs * (1.f + DURATION_MARGIN)), 2554 duration > expectedDurationMinMs * (1.f - DURATION_MARGIN) && 2555 duration < expectedDurationMaxMs * (1.f + DURATION_MARGIN)); 2556 2557 // Check for framedrop 2558 long lastSampleUs = 0; 2559 int frameDropCount = 0; 2560 int expectedFrameCount = (int) (expectedDurationMinMs / expectedFrameDurationMinMs); 2561 ArrayList<Long> timestamps = new ArrayList<Long>(expectedFrameCount); 2562 while (true) { 2563 timestamps.add(extractor.getSampleTime()); 2564 if (!extractor.advance()) { 2565 break; 2566 } 2567 } 2568 Collections.sort(timestamps); 2569 long prevSampleUs = timestamps.get(0); 2570 for (int i = 1; i < timestamps.size(); i++) { 2571 long currentSampleUs = timestamps.get(i); 2572 float frameDurationMs = (float) (currentSampleUs - prevSampleUs) / 1000; 2573 if (frameDurationMs > maxFrameDuration) { 2574 Log.w(TAG, String.format( 2575 "Frame drop at %d: expectation %f, observed %f", 2576 i, expectedFrameDurationMinMs, frameDurationMs)); 2577 frameDropCount++; 2578 } 2579 prevSampleUs = currentSampleUs; 2580 } 2581 float frameDropRate = 100.f * frameDropCount / timestamps.size(); 2582 Log.i(TAG, String.format("Frame drop rate %d/%d (%f%%)", 2583 frameDropCount, timestamps.size(), frameDropRate)); 2584 assertTrue(String.format( 2585 "Camera %s: Video frame drop rate too high: %f%%, tolerance %f%%. " + 2586 "Video size: %s, expectedDuration [%f,%f], expectedFrameDuration %f, " + 2587 "frameDropCnt %d, frameCount %d", 2588 mCamera.getId(), frameDropRate, frameDropTolerance, 2589 sz.toString(), expectedDurationMinMs, expectedDurationMaxMs, 2590 expectedFrameDurationMinMs, frameDropCount, timestamps.size()), 2591 frameDropRate < frameDropTolerance); 2592 } finally { 2593 extractor.release(); 2594 if (!DEBUG_DUMP) { 2595 outFile.delete(); 2596 } 2597 } 2598 } 2599 2600 /** 2601 * Validate video snapshot capture image object validity and test. 2602 * 2603 * <p> Check for size, format and jpeg decoding</p> 2604 * 2605 * @param image The JPEG image to be verified. 2606 * @param size The JPEG capture size to be verified against. 2607 */ 2608 private void validateVideoSnapshotCapture(Image image, Size size) { 2609 CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(), 2610 ImageFormat.JPEG, /*filePath*/null); 2611 } 2612 2613 /** 2614 * Validate if video snapshot causes frame drop. 2615 * Here frame drop is defined as frame duration >= 2 * expected frame duration. 2616 * Return the estimated number of frames dropped during video snapshot 2617 */ 2618 private int validateFrameDropAroundVideoSnapshot( 2619 SimpleCaptureCallback resultListener, long imageTimeStamp) { 2620 double expectedDurationMs = 1000.0 / mVideoFrameRate; 2621 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 2622 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 2623 while (resultListener.hasMoreResults()) { 2624 CaptureResult currentResult = 2625 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 2626 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 2627 if (currentTS == imageTimeStamp) { 2628 // validate the timestamp before and after, then return 2629 CaptureResult nextResult = 2630 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 2631 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP); 2632 double durationMs = (currentTS - prevTS) / 1000000.0; 2633 int totalFramesDropped = 0; 2634 2635 // Snapshots in legacy mode pause the preview briefly. Skip the duration 2636 // requirements for legacy mode unless this is fixed. 2637 if (!mStaticInfo.isHardwareLevelLegacy()) { 2638 mCollector.expectTrue( 2639 String.format( 2640 "Video %dx%d Frame drop detected before video snapshot: " + 2641 "duration %.2fms (expected %.2fms)", 2642 mVideoSize.getWidth(), mVideoSize.getHeight(), 2643 durationMs, expectedDurationMs 2644 ), 2645 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 2646 ); 2647 // Log a warning is there is any frame drop detected. 2648 if (durationMs >= expectedDurationMs * 2) { 2649 Log.w(TAG, String.format( 2650 "Video %dx%d Frame drop detected before video snapshot: " + 2651 "duration %.2fms (expected %.2fms)", 2652 mVideoSize.getWidth(), mVideoSize.getHeight(), 2653 durationMs, expectedDurationMs 2654 )); 2655 } 2656 2657 durationMs = (nextTS - currentTS) / 1000000.0; 2658 mCollector.expectTrue( 2659 String.format( 2660 "Video %dx%d Frame drop detected after video snapshot: " + 2661 "duration %.2fms (expected %.2fms)", 2662 mVideoSize.getWidth(), mVideoSize.getHeight(), 2663 durationMs, expectedDurationMs 2664 ), 2665 durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED) 2666 ); 2667 // Log a warning is there is any frame drop detected. 2668 if (durationMs >= expectedDurationMs * 2) { 2669 Log.w(TAG, String.format( 2670 "Video %dx%d Frame drop detected after video snapshot: " + 2671 "duration %fms (expected %fms)", 2672 mVideoSize.getWidth(), mVideoSize.getHeight(), 2673 durationMs, expectedDurationMs 2674 )); 2675 } 2676 2677 double totalDurationMs = (nextTS - prevTS) / 1000000.0; 2678 // Minus 2 for the expected 2 frames interval 2679 totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2; 2680 if (totalFramesDropped < 0) { 2681 Log.w(TAG, "totalFrameDropped is " + totalFramesDropped + 2682 ". Video frame rate might be too fast."); 2683 } 2684 totalFramesDropped = Math.max(0, totalFramesDropped); 2685 } 2686 return totalFramesDropped; 2687 } 2688 prevTS = currentTS; 2689 } 2690 throw new AssertionFailedError( 2691 "Video snapshot timestamp does not match any of capture results!"); 2692 } 2693 2694 /** 2695 * Validate frame jittering from the input simple listener's buffered results 2696 */ 2697 private void validateJittering(SimpleCaptureCallback resultListener) { 2698 double expectedDurationMs = 1000.0 / mVideoFrameRate; 2699 CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 2700 long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP); 2701 while (resultListener.hasMoreResults()) { 2702 CaptureResult currentResult = 2703 resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS); 2704 long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP); 2705 double durationMs = (currentTS - prevTS) / 1000000.0; 2706 double durationError = Math.abs(durationMs - expectedDurationMs); 2707 long frameNumber = currentResult.getFrameNumber(); 2708 mCollector.expectTrue( 2709 String.format( 2710 "Resolution %dx%d Frame %d: jittering (%.2fms) exceeds bound [%.2fms,%.2fms]", 2711 mVideoSize.getWidth(), mVideoSize.getHeight(), 2712 frameNumber, durationMs, 2713 expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS, 2714 expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS), 2715 durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS); 2716 prevTS = currentTS; 2717 } 2718 } 2719 2720 /** 2721 * Calculate a video bit rate based on the size. The bit rate is scaled 2722 * based on ratio of video size to 1080p size. 2723 */ 2724 private int getVideoBitRate(Size sz) { 2725 int rate = BIT_RATE_1080P; 2726 float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080); 2727 rate = (int)(rate * scaleFactor); 2728 2729 // Clamp to the MIN, MAX range. 2730 return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate)); 2731 } 2732 2733 /** 2734 * Check if the encoder and camera are able to support this size and frame rate. 2735 * Assume the video compression format is AVC. 2736 */ 2737 private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception { 2738 // Check camera capability. 2739 if (!isSupportedByCamera(sz, captureRate)) { 2740 return false; 2741 } 2742 2743 // Check encode capability. 2744 if (!isSupportedByAVCEncoder(sz, encodingRate)){ 2745 return false; 2746 } 2747 2748 if(VERBOSE) { 2749 Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@" 2750 + getVideoBitRate(sz) / 1000 + "Kbps"); 2751 } 2752 2753 return true; 2754 } 2755 2756 private boolean isSupportedByCamera(Size sz, int frameRate) { 2757 // Check if camera can support this sz and frame rate combination. 2758 StreamConfigurationMap config = mStaticInfo. 2759 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); 2760 2761 long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz); 2762 if (minDuration == 0) { 2763 return false; 2764 } 2765 2766 int maxFrameRate = (int) (1e9f / minDuration); 2767 return maxFrameRate >= frameRate; 2768 } 2769 2770 /** 2771 * Check if encoder can support this size and frame rate combination by querying 2772 * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate 2773 * as the bit rates targeted in this test are well below the bit rate max value specified 2774 * by AVC specification for certain level. 2775 */ 2776 private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) { 2777 MediaFormat format = MediaFormat.createVideoFormat( 2778 MediaFormat.MIMETYPE_VIDEO_AVC, sz.getWidth(), sz.getHeight()); 2779 format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate); 2780 MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS); 2781 return mcl.findEncoderForFormat(format) != null; 2782 } 2783 2784 private static class ImageWriterQueuer implements ImageReader.OnImageAvailableListener { 2785 public ImageWriterQueuer(ImageWriter writer) { 2786 mWriter = writer; 2787 } 2788 2789 public void resetInvalidSurfaceFlag() { 2790 synchronized (mLock) { 2791 mExpectInvalidSurface = false; 2792 } 2793 } 2794 2795 // Indicate that the writer surface is about to get released 2796 // and become invalid. 2797 public void expectInvalidSurface() { 2798 // If we sync on 'mLock', we risk a possible deadlock 2799 // during 'mWriter.queueInputImage(image)' which is 2800 // called while the lock is held. 2801 mExpectInvalidSurface = true; 2802 } 2803 2804 @Override 2805 public void onImageAvailable(ImageReader reader) { 2806 Image image = null; 2807 try { 2808 image = reader.acquireNextImage(); 2809 } finally { 2810 synchronized (mLock) { 2811 if (image != null && mWriter != null) { 2812 try { 2813 mWriter.queueInputImage(image); 2814 mQueuedCount++; 2815 } catch (IllegalStateException e) { 2816 // Per API documentation ISE are possible 2817 // in case the writer surface is not valid. 2818 // Re-throw in case we have some other 2819 // unexpected ISE. 2820 if (mExpectInvalidSurface) { 2821 Log.d(TAG, "Invalid writer surface"); 2822 image.close(); 2823 } else { 2824 throw e; 2825 } 2826 } 2827 } else if (image != null) { 2828 image.close(); 2829 } 2830 } 2831 } 2832 } 2833 2834 public int getQueuedCount() { 2835 synchronized (mLock) { 2836 return mQueuedCount; 2837 } 2838 } 2839 2840 public void close() { 2841 synchronized (mLock) { 2842 mWriter = null; 2843 } 2844 } 2845 2846 private Object mLock = new Object(); 2847 private ImageWriter mWriter = null; 2848 private int mQueuedCount = 0; 2849 private boolean mExpectInvalidSurface = false; 2850 } 2851 } 2852