1 /*
2  * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache
3  * License, Version 2.0 (the "License"); you may not use this file except in
4  * compliance with the License. You may obtain a copy of the License at
5  * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
6  * or agreed to in writing, software distributed under the License is
7  * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
8  * KIND, either express or implied. See the License for the specific language
9  * governing permissions and limitations under the License.
10  */
11 
12 package android.hardware.camera2.cts;
13 
14 import static android.hardware.camera2.cts.CameraTestUtils.*;
15 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*;
16 
17 import android.graphics.ImageFormat;
18 import android.graphics.SurfaceTexture;
19 import android.hardware.camera2.CameraCharacteristics;
20 import android.hardware.camera2.CameraCaptureSession;
21 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
22 import android.hardware.camera2.CameraDevice;
23 import android.hardware.camera2.CaptureRequest;
24 import android.hardware.camera2.CaptureResult;
25 import android.hardware.camera2.cts.helpers.StaticMetadata;
26 import android.hardware.camera2.params.OutputConfiguration;
27 import android.hardware.camera2.params.SessionConfiguration;
28 import android.hardware.camera2.params.StreamConfigurationMap;
29 import android.hardware.HardwareBuffer;
30 import android.util.Size;
31 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase;
32 import android.media.CamcorderProfile;
33 import android.media.EncoderProfiles;
34 import android.media.MediaCodec;
35 import android.media.MediaCodecInfo;
36 import android.media.MediaCodecInfo.CodecCapabilities;
37 import android.media.MediaCodecInfo.CodecProfileLevel;
38 import android.media.Image;
39 import android.media.ImageReader;
40 import android.media.ImageWriter;
41 import android.media.MediaCodecList;
42 import android.media.MediaExtractor;
43 import android.media.MediaFormat;
44 import android.media.MediaRecorder;
45 import android.os.Environment;
46 import android.os.Handler;
47 import android.os.HandlerThread;
48 import android.os.SystemClock;
49 import android.test.suitebuilder.annotation.LargeTest;
50 import android.util.Log;
51 import android.util.Range;
52 import android.view.Surface;
53 
54 import com.android.compatibility.common.util.MediaUtils;
55 import com.android.ex.camera2.blocking.BlockingSessionCallback;
56 
57 import junit.framework.AssertionFailedError;
58 
59 import org.junit.runners.Parameterized;
60 import org.junit.runner.RunWith;
61 import org.junit.Test;
62 
63 import java.io.File;
64 import java.util.ArrayList;
65 import java.util.Arrays;
66 import java.util.Collections;
67 import java.util.List;
68 import java.util.HashMap;
69 
70 /**
71  * CameraDevice video recording use case tests by using MediaRecorder and
72  * MediaCodec.
73  */
74 @LargeTest
75 @RunWith(Parameterized.class)
76 public class RecordingTest extends Camera2SurfaceViewTestCase {
77     private static final String TAG = "RecordingTest";
78     private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE);
79     private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG);
80     private static final int RECORDING_DURATION_MS = 3000;
81     private static final int PREVIEW_DURATION_MS = 3000;
82     private static final float DURATION_MARGIN = 0.2f;
83     private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0;
84     private static final float FRAMEDURATION_MARGIN = 0.2f;
85     private static final float VID_SNPSHT_FRMDRP_RATE_TOLERANCE = 10.0f;
86     private static final float FRMDRP_RATE_TOLERANCE = 5.0f;
87     private static final int BIT_RATE_1080P = 16000000;
88     private static final int BIT_RATE_MIN = 64000;
89     private static final int BIT_RATE_MAX = 40000000;
90     private static final int VIDEO_FRAME_RATE = 30;
91     private static final int[] mCamcorderProfileList = {
92             CamcorderProfile.QUALITY_HIGH,
93             CamcorderProfile.QUALITY_2160P,
94             CamcorderProfile.QUALITY_1080P,
95             CamcorderProfile.QUALITY_720P,
96             CamcorderProfile.QUALITY_480P,
97             CamcorderProfile.QUALITY_CIF,
98             CamcorderProfile.QUALITY_QCIF,
99             CamcorderProfile.QUALITY_QVGA,
100             CamcorderProfile.QUALITY_LOW,
101     };
102     private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5;
103     private static final int BURST_VIDEO_SNAPSHOT_NUM = 3;
104     private static final int SLOWMO_SLOW_FACTOR = 4;
105     private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4;
106     private List<Size> mSupportedVideoSizes;
107     private Surface mRecordingSurface;
108     private Surface mPersistentSurface;
109     private MediaRecorder mMediaRecorder;
110     private String mOutMediaFileName;
111     private int mVideoFrameRate;
112     private Size mVideoSize;
113     private long mRecordingStartTime;
114 
115     private Surface mIntermediateSurface;
116     private ImageReader mIntermediateReader;
117     private ImageWriter mIntermediateWriter;
118     private ImageWriterQueuer mQueuer;
119     private HandlerThread mIntermediateThread;
120     private Handler mIntermediateHandler;
121 
122     @Override
setUp()123     public void setUp() throws Exception {
124         super.setUp();
125     }
126 
127     @Override
tearDown()128     public void tearDown() throws Exception {
129         super.tearDown();
130     }
131 
doBasicRecording(boolean useVideoStab)132     private void doBasicRecording(boolean useVideoStab) throws Exception {
133         doBasicRecording(useVideoStab, false);
134     }
135 
doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface)136     private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface)
137             throws Exception {
138         doBasicRecording(useVideoStab, useIntermediateSurface, false);
139     }
140 
doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)141     private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface,
142             boolean useEncoderProfiles) throws Exception {
143         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
144             try {
145                 Log.i(TAG, "Testing basic recording for camera " + mCameraIdsUnderTest[i]);
146                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
147                 if (!staticInfo.isColorOutputSupported()) {
148                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
149                             " does not support color outputs, skipping");
150                     continue;
151                 }
152 
153                 // External camera doesn't support CamcorderProfile recording
154                 if (staticInfo.isExternalCamera()) {
155                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
156                             " does not support CamcorderProfile, skipping");
157                     continue;
158                 }
159 
160                 if (!staticInfo.isVideoStabilizationSupported() && useVideoStab) {
161                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
162                             " does not support video stabilization, skipping the stabilization"
163                             + " test");
164                     continue;
165                 }
166 
167                 // Re-use the MediaRecorder object for the same camera device.
168                 mMediaRecorder = new MediaRecorder();
169                 openDevice(mCameraIdsUnderTest[i]);
170                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
171 
172                 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab,
173                         useIntermediateSurface, useEncoderProfiles);
174             } finally {
175                 closeDevice();
176                 releaseRecorder();
177             }
178         }
179     }
180 
181     /**
182      * <p>
183      * Test basic video stabilitzation camera recording.
184      * </p>
185      * <p>
186      * This test covers the typical basic use case of camera recording with video
187      * stabilization is enabled, if video stabilization is supported.
188      * MediaRecorder is used to record the audio and video, CamcorderProfile is
189      * used to configure the MediaRecorder. It goes through the pre-defined
190      * CamcorderProfile list, test each profile configuration and validate the
191      * recorded video. Preview is set to the video size.
192      * </p>
193      */
194     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicVideoStabilizationRecording()195     public void testBasicVideoStabilizationRecording() throws Exception {
196         doBasicRecording(/*useVideoStab*/true);
197     }
198 
199     /**
200      * <p>
201      * Test basic camera recording.
202      * </p>
203      * <p>
204      * This test covers the typical basic use case of camera recording.
205      * MediaRecorder is used to record the audio and video, CamcorderProfile is
206      * used to configure the MediaRecorder. It goes through the pre-defined
207      * CamcorderProfile list, test each profile configuration and validate the
208      * recorded video. Preview is set to the video size.
209      * </p>
210      */
211     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicRecording()212     public void testBasicRecording() throws Exception {
213         doBasicRecording(/*useVideoStab*/false);
214     }
215 
216     /**
217      * <p>
218      * Test camera recording with intermediate surface.
219      * </p>
220      * <p>
221      * This test is similar to testBasicRecording with a tweak where an intermediate
222      * surface is setup between camera and MediaRecorder, giving application a chance
223      * to decide whether to send a frame to recorder or not.
224      * </p>
225      */
226     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testIntermediateSurfaceRecording()227     public void testIntermediateSurfaceRecording() throws Exception {
228         doBasicRecording(/*useVideoStab*/false, /*useIntermediateSurface*/true);
229     }
230 
231     /**
232      * <p>
233      * Test basic camera recording using encoder profiles.
234      * </p>
235      * <p>
236      * This test covers the typical basic use case of camera recording.
237      * MediaRecorder is used to record the audio and video,
238      * EncoderProfiles are used to configure the MediaRecorder. It
239      * goes through the pre-defined CamcorderProfile list, test each
240      * encoder profile combination and validate the recorded video.
241      * Preview is set to the video size.
242      * </p>
243      */
244     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicEncoderProfilesRecording()245     public void testBasicEncoderProfilesRecording() throws Exception {
246         doBasicRecording(/*useVideoStab*/false,  /*useIntermediateSurface*/false,
247                 /*useEncoderProfiles*/true);
248     }
249 
250     /**
251      * <p>
252      * Test basic camera recording from a persistent input surface.
253      * </p>
254      * <p>
255      * This test is similar to testBasicRecording except that MediaRecorder records
256      * from a persistent input surface that's used across multiple recording sessions.
257      * </p>
258      */
259     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testRecordingFromPersistentSurface()260     public void testRecordingFromPersistentSurface() throws Exception {
261         if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) {
262             return; // skipped
263         }
264         mPersistentSurface = MediaCodec.createPersistentInputSurface();
265         assertNotNull("Failed to create persistent input surface!", mPersistentSurface);
266 
267         try {
268             doBasicRecording(/*useVideoStab*/false);
269         } finally {
270             mPersistentSurface.release();
271             mPersistentSurface = null;
272         }
273     }
274 
275     /**
276      * <p>
277      * Test camera recording for all supported sizes by using MediaRecorder.
278      * </p>
279      * <p>
280      * This test covers camera recording for all supported sizes by camera. MediaRecorder
281      * is used to encode the video. Preview is set to the video size. Recorded videos are
282      * validated according to the recording configuration.
283      * </p>
284      */
285     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testSupportedVideoSizes()286     public void testSupportedVideoSizes() throws Exception {
287         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
288             try {
289                 Log.i(TAG, "Testing supported video size recording for camera " + mCameraIdsUnderTest[i]);
290                 if (!mAllStaticInfo.get(mCameraIdsUnderTest[i]).isColorOutputSupported()) {
291                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
292                             " does not support color outputs, skipping");
293                     continue;
294                 }
295                 // Re-use the MediaRecorder object for the same camera device.
296                 mMediaRecorder = new MediaRecorder();
297                 openDevice(mCameraIdsUnderTest[i]);
298 
299                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
300 
301                 recordingSizeTestByCamera();
302             } finally {
303                 closeDevice();
304                 releaseRecorder();
305             }
306         }
307     }
308 
309     /**
310      * Test different start/stop orders of Camera and Recorder.
311      *
312      * <p>The recording should be working fine for any kind of start/stop orders.</p>
313      */
314     @Test
testCameraRecorderOrdering()315     public void testCameraRecorderOrdering() {
316         // TODO: need implement
317     }
318 
319     /**
320      * <p>
321      * Test camera recording for all supported sizes by using MediaCodec.
322      * </p>
323      * <p>
324      * This test covers video only recording for all supported sizes (camera and
325      * encoder). MediaCodec is used to encode the video. The recorded videos are
326      * validated according to the recording configuration.
327      * </p>
328      */
329     @Test
testMediaCodecRecording()330     public void testMediaCodecRecording() throws Exception {
331         // TODO. Need implement.
332     }
333 
334     /**
335      * <p>
336      * Test video snapshot for each camera.
337      * </p>
338      * <p>
339      * This test covers video snapshot typical use case. The MediaRecorder is used to record the
340      * video for each available video size. The largest still capture size is selected to
341      * capture the JPEG image. The still capture images are validated according to the capture
342      * configuration. The timestamp of capture result before and after video snapshot is also
343      * checked to make sure no frame drop caused by video snapshot.
344      * </p>
345      */
346     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testVideoSnapshot()347     public void testVideoSnapshot() throws Exception {
348         videoSnapshotHelper(/*burstTest*/false);
349     }
350 
351     /**
352      * <p>
353      * Test burst video snapshot for each camera.
354      * </p>
355      * <p>
356      * This test covers burst video snapshot capture. The MediaRecorder is used to record the
357      * video for each available video size. The largest still capture size is selected to
358      * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be
359      * sent during the test. The still capture images are validated according to the capture
360      * configuration.
361      * </p>
362      */
363     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBurstVideoSnapshot()364     public void testBurstVideoSnapshot() throws Exception {
365         videoSnapshotHelper(/*burstTest*/true);
366     }
367 
368     /**
369      * Test timelapse recording, where capture rate is slower than video (playback) frame rate.
370      */
371     @Test
testTimelapseRecording()372     public void testTimelapseRecording() throws Exception {
373         // TODO. Need implement.
374     }
375 
376     @Test
testSlowMotionRecording()377     public void testSlowMotionRecording() throws Exception {
378         slowMotionRecording();
379     }
380 
381     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testConstrainedHighSpeedRecording()382     public void testConstrainedHighSpeedRecording() throws Exception {
383         constrainedHighSpeedRecording();
384     }
385 
386     @Test
testAbandonedHighSpeedRequest()387     public void testAbandonedHighSpeedRequest() throws Exception {
388         for (String id : mCameraIdsUnderTest) {
389             try {
390                 Log.i(TAG, "Testing bad suface for createHighSpeedRequestList for camera " + id);
391                 StaticMetadata staticInfo = mAllStaticInfo.get(id);
392                 if (!staticInfo.isColorOutputSupported()) {
393                     Log.i(TAG, "Camera " + id +
394                             " does not support color outputs, skipping");
395                     continue;
396                 }
397                 if (!staticInfo.isConstrainedHighSpeedVideoSupported()) {
398                     Log.i(TAG, "Camera " + id +
399                             " does not support constrained high speed video, skipping");
400                     continue;
401                 }
402 
403                 // Re-use the MediaRecorder object for the same camera device.
404                 mMediaRecorder = new MediaRecorder();
405                 openDevice(id);
406 
407                 StreamConfigurationMap config =
408                         mStaticInfo.getValueFromKeyNonNull(
409                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
410                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
411                 Size size = highSpeedVideoSizes[0];
412                 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size);
413                 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " +
414                         "size " + size, fpsRange);
415                 if (fpsRange == null) {
416                     continue;
417                 }
418 
419                 int captureRate = fpsRange.getLower();
420                 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR;
421                 // Skip the test if the highest recording FPS supported by CamcorderProfile
422                 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
423                     Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
424                             + " is not supported by CamcorderProfile");
425                     continue;
426                 }
427 
428                 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
429                 prepareRecording(size, videoFramerate, captureRate);
430                 updatePreviewSurfaceWithVideo(size, captureRate);
431 
432                 List<Surface> outputSurfaces = new ArrayList<Surface>(2);
433                 assertTrue("Both preview and recording surfaces should be valid",
434                         mPreviewSurface.isValid() && mRecordingSurface.isValid());
435 
436                 outputSurfaces.add(mPreviewSurface);
437                 outputSurfaces.add(mRecordingSurface);
438 
439                 mSessionListener = new BlockingSessionCallback();
440                 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true,
441                         mSessionListener, mHandler);
442 
443                 CaptureRequest.Builder requestBuilder =
444                         mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
445                 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
446                 requestBuilder.addTarget(mPreviewSurface);
447                 requestBuilder.addTarget(mRecordingSurface);
448 
449                 // 1. Test abandoned MediaRecorder
450                 releaseRecorder();
451                 try {
452                     List<CaptureRequest> slowMoRequests =
453                             ((CameraConstrainedHighSpeedCaptureSession) mSession).
454                             createHighSpeedRequestList(requestBuilder.build());
455                     fail("Create high speed request on abandoned surface must fail!");
456                 } catch (IllegalArgumentException e) {
457                     Log.i(TAG, "Release recording surface test passed");
458                     // expected
459                 }
460 
461                 // 2. Test abandoned preview surface
462                 mMediaRecorder = new MediaRecorder();
463                 SurfaceTexture preview = new SurfaceTexture(/*random int*/ 1);
464                 Surface previewSurface = new Surface(preview);
465                 preview.setDefaultBufferSize(size.getWidth(), size.getHeight());
466 
467                 outputSurfaces = new ArrayList<Surface>();
468                 outputSurfaces.add(previewSurface);
469 
470                 prepareRecording(size, videoFramerate, captureRate);
471                 updatePreviewSurfaceWithVideo(size, captureRate);
472 
473                 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true,
474                         mSessionListener, mHandler);
475 
476                 requestBuilder = mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
477                 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
478                 requestBuilder.addTarget(previewSurface);
479 
480                 // Abandon preview surface.
481                 previewSurface.release();
482 
483                 try {
484                     List<CaptureRequest> slowMoRequests =
485                             ((CameraConstrainedHighSpeedCaptureSession) mSession).
486                             createHighSpeedRequestList(requestBuilder.build());
487                     fail("Create high speed request on abandoned preview surface must fail!");
488                 } catch (IllegalArgumentException e) {
489                     Log.i(TAG, "Release preview surface test passed");
490                     // expected
491                 }
492             } finally {
493                 closeDevice();
494                 releaseRecorder();
495             }
496         }
497     }
498 
499     /**
500      * <p>
501      * Test recording framerate accuracy when switching from low FPS to high FPS.
502      * </p>
503      * <p>
504      * This test first record a video with profile of lowest framerate then record a video with
505      * profile of highest framerate. Make sure that the video framerate are still accurate.
506      * </p>
507      */
508     @Test
testRecordingFramerateLowToHigh()509     public void testRecordingFramerateLowToHigh() throws Exception {
510         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
511             try {
512                 Log.i(TAG, "Testing recording framerate low to high for camera " + mCameraIdsUnderTest[i]);
513                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
514                 if (!staticInfo.isColorOutputSupported()) {
515                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
516                             " does not support color outputs, skipping");
517                     continue;
518                 }
519                 if (staticInfo.isExternalCamera()) {
520                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
521                             " does not support CamcorderProfile, skipping");
522                     continue;
523                 }
524                 // Re-use the MediaRecorder object for the same camera device.
525                 mMediaRecorder = new MediaRecorder();
526                 openDevice(mCameraIdsUnderTest[i]);
527 
528                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
529 
530                 int minFpsProfileId = -1, minFps = 1000;
531                 int maxFpsProfileId = -1, maxFps = 0;
532                 int cameraId = Integer.valueOf(mCamera.getId());
533 
534                 for (int profileId : mCamcorderProfileList) {
535                     if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
536                         continue;
537                     }
538                     CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
539                     if (profile.videoFrameRate < minFps) {
540                         minFpsProfileId = profileId;
541                         minFps = profile.videoFrameRate;
542                     }
543                     if (profile.videoFrameRate > maxFps) {
544                         maxFpsProfileId = profileId;
545                         maxFps = profile.videoFrameRate;
546                     }
547                 }
548 
549                 int camcorderProfileList[] = new int[] {minFpsProfileId, maxFpsProfileId};
550                 basicRecordingTestByCamera(camcorderProfileList, /*useVideoStab*/false);
551             } finally {
552                 closeDevice();
553                 releaseRecorder();
554             }
555         }
556     }
557 
558     /**
559      * <p>
560      * Test preview and video surfaces sharing the same camera stream.
561      * </p>
562      */
563     @Test
testVideoPreviewSurfaceSharing()564     public void testVideoPreviewSurfaceSharing() throws Exception {
565         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
566             try {
567                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
568                 if (staticInfo.isHardwareLevelLegacy()) {
569                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] + " is legacy, skipping");
570                     continue;
571                 }
572                 if (!staticInfo.isColorOutputSupported()) {
573                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
574                             " does not support color outputs, skipping");
575                     continue;
576                 }
577                 // Re-use the MediaRecorder object for the same camera device.
578                 mMediaRecorder = new MediaRecorder();
579                 openDevice(mCameraIdsUnderTest[i]);
580 
581                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
582 
583                 videoPreviewSurfaceSharingTestByCamera();
584             } finally {
585                 closeDevice();
586                 releaseRecorder();
587             }
588         }
589     }
590 
591     /**
592      * <p>
593      * Test recording with same recording surface and different preview surfaces.
594      * </p>
595      * <p>
596      * This test maintains persistent video surface while changing preview surface.
597      * This exercises format/dataspace override behavior of the camera device.
598      * </p>
599      */
600     @Test
testRecordingWithDifferentPreviewSizes()601     public void testRecordingWithDifferentPreviewSizes() throws Exception {
602         if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) {
603             return; // skipped
604         }
605         mPersistentSurface = MediaCodec.createPersistentInputSurface();
606         assertNotNull("Failed to create persistent input surface!", mPersistentSurface);
607 
608         try {
609             doRecordingWithDifferentPreviewSizes();
610         } finally {
611             mPersistentSurface.release();
612             mPersistentSurface = null;
613         }
614     }
615 
doRecordingWithDifferentPreviewSizes()616     public void doRecordingWithDifferentPreviewSizes() throws Exception {
617         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
618             try {
619                 Log.i(TAG, "Testing recording with different preview sizes for camera " +
620                         mCameraIdsUnderTest[i]);
621                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
622                 if (!staticInfo.isColorOutputSupported()) {
623                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
624                             " does not support color outputs, skipping");
625                     continue;
626                 }
627                 if (staticInfo.isExternalCamera()) {
628                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
629                             " does not support CamcorderProfile, skipping");
630                     continue;
631                 }
632                 // Re-use the MediaRecorder object for the same camera device.
633                 mMediaRecorder = new MediaRecorder();
634                 openDevice(mCameraIdsUnderTest[i]);
635 
636                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
637 
638                 Size maxPreviewSize = mOrderedPreviewSizes.get(0);
639                 List<Range<Integer> > fpsRanges = Arrays.asList(
640                         mStaticInfo.getAeAvailableTargetFpsRangesChecked());
641                 int cameraId = Integer.valueOf(mCamera.getId());
642                 int maxVideoFrameRate = -1;
643                 for (int profileId : mCamcorderProfileList) {
644                     if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
645                         continue;
646                     }
647                     CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
648 
649                     Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
650                     Range<Integer> fpsRange = new Range(
651                             profile.videoFrameRate, profile.videoFrameRate);
652                     if (maxVideoFrameRate < profile.videoFrameRate) {
653                         maxVideoFrameRate = profile.videoFrameRate;
654                     }
655 
656                     if (allowedUnsupported(cameraId, profileId)) {
657                         continue;
658                     }
659 
660                     if (mStaticInfo.isHardwareLevelLegacy() &&
661                             (videoSz.getWidth() > maxPreviewSize.getWidth() ||
662                              videoSz.getHeight() > maxPreviewSize.getHeight())) {
663                         // Skip. Legacy mode can only do recording up to max preview size
664                         continue;
665                     }
666                     assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId +
667                                     " must be one of the camera device supported video size!",
668                                     mSupportedVideoSizes.contains(videoSz));
669                     assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId +
670                             ") must be one of the camera device available FPS range!",
671                             fpsRanges.contains(fpsRange));
672 
673                     // Configure preview and recording surfaces.
674                     mOutMediaFileName = mDebugFileNameBase + "/test_video_surface_reconfig.mp4";
675 
676                     // prepare preview surface by using video size.
677                     List<Size> previewSizes = getPreviewSizesForVideo(videoSz,
678                             profile.videoFrameRate);
679                     if (previewSizes.size() <= 1) {
680                         continue;
681                     }
682 
683                     // 1. Do video recording using largest compatbile preview sizes
684                     prepareRecordingWithProfile(profile);
685                     updatePreviewSurface(previewSizes.get(0));
686                     SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
687                     startRecording(
688                             /* useMediaRecorder */true, resultListener,
689                             /*useVideoStab*/false, fpsRange, false);
690                     SystemClock.sleep(RECORDING_DURATION_MS);
691                     stopRecording(/* useMediaRecorder */true, /* useIntermediateSurface */false,
692                             /* stopStreaming */false);
693 
694                     // 2. Reconfigure with the same recording surface, but switch to a smaller
695                     // preview size.
696                     prepareRecordingWithProfile(profile);
697                     updatePreviewSurface(previewSizes.get(1));
698                     SimpleCaptureCallback resultListener2 = new SimpleCaptureCallback();
699                     startRecording(
700                             /* useMediaRecorder */true, resultListener2,
701                             /*useVideoStab*/false, fpsRange, false);
702                     SystemClock.sleep(RECORDING_DURATION_MS);
703                     stopRecording(/* useMediaRecorder */true);
704                     break;
705                 }
706             } finally {
707                 closeDevice();
708                 releaseRecorder();
709             }
710         }
711     }
712 
713     /**
714      * Test camera preview and video surface sharing for maximum supported size.
715      */
videoPreviewSurfaceSharingTestByCamera()716     private void videoPreviewSurfaceSharingTestByCamera() throws Exception {
717         for (Size sz : mOrderedPreviewSizes) {
718             if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) {
719                 continue;
720             }
721 
722             if (VERBOSE) {
723                 Log.v(TAG, "Testing camera recording with video size " + sz.toString());
724             }
725 
726             // Configure preview and recording surfaces.
727             mOutMediaFileName = mDebugFileNameBase + "/test_video_share.mp4";
728             if (DEBUG_DUMP) {
729                 mOutMediaFileName = mDebugFileNameBase + "/test_video_share_" + mCamera.getId() +
730                     "_" + sz.toString() + ".mp4";
731             }
732 
733             // Allow external camera to use variable fps range
734             Range<Integer> fpsRange = null;
735             if (mStaticInfo.isExternalCamera()) {
736                 Range<Integer>[] availableFpsRange =
737                         mStaticInfo.getAeAvailableTargetFpsRangesChecked();
738 
739                 boolean foundRange = false;
740                 int minFps = 0;
741                 for (int i = 0; i < availableFpsRange.length; i += 1) {
742                     if (minFps < availableFpsRange[i].getLower()
743                             && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) {
744                         minFps = availableFpsRange[i].getLower();
745                         foundRange = true;
746                     }
747                 }
748                 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange);
749                 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE);
750             }
751 
752             // Use AVC and AAC a/v compression format.
753             prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE);
754 
755             // prepare preview surface by using video size.
756             updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE);
757 
758             // Start recording
759             SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
760             if (!startSharedRecording(/* useMediaRecorder */true, resultListener,
761                     /*useVideoStab*/false, fpsRange)) {
762                 mMediaRecorder.reset();
763                 continue;
764             }
765 
766             // Record certain duration.
767             SystemClock.sleep(RECORDING_DURATION_MS);
768 
769             // Stop recording and preview
770             stopRecording(/* useMediaRecorder */true);
771             // Convert number of frames camera produced into the duration in unit of ms.
772             float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE;
773             float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs;
774             float durationMaxMs = durationMinMs;
775             float frameDurationMaxMs = 0.f;
776             if (fpsRange != null) {
777                 frameDurationMaxMs = 1000.0f / fpsRange.getLower();
778                 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs;
779             }
780 
781             // Validation.
782             validateRecording(sz, durationMinMs, durationMaxMs,
783                     frameDurationMinMs, frameDurationMaxMs,
784                     FRMDRP_RATE_TOLERANCE);
785 
786             break;
787         }
788     }
789 
790     /**
791      * Test slow motion recording where capture rate (camera output) is different with
792      * video (playback) frame rate for each camera if high speed recording is supported
793      * by both camera and encoder.
794      *
795      * <p>
796      * Normal recording use cases make the capture rate (camera output frame
797      * rate) the same as the video (playback) frame rate. This guarantees that
798      * the motions in the scene play at the normal speed. If the capture rate is
799      * faster than video frame rate, for a given time duration, more number of
800      * frames are captured than it can be played in the same time duration. This
801      * generates "slow motion" effect during playback.
802      * </p>
803      */
slowMotionRecording()804     private void slowMotionRecording() throws Exception {
805         for (String id : mCameraIdsUnderTest) {
806             try {
807                 Log.i(TAG, "Testing slow motion recording for camera " + id);
808                 StaticMetadata staticInfo = mAllStaticInfo.get(id);
809                 if (!staticInfo.isColorOutputSupported()) {
810                     Log.i(TAG, "Camera " + id +
811                             " does not support color outputs, skipping");
812                     continue;
813                 }
814                 if (!staticInfo.isHighSpeedVideoSupported()) {
815                     continue;
816                 }
817 
818                 // Re-use the MediaRecorder object for the same camera device.
819                 mMediaRecorder = new MediaRecorder();
820                 openDevice(id);
821 
822                 StreamConfigurationMap config =
823                         mStaticInfo.getValueFromKeyNonNull(
824                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
825                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
826                 for (Size size : highSpeedVideoSizes) {
827                     Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size);
828                     mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " +
829                             "size " + size, fpsRange);
830                     if (fpsRange == null) {
831                         continue;
832                     }
833 
834                     int captureRate = fpsRange.getLower();
835                     int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR;
836                     // Skip the test if the highest recording FPS supported by CamcorderProfile
837                     if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
838                         Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
839                                 + " is not supported by CamcorderProfile");
840                         continue;
841                     }
842 
843                     mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video.mp4";
844                     if (DEBUG_DUMP) {
845                         mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video_" + id + "_"
846                                 + size.toString() + ".mp4";
847                     }
848 
849                     prepareRecording(size, videoFramerate, captureRate);
850 
851                     // prepare preview surface by using video size.
852                     updatePreviewSurfaceWithVideo(size, captureRate);
853 
854                     // Start recording
855                     SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
856                     startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate,
857                             fpsRange, resultListener, /*useHighSpeedSession*/false);
858 
859                     // Record certain duration.
860                     SystemClock.sleep(RECORDING_DURATION_MS);
861 
862                     // Stop recording and preview
863                     stopRecording(/*useMediaRecorder*/true);
864                     // Convert number of frames camera produced into the duration in unit of ms.
865                     float frameDurationMs = 1000.0f / videoFramerate;
866                     float durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
867 
868                     // Validation.
869                     validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
870                 }
871 
872             } finally {
873                 closeDevice();
874                 releaseRecorder();
875             }
876         }
877     }
878 
constrainedHighSpeedRecording()879     private void constrainedHighSpeedRecording() throws Exception {
880         for (String id : mCameraIdsUnderTest) {
881             try {
882                 Log.i(TAG, "Testing constrained high speed recording for camera " + id);
883 
884                 if (!mAllStaticInfo.get(id).isConstrainedHighSpeedVideoSupported()) {
885                     Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping.");
886                     continue;
887                 }
888 
889                 // Re-use the MediaRecorder object for the same camera device.
890                 mMediaRecorder = new MediaRecorder();
891                 openDevice(id);
892 
893                 StreamConfigurationMap config =
894                         mStaticInfo.getValueFromKeyNonNull(
895                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
896                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
897                 for (Size size : highSpeedVideoSizes) {
898                     List<Range<Integer>> fixedFpsRanges =
899                             getHighSpeedFixedFpsRangeForSize(config, size);
900                     mCollector.expectTrue("Unable to find the fixed frame rate fps range for " +
901                             "size " + size, fixedFpsRanges.size() > 0);
902                     // Test recording for each FPS range
903                     for (Range<Integer> fpsRange : fixedFpsRanges) {
904                         int captureRate = fpsRange.getLower();
905                         final int VIDEO_FRAME_RATE = 30;
906                         // Skip the test if the highest recording FPS supported by CamcorderProfile
907                         if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
908                             Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
909                                     + " is not supported by CamcorderProfile");
910                             continue;
911                         }
912 
913                         SimpleCaptureCallback previewResultListener = new SimpleCaptureCallback();
914 
915                         // prepare preview surface by using video size.
916                         updatePreviewSurfaceWithVideo(size, captureRate);
917 
918                         startConstrainedPreview(fpsRange, previewResultListener);
919 
920                         mOutMediaFileName = mDebugFileNameBase + "/test_cslowMo_video_" +
921                             captureRate + "fps_" + id + "_" + size.toString() + ".mp4";
922 
923                         prepareRecording(size, VIDEO_FRAME_RATE, captureRate);
924 
925                         SystemClock.sleep(PREVIEW_DURATION_MS);
926 
927                         stopCameraStreaming();
928 
929                         SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
930                         // Start recording
931                         startSlowMotionRecording(/*useMediaRecorder*/true, VIDEO_FRAME_RATE,
932                                 captureRate, fpsRange, resultListener,
933                                 /*useHighSpeedSession*/true);
934 
935                         // Record certain duration.
936                         SystemClock.sleep(RECORDING_DURATION_MS);
937 
938                         // Stop recording and preview
939                         stopRecording(/*useMediaRecorder*/true);
940 
941                         startConstrainedPreview(fpsRange, previewResultListener);
942 
943                         // Convert number of frames camera produced into the duration in unit of ms.
944                         float frameDurationMs = 1000.0f / VIDEO_FRAME_RATE;
945                         float durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
946 
947                         // Validation.
948                         validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
949 
950                         SystemClock.sleep(PREVIEW_DURATION_MS);
951 
952                         stopCameraStreaming();
953                     }
954                 }
955 
956             } finally {
957                 closeDevice();
958                 releaseRecorder();
959             }
960         }
961     }
962 
963     /**
964      * Get high speed FPS from CamcorderProfiles for a given size.
965      *
966      * @param size The size used to search the CamcorderProfiles for the FPS.
967      * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles.
968      */
getFpsFromHighSpeedProfileForSize(Size size)969     private int getFpsFromHighSpeedProfileForSize(Size size) {
970         for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P;
971                 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) {
972             if (CamcorderProfile.hasProfile(quality)) {
973                 CamcorderProfile profile = CamcorderProfile.get(quality);
974                 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){
975                     return profile.videoFrameRate;
976                 }
977             }
978         }
979 
980         return 0;
981     }
982 
getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)983     private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config,
984             Size size) {
985         Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size);
986         Range<Integer> maxRange = availableFpsRanges[0];
987         boolean foundRange = false;
988         for (Range<Integer> range : availableFpsRanges) {
989             if (range.getLower().equals(range.getUpper()) && range.getLower() >= maxRange.getLower()) {
990                 foundRange = true;
991                 maxRange = range;
992             }
993         }
994 
995         if (!foundRange) {
996             return null;
997         }
998         return maxRange;
999     }
1000 
getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)1001     private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config,
1002             Size size) {
1003         Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size);
1004         List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>();
1005         for (Range<Integer> range : availableFpsRanges) {
1006             if (range.getLower().equals(range.getUpper())) {
1007                 fixedRanges.add(range);
1008             }
1009         }
1010         return fixedRanges;
1011     }
1012 
startConstrainedPreview(Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener)1013     private void startConstrainedPreview(Range<Integer> fpsRange,
1014             CameraCaptureSession.CaptureCallback listener) throws Exception {
1015         List<Surface> outputSurfaces = new ArrayList<Surface>(1);
1016         assertTrue("Preview surface should be valid", mPreviewSurface.isValid());
1017         outputSurfaces.add(mPreviewSurface);
1018         mSessionListener = new BlockingSessionCallback();
1019 
1020         List<CaptureRequest> slowMoRequests = null;
1021         CaptureRequest.Builder requestBuilder =
1022             mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1023         requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1024         requestBuilder.addTarget(mPreviewSurface);
1025         CaptureRequest initialRequest = requestBuilder.build();
1026         CameraTestUtils.checkSessionConfigurationWithSurfaces(mCamera, mHandler,
1027                 outputSurfaces, /*inputConfig*/ null, SessionConfiguration.SESSION_HIGH_SPEED,
1028                 /*defaultSupport*/ true, "Constrained session configuration query failed");
1029         mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener,
1030                 mHandler, initialRequest);
1031         slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession).
1032             createHighSpeedRequestList(initialRequest);
1033 
1034         mSession.setRepeatingBurst(slowMoRequests, listener, mHandler);
1035     }
1036 
startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, int captureRate, Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession)1037     private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate,
1038             int captureRate, Range<Integer> fpsRange,
1039             CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession)
1040             throws Exception {
1041         List<Surface> outputSurfaces = new ArrayList<Surface>(2);
1042         assertTrue("Both preview and recording surfaces should be valid",
1043                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1044         outputSurfaces.add(mPreviewSurface);
1045         outputSurfaces.add(mRecordingSurface);
1046         // Video snapshot surface
1047         if (mReaderSurface != null) {
1048             outputSurfaces.add(mReaderSurface);
1049         }
1050         mSessionListener = new BlockingSessionCallback();
1051 
1052         // Create slow motion request list
1053         List<CaptureRequest> slowMoRequests = null;
1054         if (useHighSpeedSession) {
1055             CaptureRequest.Builder requestBuilder =
1056                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1057             requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1058             requestBuilder.addTarget(mPreviewSurface);
1059             requestBuilder.addTarget(mRecordingSurface);
1060             CaptureRequest initialRequest = requestBuilder.build();
1061             mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener,
1062                     mHandler, initialRequest);
1063             slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession).
1064                     createHighSpeedRequestList(initialRequest);
1065         } else {
1066             CaptureRequest.Builder recordingRequestBuilder =
1067                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1068             recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE,
1069                     CaptureRequest.CONTROL_MODE_USE_SCENE_MODE);
1070             recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE,
1071                     CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO);
1072 
1073             CaptureRequest.Builder recordingOnlyBuilder =
1074                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1075             recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE,
1076                     CaptureRequest.CONTROL_MODE_USE_SCENE_MODE);
1077             recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE,
1078                     CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO);
1079             int slowMotionFactor = captureRate / videoFrameRate;
1080 
1081             // Make sure camera output frame rate is set to correct value.
1082             recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1083             recordingRequestBuilder.addTarget(mRecordingSurface);
1084             recordingRequestBuilder.addTarget(mPreviewSurface);
1085             recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1086             recordingOnlyBuilder.addTarget(mRecordingSurface);
1087 
1088             CaptureRequest initialRequest = recordingRequestBuilder.build();
1089             mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces,
1090                     mSessionListener, mHandler, initialRequest);
1091 
1092             slowMoRequests = new ArrayList<CaptureRequest>();
1093             slowMoRequests.add(initialRequest);// Preview + recording.
1094 
1095             for (int i = 0; i < slowMotionFactor - 1; i++) {
1096                 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only.
1097             }
1098         }
1099 
1100         mSession.setRepeatingBurst(slowMoRequests, listener, mHandler);
1101 
1102         if (useMediaRecorder) {
1103             mMediaRecorder.start();
1104         } else {
1105             // TODO: need implement MediaCodec path.
1106         }
1107 
1108     }
1109 
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)1110     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)
1111             throws Exception {
1112         basicRecordingTestByCamera(camcorderProfileList, useVideoStab, false);
1113     }
1114 
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface)1115     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab,
1116             boolean useIntermediateSurface) throws Exception {
1117         basicRecordingTestByCamera(camcorderProfileList, useVideoStab,
1118                 useIntermediateSurface, false);
1119     }
1120 
1121     /**
1122      * Test camera recording by using each available CamcorderProfile for a
1123      * given camera. preview size is set to the video size.
1124      */
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)1125     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab,
1126             boolean useIntermediateSurface, boolean useEncoderProfiles) throws Exception {
1127         Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1128         List<Range<Integer> > fpsRanges = Arrays.asList(
1129                 mStaticInfo.getAeAvailableTargetFpsRangesChecked());
1130         int cameraId = Integer.valueOf(mCamera.getId());
1131         int maxVideoFrameRate = -1;
1132 
1133         // only validate recording for non-perf measurement runs
1134         boolean validateRecording = !isPerfMeasure();
1135         for (int profileId : camcorderProfileList) {
1136             if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
1137                 continue;
1138             }
1139 
1140             CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
1141             Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1142 
1143             Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate);
1144             if (maxVideoFrameRate < profile.videoFrameRate) {
1145                 maxVideoFrameRate = profile.videoFrameRate;
1146             }
1147 
1148             if (allowedUnsupported(cameraId, profileId)) {
1149                 continue;
1150             }
1151 
1152             if (mStaticInfo.isHardwareLevelLegacy() &&
1153                     (videoSz.getWidth() > maxPreviewSize.getWidth() ||
1154                      videoSz.getHeight() > maxPreviewSize.getHeight())) {
1155                 // Skip. Legacy mode can only do recording up to max preview size
1156                 continue;
1157             }
1158             assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId +
1159                             " must be one of the camera device supported video size!",
1160                             mSupportedVideoSizes.contains(videoSz));
1161             assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId +
1162                     ") must be one of the camera device available FPS range!",
1163                     fpsRanges.contains(fpsRange));
1164 
1165 
1166             if (useEncoderProfiles) {
1167                 // Iterate through all video-audio codec combination
1168                 EncoderProfiles profiles = CamcorderProfile.getAll(mCamera.getId(), profileId);
1169                 for (EncoderProfiles.VideoProfile videoProfile : profiles.getVideoProfiles()) {
1170                     boolean hasAudioProfile = false;
1171                     for (EncoderProfiles.AudioProfile audioProfile : profiles.getAudioProfiles()) {
1172                         hasAudioProfile = true;
1173                         doBasicRecordingByProfile(profiles, videoProfile, audioProfile,
1174                                 useVideoStab, useIntermediateSurface, validateRecording);
1175                         // Only measure the default video profile of the largest video
1176                         // recording size when measuring perf
1177                         if (isPerfMeasure()) {
1178                             break;
1179                         }
1180                     }
1181                     // Timelapse profiles do not have audio track
1182                     if (!hasAudioProfile) {
1183                         doBasicRecordingByProfile(profiles, videoProfile, /* audioProfile */null,
1184                                 useVideoStab, useIntermediateSurface, validateRecording);
1185                     }
1186                 }
1187             } else {
1188                 doBasicRecordingByProfile(
1189                         profile, useVideoStab, useIntermediateSurface, validateRecording);
1190             }
1191 
1192             if (isPerfMeasure()) {
1193                 // Only measure the largest video recording size when measuring perf
1194                 break;
1195             }
1196         }
1197         if (maxVideoFrameRate != -1) {
1198             // At least one CamcorderProfile is present, check FPS
1199             assertTrue("At least one CamcorderProfile must support >= 24 FPS",
1200                     maxVideoFrameRate >= 24);
1201         }
1202     }
1203 
doBasicRecordingByProfile( CamcorderProfile profile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1204     private void doBasicRecordingByProfile(
1205             CamcorderProfile profile, boolean userVideoStab,
1206             boolean useIntermediateSurface, boolean validate) throws Exception {
1207         Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1208         int frameRate = profile.videoFrameRate;
1209 
1210         if (VERBOSE) {
1211             Log.v(TAG, "Testing camera recording with video size " + videoSz.toString());
1212         }
1213 
1214         // Configure preview and recording surfaces.
1215         mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1216         if (DEBUG_DUMP) {
1217             mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1218                     + videoSz.toString() + ".mp4";
1219         }
1220 
1221         setupMediaRecorder(profile);
1222         completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate);
1223     }
1224 
doBasicRecordingByProfile( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1225     private void doBasicRecordingByProfile(
1226             EncoderProfiles profiles,
1227             EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile,
1228             boolean userVideoStab, boolean useIntermediateSurface, boolean validate)
1229                     throws Exception {
1230         Size videoSz = new Size(videoProfile.getWidth(), videoProfile.getHeight());
1231         int frameRate = videoProfile.getFrameRate();
1232 
1233         if (VERBOSE) {
1234             Log.v(TAG, "Testing camera recording with video size " + videoSz.toString() +
1235                   ", video codec " + videoProfile.getMediaType() + ", and audio codec " +
1236                   (audioProfile == null ? "(null)" : audioProfile.getMediaType()));
1237         }
1238 
1239         // Configure preview and recording surfaces.
1240         mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1241         if (DEBUG_DUMP) {
1242             mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1243                     + videoSz.toString() + "_" + videoProfile.getCodec();
1244             if (audioProfile != null) {
1245                 mOutMediaFileName += "_" + audioProfile.getCodec();
1246             }
1247             mOutMediaFileName += ".mp4";
1248         }
1249 
1250         setupMediaRecorder(profiles, videoProfile, audioProfile);
1251         completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate);
1252     }
1253 
completeBasicRecording( Size videoSz, int frameRate, boolean useVideoStab, boolean useIntermediateSurface, boolean validate)1254     private void completeBasicRecording(
1255             Size videoSz, int frameRate, boolean useVideoStab,
1256             boolean useIntermediateSurface, boolean validate) throws Exception {
1257         prepareRecording(useIntermediateSurface);
1258 
1259         // prepare preview surface by using video size.
1260         updatePreviewSurfaceWithVideo(videoSz, frameRate);
1261 
1262         // Start recording
1263         SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1264         startRecording(/* useMediaRecorder */true, resultListener, useVideoStab,
1265                 useIntermediateSurface);
1266 
1267         // Record certain duration.
1268         SystemClock.sleep(RECORDING_DURATION_MS);
1269 
1270         // Stop recording and preview
1271         stopRecording(/* useMediaRecorder */true, useIntermediateSurface,
1272                 /* stopCameraStreaming */true);
1273         // Convert number of frames camera produced into the duration in unit of ms.
1274         float frameDurationMs = 1000.0f / frameRate;
1275         float durationMs = 0.f;
1276         if (useIntermediateSurface) {
1277             durationMs = mQueuer.getQueuedCount() * frameDurationMs;
1278         } else {
1279             durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
1280         }
1281 
1282         if (VERBOSE) {
1283             Log.v(TAG, "video frame rate: " + frameRate +
1284                             ", num of frames produced: " + resultListener.getTotalNumFrames());
1285         }
1286 
1287         if (validate) {
1288             validateRecording(videoSz, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
1289         }
1290     }
1291 
1292     /**
1293      * Test camera recording for each supported video size by camera, preview
1294      * size is set to the video size.
1295      */
recordingSizeTestByCamera()1296     private void recordingSizeTestByCamera() throws Exception {
1297         for (Size sz : mSupportedVideoSizes) {
1298             if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) {
1299                 continue;
1300             }
1301 
1302             if (VERBOSE) {
1303                 Log.v(TAG, "Testing camera recording with video size " + sz.toString());
1304             }
1305 
1306             // Configure preview and recording surfaces.
1307             mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1308             if (DEBUG_DUMP) {
1309                 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1310                         + sz.toString() + ".mp4";
1311             }
1312 
1313             // Allow external camera to use variable fps range
1314             Range<Integer> fpsRange = null;
1315             if (mStaticInfo.isExternalCamera()) {
1316                 Range<Integer>[] availableFpsRange =
1317                         mStaticInfo.getAeAvailableTargetFpsRangesChecked();
1318 
1319                 boolean foundRange = false;
1320                 int minFps = 0;
1321                 for (int i = 0; i < availableFpsRange.length; i += 1) {
1322                     if (minFps < availableFpsRange[i].getLower()
1323                             && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) {
1324                         minFps = availableFpsRange[i].getLower();
1325                         foundRange = true;
1326                     }
1327                 }
1328                 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange);
1329                 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE);
1330             }
1331 
1332             // Use AVC and AAC a/v compression format.
1333             prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE);
1334 
1335             // prepare preview surface by using video size.
1336             updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE);
1337 
1338             // Start recording
1339             SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1340             startRecording(
1341                     /* useMediaRecorder */true, resultListener,
1342                     /*useVideoStab*/false, fpsRange, false);
1343 
1344             // Record certain duration.
1345             SystemClock.sleep(RECORDING_DURATION_MS);
1346 
1347             // Stop recording and preview
1348             stopRecording(/* useMediaRecorder */true);
1349             // Convert number of frames camera produced into the duration in unit of ms.
1350             float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE;
1351             float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs;
1352             float durationMaxMs = durationMinMs;
1353             float frameDurationMaxMs = 0.f;
1354             if (fpsRange != null) {
1355                 frameDurationMaxMs = 1000.0f / fpsRange.getLower();
1356                 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs;
1357             }
1358 
1359             // Validation.
1360             validateRecording(sz, durationMinMs, durationMaxMs,
1361                     frameDurationMinMs, frameDurationMaxMs,
1362                     FRMDRP_RATE_TOLERANCE);
1363         }
1364     }
1365 
1366     /**
1367      * Initialize the supported video sizes.
1368      */
initSupportedVideoSize(String cameraId)1369     private void initSupportedVideoSize(String cameraId)  throws Exception {
1370         int id = Integer.valueOf(cameraId);
1371         Size maxVideoSize = SIZE_BOUND_720P;
1372         if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_2160P)) {
1373             maxVideoSize = SIZE_BOUND_2160P;
1374         } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_QHD)) {
1375             maxVideoSize = SIZE_BOUND_QHD;
1376         } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_2K)) {
1377             maxVideoSize = SIZE_BOUND_2K;
1378         } else if (CamcorderProfile.hasProfile(id, CamcorderProfile.QUALITY_1080P)) {
1379             maxVideoSize = SIZE_BOUND_1080P;
1380         }
1381 
1382         mSupportedVideoSizes =
1383                 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize);
1384     }
1385 
1386     /**
1387      * Simple wrapper to wrap normal/burst video snapshot tests
1388      */
videoSnapshotHelper(boolean burstTest)1389     private void videoSnapshotHelper(boolean burstTest) throws Exception {
1390             for (String id : mCameraIdsUnderTest) {
1391                 try {
1392                     Log.i(TAG, "Testing video snapshot for camera " + id);
1393 
1394                     StaticMetadata staticInfo = mAllStaticInfo.get(id);
1395                     if (!staticInfo.isColorOutputSupported()) {
1396                         Log.i(TAG, "Camera " + id +
1397                                 " does not support color outputs, skipping");
1398                         continue;
1399                     }
1400 
1401                     if (staticInfo.isExternalCamera()) {
1402                         Log.i(TAG, "Camera " + id +
1403                                 " does not support CamcorderProfile, skipping");
1404                         continue;
1405                     }
1406 
1407                     // Re-use the MediaRecorder object for the same camera device.
1408                     mMediaRecorder = new MediaRecorder();
1409 
1410                     openDevice(id);
1411 
1412                     initSupportedVideoSize(id);
1413 
1414                     videoSnapshotTestByCamera(burstTest);
1415                 } finally {
1416                     closeDevice();
1417                     releaseRecorder();
1418                 }
1419             }
1420     }
1421 
1422     /**
1423      * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported.
1424      *
1425      * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p>
1426      *
1427      * @param profileId a {@link CamcorderProfile} ID to check.
1428      * @return {@code true} if supported.
1429      */
allowedUnsupported(int cameraId, int profileId)1430     private boolean allowedUnsupported(int cameraId, int profileId) {
1431         if (!mStaticInfo.isHardwareLevelLegacy()) {
1432             return false;
1433         }
1434 
1435         switch(profileId) {
1436             case CamcorderProfile.QUALITY_2160P:
1437             case CamcorderProfile.QUALITY_1080P:
1438             case CamcorderProfile.QUALITY_HIGH:
1439                 return !CamcorderProfile.hasProfile(cameraId, profileId) ||
1440                         CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080;
1441         }
1442         return false;
1443     }
1444 
1445     /**
1446      * Test video snapshot for each  available CamcorderProfile for a given camera.
1447      *
1448      * <p>
1449      * Preview size is set to the video size. For the burst test, frame drop and jittering
1450      * is not checked.
1451      * </p>
1452      *
1453      * @param burstTest Perform burst capture or single capture. For burst capture
1454      *                  {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent.
1455      */
videoSnapshotTestByCamera(boolean burstTest)1456     private void videoSnapshotTestByCamera(boolean burstTest)
1457             throws Exception {
1458         final int NUM_SINGLE_SHOT_TEST = 5;
1459         final int FRAMEDROP_TOLERANCE = 8;
1460         final int FRAME_SIZE_15M = 15000000;
1461         final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f;
1462         int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE;
1463 
1464         for (int profileId : mCamcorderProfileList) {
1465             int cameraId = Integer.valueOf(mCamera.getId());
1466             if (!CamcorderProfile.hasProfile(cameraId, profileId) ||
1467                     allowedUnsupported(cameraId, profileId)) {
1468                 continue;
1469             }
1470 
1471             CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
1472             Size QCIF = new Size(176, 144);
1473             Size FULL_HD = new Size(1920, 1080);
1474             Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1475             Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1476 
1477             if (mStaticInfo.isHardwareLevelLegacy() &&
1478                     (videoSz.getWidth() > maxPreviewSize.getWidth() ||
1479                      videoSz.getHeight() > maxPreviewSize.getHeight())) {
1480                 // Skip. Legacy mode can only do recording up to max preview size
1481                 continue;
1482             }
1483 
1484             if (!mSupportedVideoSizes.contains(videoSz)) {
1485                 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " +
1486                         profileId + " must be one of the camera device supported video size!");
1487                 continue;
1488             }
1489 
1490             // For LEGACY, find closest supported smaller or equal JPEG size to the current video
1491             // size; if no size is smaller than the video, pick the smallest JPEG size.  The assert
1492             // for video size above guarantees that for LIMITED or FULL, we select videoSz here.
1493             // Also check for minFrameDuration here to make sure jpeg stream won't slow down
1494             // video capture
1495             Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1);
1496             // Allow a bit tolerance so we don't fail for a few nano seconds of difference
1497             final float FRAME_DURATION_TOLERANCE = 0.01f;
1498             long videoFrameDuration = (long) (1e9 / profile.videoFrameRate *
1499                     (1.0 + FRAME_DURATION_TOLERANCE));
1500             HashMap<Size, Long> minFrameDurationMap = mStaticInfo.
1501                     getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG);
1502             for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) {
1503                 Size candidateSize = mOrderedStillSizes.get(i);
1504                 if (mStaticInfo.isHardwareLevelLegacy()) {
1505                     // Legacy level doesn't report min frame duration
1506                     if (candidateSize.getWidth() <= videoSz.getWidth() &&
1507                             candidateSize.getHeight() <= videoSz.getHeight()) {
1508                         videoSnapshotSz = candidateSize;
1509                     }
1510                 } else {
1511                     Long jpegFrameDuration = minFrameDurationMap.get(candidateSize);
1512                     assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize,
1513                             jpegFrameDuration != null);
1514                     if (candidateSize.getWidth() <= videoSz.getWidth() &&
1515                             candidateSize.getHeight() <= videoSz.getHeight() &&
1516                             jpegFrameDuration <= videoFrameDuration) {
1517                         videoSnapshotSz = candidateSize;
1518                     }
1519                 }
1520             }
1521             Size defaultvideoSnapshotSz = videoSnapshotSz;
1522 
1523             /**
1524              * Only test full res snapshot when below conditions are all true.
1525              * 1. Camera is at least a LIMITED device.
1526              * 2. video size is up to max preview size, which will be bounded by 1080p.
1527              * 3. Full resolution jpeg stream can keep up to video stream speed.
1528              *    When full res jpeg stream cannot keep up to video stream speed, search
1529              *    the largest jpeg size that can susptain video speed instead.
1530              */
1531             if (mStaticInfo.isHardwareLevelAtLeastLimited() &&
1532                     videoSz.getWidth() <= maxPreviewSize.getWidth() &&
1533                     videoSz.getHeight() <= maxPreviewSize.getHeight()) {
1534                 for (Size jpegSize : mOrderedStillSizes) {
1535                     Long jpegFrameDuration = minFrameDurationMap.get(jpegSize);
1536                     assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize,
1537                             jpegFrameDuration != null);
1538                     if (jpegFrameDuration <= videoFrameDuration) {
1539                         videoSnapshotSz = jpegSize;
1540                         break;
1541                     }
1542                     if (jpegSize.equals(videoSz)) {
1543                         throw new AssertionFailedError(
1544                                 "Cannot find adequate video snapshot size for video size" +
1545                                         videoSz);
1546                     }
1547                 }
1548             }
1549 
1550             if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M)
1551                 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR);
1552 
1553             createImageReader(
1554                     videoSnapshotSz, ImageFormat.JPEG,
1555                     MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1556 
1557             // Full or better devices should support whatever video snapshot size calculated above.
1558             // Limited devices may only be able to support the default one.
1559             if (mStaticInfo.isHardwareLevelLimited()) {
1560                 List<Surface> outputs = new ArrayList<Surface>();
1561                 outputs.add(mPreviewSurface);
1562                 outputs.add(mRecordingSurface);
1563                 outputs.add(mReaderSurface);
1564                 boolean isSupported = isStreamConfigurationSupported(
1565                         mCamera, outputs, mSessionListener, mHandler);
1566                 if (!isSupported) {
1567                     videoSnapshotSz = defaultvideoSnapshotSz;
1568                     createImageReader(
1569                             videoSnapshotSz, ImageFormat.JPEG,
1570                             MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1571                 }
1572             }
1573 
1574             if (videoSz.equals(QCIF) &&
1575                     ((videoSnapshotSz.getWidth() > FULL_HD.getWidth()) ||
1576                      (videoSnapshotSz.getHeight() > FULL_HD.getHeight()))) {
1577                 List<Surface> outputs = new ArrayList<Surface>();
1578                 outputs.add(mPreviewSurface);
1579                 outputs.add(mRecordingSurface);
1580                 outputs.add(mReaderSurface);
1581                 boolean isSupported = isStreamConfigurationSupported(
1582                         mCamera, outputs, mSessionListener, mHandler);
1583                 if (!isSupported) {
1584                     videoSnapshotSz = defaultvideoSnapshotSz;
1585                     createImageReader(
1586                             videoSnapshotSz, ImageFormat.JPEG,
1587                             MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1588                 }
1589             }
1590 
1591             Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz +
1592                     " for video size " + videoSz);
1593 
1594             if (VERBOSE) {
1595                 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString());
1596             }
1597 
1598             // Configure preview and recording surfaces.
1599             mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1600             if (DEBUG_DUMP) {
1601                 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + cameraId + "_"
1602                         + videoSz.toString() + ".mp4";
1603             }
1604 
1605             int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST;
1606             int totalDroppedFrames = 0;
1607 
1608             for (int numTested = 0; numTested < numTestIterations; numTested++) {
1609                 prepareRecordingWithProfile(profile);
1610 
1611                 // prepare video snapshot
1612                 SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1613                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener();
1614                 CaptureRequest.Builder videoSnapshotRequestBuilder =
1615                         mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ?
1616                                 CameraDevice.TEMPLATE_RECORD :
1617                                 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT);
1618 
1619                 // prepare preview surface by using video size.
1620                 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate);
1621 
1622                 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener);
1623                 Range<Integer> fpsRange = Range.create(profile.videoFrameRate,
1624                         profile.videoFrameRate);
1625                 videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE,
1626                         fpsRange);
1627                 if (mStaticInfo.isVideoStabilizationSupported()) {
1628                     videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1629                             CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1630                 }
1631                 CaptureRequest request = videoSnapshotRequestBuilder.build();
1632 
1633                 // Start recording
1634                 startRecording(/* useMediaRecorder */true, resultListener,
1635                         /*useVideoStab*/mStaticInfo.isVideoStabilizationSupported());
1636                 long startTime = SystemClock.elapsedRealtime();
1637 
1638                 // Record certain duration.
1639                 SystemClock.sleep(RECORDING_DURATION_MS / 2);
1640 
1641                 // take video snapshot
1642                 if (burstTest) {
1643                     List<CaptureRequest> requests =
1644                             new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM);
1645                     for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) {
1646                         requests.add(request);
1647                     }
1648                     mSession.captureBurst(requests, resultListener, mHandler);
1649                 } else {
1650                     mSession.capture(request, resultListener, mHandler);
1651                 }
1652 
1653                 // make sure recording is still going after video snapshot
1654                 SystemClock.sleep(RECORDING_DURATION_MS / 2);
1655 
1656                 // Stop recording and preview
1657                 float durationMs = (float) stopRecording(/* useMediaRecorder */true);
1658                 // For non-burst test, use number of frames to also double check video frame rate.
1659                 // Burst video snapshot is allowed to cause frame rate drop, so do not use number
1660                 // of frames to estimate duration
1661                 if (!burstTest) {
1662                     durationMs = resultListener.getTotalNumFrames() * 1000.0f /
1663                         profile.videoFrameRate;
1664                 }
1665 
1666                 float frameDurationMs = 1000.0f / profile.videoFrameRate;
1667                 // Validation recorded video
1668                 validateRecording(videoSz, durationMs,
1669                         frameDurationMs, VID_SNPSHT_FRMDRP_RATE_TOLERANCE);
1670 
1671                 if (burstTest) {
1672                     for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) {
1673                         Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS);
1674                         validateVideoSnapshotCapture(image, videoSnapshotSz);
1675                         image.close();
1676                     }
1677                 } else {
1678                     // validate video snapshot image
1679                     Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS);
1680                     validateVideoSnapshotCapture(image, videoSnapshotSz);
1681 
1682                     // validate if there is framedrop around video snapshot
1683                     totalDroppedFrames +=  validateFrameDropAroundVideoSnapshot(
1684                             resultListener, image.getTimestamp());
1685 
1686                     //TODO: validate jittering. Should move to PTS
1687                     //validateJittering(resultListener);
1688 
1689                     image.close();
1690                 }
1691             }
1692 
1693             if (!burstTest) {
1694                 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " +
1695                         "detected in %d trials is %d frames.", cameraId, videoSz.toString(),
1696                         numTestIterations, totalDroppedFrames));
1697                 mCollector.expectLessOrEqual(
1698                         String.format(
1699                                 "Camera %d Video size %s: Number of dropped frames %d must not"
1700                                 + " be larger than %d",
1701                                 cameraId, videoSz.toString(), totalDroppedFrames,
1702                                 kFrameDrop_Tolerence),
1703                         kFrameDrop_Tolerence, totalDroppedFrames);
1704             }
1705             closeImageReader();
1706         }
1707     }
1708 
1709     /**
1710      * Configure video snapshot request according to the still capture size
1711      */
prepareVideoSnapshot( CaptureRequest.Builder requestBuilder, ImageReader.OnImageAvailableListener imageListener)1712     private void prepareVideoSnapshot(
1713             CaptureRequest.Builder requestBuilder,
1714             ImageReader.OnImageAvailableListener imageListener)
1715             throws Exception {
1716         mReader.setOnImageAvailableListener(imageListener, mHandler);
1717         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1718         requestBuilder.addTarget(mRecordingSurface);
1719         assertNotNull("Preview surface must be non-null!", mPreviewSurface);
1720         requestBuilder.addTarget(mPreviewSurface);
1721         assertNotNull("Reader surface must be non-null!", mReaderSurface);
1722         requestBuilder.addTarget(mReaderSurface);
1723     }
1724 
1725     /**
1726      * Find compatible preview sizes for video size and framerate.
1727      *
1728      * <p>Preview size will be capped with max preview size.</p>
1729      *
1730      * @param videoSize The video size used for preview.
1731      * @param videoFrameRate The video frame rate
1732      */
getPreviewSizesForVideo(Size videoSize, int videoFrameRate)1733     private List<Size> getPreviewSizesForVideo(Size videoSize, int videoFrameRate) {
1734         if (mOrderedPreviewSizes == null) {
1735             throw new IllegalStateException("supported preview size list is not initialized yet");
1736         }
1737         final float FRAME_DURATION_TOLERANCE = 0.01f;
1738         long videoFrameDuration = (long) (1e9 / videoFrameRate *
1739                 (1.0 + FRAME_DURATION_TOLERANCE));
1740         HashMap<Size, Long> minFrameDurationMap = mStaticInfo.
1741                 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE);
1742         Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1743         ArrayList<Size> previewSizes = new ArrayList<>();
1744         if (videoSize.getWidth() > maxPreviewSize.getWidth() ||
1745                 videoSize.getHeight() > maxPreviewSize.getHeight()) {
1746             for (Size s : mOrderedPreviewSizes) {
1747                 Long frameDuration = minFrameDurationMap.get(s);
1748                 if (mStaticInfo.isHardwareLevelLegacy()) {
1749                     // Legacy doesn't report min frame duration
1750                     frameDuration = new Long(0);
1751                 }
1752                 assertTrue("Cannot find minimum frame duration for private size" + s,
1753                         frameDuration != null);
1754                 if (frameDuration <= videoFrameDuration &&
1755                         s.getWidth() <= videoSize.getWidth() &&
1756                         s.getHeight() <= videoSize.getHeight()) {
1757                     Log.v(TAG, "Add preview size " + s.toString() + " for video size " +
1758                             videoSize.toString());
1759                     previewSizes.add(s);
1760                 }
1761             }
1762         }
1763 
1764         if (previewSizes.isEmpty()) {
1765             previewSizes.add(videoSize);
1766         }
1767 
1768         return previewSizes;
1769     }
1770 
1771     /**
1772      * Update preview size with video size.
1773      *
1774      * <p>Preview size will be capped with max preview size.</p>
1775      *
1776      * @param videoSize The video size used for preview.
1777      * @param videoFrameRate The video frame rate
1778      *
1779      */
updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate)1780     private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) {
1781         List<Size> previewSizes = getPreviewSizesForVideo(videoSize, videoFrameRate);
1782         updatePreviewSurface(previewSizes.get(0));
1783     }
1784 
prepareRecordingWithProfile(CamcorderProfile profile)1785     private void prepareRecordingWithProfile(CamcorderProfile profile) throws Exception {
1786         prepareRecordingWithProfile(profile, false);
1787     }
1788 
1789     /**
1790      * Configure MediaRecorder recording session with CamcorderProfile, prepare
1791      * the recording surface.
1792      */
prepareRecordingWithProfile(CamcorderProfile profile, boolean useIntermediateSurface)1793     private void prepareRecordingWithProfile(CamcorderProfile profile,
1794             boolean useIntermediateSurface) throws Exception {
1795         // Prepare MediaRecorder.
1796         setupMediaRecorder(profile);
1797         prepareRecording(useIntermediateSurface);
1798     }
1799 
setupMediaRecorder(CamcorderProfile profile)1800     private void setupMediaRecorder(CamcorderProfile profile) throws Exception {
1801         // Set-up MediaRecorder.
1802         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1803         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1804         mMediaRecorder.setProfile(profile);
1805 
1806         mVideoFrameRate = profile.videoFrameRate;
1807         mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1808     }
1809 
setupMediaRecorder( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile)1810     private void setupMediaRecorder(
1811             EncoderProfiles profiles,
1812             EncoderProfiles.VideoProfile videoProfile,
1813             EncoderProfiles.AudioProfile audioProfile) throws Exception {
1814         // Set-up MediaRecorder.
1815         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1816         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1817         mMediaRecorder.setOutputFormat(profiles.getRecommendedFileFormat());
1818         mMediaRecorder.setVideoProfile(videoProfile);
1819         if (audioProfile != null) {
1820             mMediaRecorder.setAudioProfile(audioProfile);
1821         }
1822 
1823         mVideoFrameRate = videoProfile.getFrameRate();
1824         mVideoSize = new Size(videoProfile.getWidth(), videoProfile.getHeight());
1825     }
1826 
prepareRecording(boolean useIntermediateSurface)1827     private void prepareRecording(boolean useIntermediateSurface) throws Exception {
1828         // Continue preparing MediaRecorder
1829         mMediaRecorder.setOutputFile(mOutMediaFileName);
1830         if (mPersistentSurface != null) {
1831             mMediaRecorder.setInputSurface(mPersistentSurface);
1832             mRecordingSurface = mPersistentSurface;
1833         }
1834         mMediaRecorder.prepare();
1835         if (mPersistentSurface == null) {
1836             mRecordingSurface = mMediaRecorder.getSurface();
1837         }
1838         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1839 
1840         if (useIntermediateSurface) {
1841             mIntermediateReader = ImageReader.newInstance(
1842                     mVideoSize.getWidth(), mVideoSize.getHeight(),
1843                     ImageFormat.PRIVATE, /*maxImages*/3, HardwareBuffer.USAGE_VIDEO_ENCODE);
1844 
1845             mIntermediateSurface = mIntermediateReader.getSurface();
1846             mIntermediateWriter = ImageWriter.newInstance(mRecordingSurface, /*maxImages*/3,
1847                     ImageFormat.PRIVATE);
1848             mQueuer = new ImageWriterQueuer(mIntermediateWriter);
1849 
1850             mIntermediateThread = new HandlerThread(TAG);
1851             mIntermediateThread.start();
1852             mIntermediateHandler = new Handler(mIntermediateThread.getLooper());
1853             mIntermediateReader.setOnImageAvailableListener(mQueuer, mIntermediateHandler);
1854         }
1855     }
1856 
1857     /**
1858      * Configure MediaRecorder recording session with CamcorderProfile, prepare
1859      * the recording surface. Use AVC for video compression, AAC for audio compression.
1860      * Both are required for android devices by android CDD.
1861      */
prepareRecording(Size sz, int videoFrameRate, int captureRate)1862     private void prepareRecording(Size sz, int videoFrameRate, int captureRate)
1863             throws Exception {
1864         // Prepare MediaRecorder.
1865         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1866         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1867         mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP);
1868         mMediaRecorder.setOutputFile(mOutMediaFileName);
1869         mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz));
1870         mMediaRecorder.setVideoFrameRate(videoFrameRate);
1871         mMediaRecorder.setCaptureRate(captureRate);
1872         mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight());
1873         mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264);
1874         mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
1875         if (mPersistentSurface != null) {
1876             mMediaRecorder.setInputSurface(mPersistentSurface);
1877             mRecordingSurface = mPersistentSurface;
1878         }
1879         mMediaRecorder.prepare();
1880         if (mPersistentSurface == null) {
1881             mRecordingSurface = mMediaRecorder.getSurface();
1882         }
1883         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1884         mVideoFrameRate = videoFrameRate;
1885         mVideoSize = sz;
1886     }
1887 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab)1888     private void startRecording(boolean useMediaRecorder,
1889             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab) throws Exception {
1890         startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null,
1891                 /*useIntermediateSurface*/false);
1892     }
1893 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, boolean useIntermediateSurface)1894     private void startRecording(boolean useMediaRecorder,
1895             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1896             boolean useIntermediateSurface) throws Exception {
1897         startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null,
1898                 useIntermediateSurface);
1899     }
1900 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange, boolean useIntermediateSurface)1901     private void startRecording(boolean useMediaRecorder,
1902             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1903             Range<Integer> variableFpsRange, boolean useIntermediateSurface) throws Exception {
1904         if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) {
1905             throw new IllegalArgumentException("Video stabilization is not supported");
1906         }
1907 
1908         List<Surface> outputSurfaces = new ArrayList<Surface>(2);
1909         assertTrue("Both preview and recording surfaces should be valid",
1910                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1911         outputSurfaces.add(mPreviewSurface);
1912         if (useIntermediateSurface) {
1913             outputSurfaces.add(mIntermediateSurface);
1914         } else {
1915             outputSurfaces.add(mRecordingSurface);
1916         }
1917 
1918         // Video snapshot surface
1919         if (mReaderSurface != null) {
1920             outputSurfaces.add(mReaderSurface);
1921         }
1922         mSessionListener = new BlockingSessionCallback();
1923 
1924         CaptureRequest.Builder recordingRequestBuilder =
1925                 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1926         // Make sure camera output frame rate is set to correct value.
1927         Range<Integer> fpsRange = (variableFpsRange == null) ?
1928                 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange;
1929 
1930         recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1931         if (useVideoStab) {
1932             recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1933                     CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1934         }
1935         if (useIntermediateSurface) {
1936             recordingRequestBuilder.addTarget(mIntermediateSurface);
1937             if (mQueuer != null) {
1938                 mQueuer.resetInvalidSurfaceFlag();
1939             }
1940         } else {
1941             recordingRequestBuilder.addTarget(mRecordingSurface);
1942         }
1943         recordingRequestBuilder.addTarget(mPreviewSurface);
1944         CaptureRequest recordingRequest = recordingRequestBuilder.build();
1945         mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces, mSessionListener,
1946                 mHandler, recordingRequest);
1947         mSession.setRepeatingRequest(recordingRequest, listener, mHandler);
1948 
1949         if (useMediaRecorder) {
1950             mMediaRecorder.start();
1951         } else {
1952             // TODO: need implement MediaCodec path.
1953         }
1954         mRecordingStartTime = SystemClock.elapsedRealtime();
1955     }
1956 
1957     /**
1958      * Start video recording with preview and video surfaces sharing the same
1959      * camera stream.
1960      *
1961      * @return true if success, false if sharing is not supported.
1962      */
startSharedRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange)1963     private boolean startSharedRecording(boolean useMediaRecorder,
1964             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1965             Range<Integer> variableFpsRange) throws Exception {
1966         if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) {
1967             throw new IllegalArgumentException("Video stabilization is not supported");
1968         }
1969 
1970         List<OutputConfiguration> outputConfigs = new ArrayList<OutputConfiguration>(2);
1971         assertTrue("Both preview and recording surfaces should be valid",
1972                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1973         OutputConfiguration sharedConfig = new OutputConfiguration(mPreviewSurface);
1974         sharedConfig.enableSurfaceSharing();
1975         sharedConfig.addSurface(mRecordingSurface);
1976         outputConfigs.add(sharedConfig);
1977 
1978         CaptureRequest.Builder recordingRequestBuilder =
1979                 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1980         // Make sure camera output frame rate is set to correct value.
1981         Range<Integer> fpsRange = (variableFpsRange == null) ?
1982                 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange;
1983         recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1984         if (useVideoStab) {
1985             recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1986                     CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1987         }
1988         CaptureRequest recordingRequest = recordingRequestBuilder.build();
1989 
1990         mSessionListener = new BlockingSessionCallback();
1991         mSession = tryConfigureCameraSessionWithConfig(mCamera, outputConfigs, recordingRequest,
1992                 mSessionListener, mHandler);
1993 
1994         if (mSession == null) {
1995             Log.i(TAG, "Sharing between preview and video is not supported");
1996             return false;
1997         }
1998 
1999         recordingRequestBuilder.addTarget(mRecordingSurface);
2000         recordingRequestBuilder.addTarget(mPreviewSurface);
2001         mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler);
2002 
2003         if (useMediaRecorder) {
2004             mMediaRecorder.start();
2005         } else {
2006             // TODO: need implement MediaCodec path.
2007         }
2008         mRecordingStartTime = SystemClock.elapsedRealtime();
2009         return true;
2010     }
2011 
2012 
stopCameraStreaming()2013     private void stopCameraStreaming() throws Exception {
2014         if (VERBOSE) {
2015             Log.v(TAG, "Stopping camera streaming and waiting for idle");
2016         }
2017         // Stop repeating, wait for captures to complete, and disconnect from
2018         // surfaces
2019         mSession.close();
2020         mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS);
2021     }
2022 
stopRecording(boolean useMediaRecorder)2023     private int stopRecording(boolean useMediaRecorder) throws Exception {
2024         return stopRecording(useMediaRecorder, /*useIntermediateSurface*/false,
2025                 /*stopStreaming*/true);
2026     }
2027 
2028     // Stop recording and return the estimated video duration in milliseconds.
stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface, boolean stopStreaming)2029     private int stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface,
2030             boolean stopStreaming) throws Exception {
2031         long stopRecordingTime = SystemClock.elapsedRealtime();
2032         if (useMediaRecorder) {
2033             if (stopStreaming) {
2034                 stopCameraStreaming();
2035             }
2036             if (useIntermediateSurface) {
2037                 mIntermediateReader.setOnImageAvailableListener(null, null);
2038                 mQueuer.expectInvalidSurface();
2039             }
2040 
2041             mMediaRecorder.stop();
2042             // Can reuse the MediaRecorder object after reset.
2043             mMediaRecorder.reset();
2044         } else {
2045             // TODO: need implement MediaCodec path.
2046         }
2047 
2048         if (useIntermediateSurface) {
2049             mIntermediateReader.close();
2050             mQueuer.close();
2051             mIntermediateWriter.close();
2052             mIntermediateSurface.release();
2053             mIntermediateReader = null;
2054             mIntermediateSurface = null;
2055             mIntermediateWriter = null;
2056             mIntermediateThread.quitSafely();
2057             mIntermediateHandler = null;
2058         }
2059 
2060         if (mPersistentSurface == null && mRecordingSurface != null) {
2061             mRecordingSurface.release();
2062             mRecordingSurface = null;
2063         }
2064         return (int) (stopRecordingTime - mRecordingStartTime);
2065     }
2066 
releaseRecorder()2067     private void releaseRecorder() {
2068         if (mMediaRecorder != null) {
2069             mMediaRecorder.release();
2070             mMediaRecorder = null;
2071         }
2072     }
2073 
validateRecording( Size sz, float expectedDurationMs, float expectedFrameDurationMs, float frameDropTolerance)2074     private void validateRecording(
2075             Size sz, float expectedDurationMs, float expectedFrameDurationMs,
2076             float frameDropTolerance) throws Exception {
2077         validateRecording(sz,
2078                 expectedDurationMs,  /*fixed FPS recording*/0.f,
2079                 expectedFrameDurationMs, /*fixed FPS recording*/0.f,
2080                 frameDropTolerance);
2081     }
2082 
validateRecording( Size sz, float expectedDurationMinMs, float expectedDurationMaxMs, float expectedFrameDurationMinMs, float expectedFrameDurationMaxMs, float frameDropTolerance)2083     private void validateRecording(
2084             Size sz,
2085             float expectedDurationMinMs,      // Min duration (maxFps)
2086             float expectedDurationMaxMs,      // Max duration (minFps). 0.f for fixed fps recording
2087             float expectedFrameDurationMinMs, // maxFps
2088             float expectedFrameDurationMaxMs, // minFps. 0.f for fixed fps recording
2089             float frameDropTolerance) throws Exception {
2090         File outFile = new File(mOutMediaFileName);
2091         assertTrue("No video is recorded", outFile.exists());
2092         float maxFrameDuration = expectedFrameDurationMinMs * (1.0f + FRAMEDURATION_MARGIN);
2093         if (expectedFrameDurationMaxMs > 0.f) {
2094             maxFrameDuration = expectedFrameDurationMaxMs * (1.0f + FRAMEDURATION_MARGIN);
2095         }
2096 
2097         if (expectedDurationMaxMs == 0.f) {
2098             expectedDurationMaxMs = expectedDurationMinMs;
2099         }
2100 
2101         MediaExtractor extractor = new MediaExtractor();
2102         try {
2103             extractor.setDataSource(mOutMediaFileName);
2104             long durationUs = 0;
2105             int width = -1, height = -1;
2106             int numTracks = extractor.getTrackCount();
2107             int selectedTrack = -1;
2108             final String VIDEO_MIME_TYPE = "video";
2109             for (int i = 0; i < numTracks; i++) {
2110                 MediaFormat format = extractor.getTrackFormat(i);
2111                 String mime = format.getString(MediaFormat.KEY_MIME);
2112                 if (mime.contains(VIDEO_MIME_TYPE)) {
2113                     Log.i(TAG, "video format is: " + format.toString());
2114                     durationUs = format.getLong(MediaFormat.KEY_DURATION);
2115                     width = format.getInteger(MediaFormat.KEY_WIDTH);
2116                     height = format.getInteger(MediaFormat.KEY_HEIGHT);
2117                     selectedTrack = i;
2118                     extractor.selectTrack(i);
2119                     break;
2120                 }
2121             }
2122             if (selectedTrack < 0) {
2123                 throw new AssertionFailedError(
2124                         "Cannot find video track!");
2125             }
2126 
2127             Size videoSz = new Size(width, height);
2128             assertTrue("Video size doesn't match, expected " + sz.toString() +
2129                     " got " + videoSz.toString(), videoSz.equals(sz));
2130             float duration = (float) (durationUs / 1000);
2131             if (VERBOSE) {
2132                 Log.v(TAG, String.format("Video duration: recorded %fms, expected [%f,%f]ms",
2133                                          duration, expectedDurationMinMs, expectedDurationMaxMs));
2134             }
2135 
2136             // Do rest of validation only for better-than-LEGACY devices
2137             if (mStaticInfo.isHardwareLevelLegacy()) return;
2138 
2139             // TODO: Don't skip this one for video snapshot on LEGACY
2140             assertTrue(String.format(
2141                     "Camera %s: Video duration doesn't match: recorded %fms, expected [%f,%f]ms.",
2142                     mCamera.getId(), duration,
2143                     expectedDurationMinMs * (1.f - DURATION_MARGIN),
2144                     expectedDurationMaxMs * (1.f + DURATION_MARGIN)),
2145                     duration > expectedDurationMinMs * (1.f - DURATION_MARGIN) &&
2146                             duration < expectedDurationMaxMs * (1.f + DURATION_MARGIN));
2147 
2148             // Check for framedrop
2149             long lastSampleUs = 0;
2150             int frameDropCount = 0;
2151             int expectedFrameCount = (int) (expectedDurationMinMs / expectedFrameDurationMinMs);
2152             ArrayList<Long> timestamps = new ArrayList<Long>(expectedFrameCount);
2153             while (true) {
2154                 timestamps.add(extractor.getSampleTime());
2155                 if (!extractor.advance()) {
2156                     break;
2157                 }
2158             }
2159             Collections.sort(timestamps);
2160             long prevSampleUs = timestamps.get(0);
2161             for (int i = 1; i < timestamps.size(); i++) {
2162                 long currentSampleUs = timestamps.get(i);
2163                 float frameDurationMs = (float) (currentSampleUs - prevSampleUs) / 1000;
2164                 if (frameDurationMs > maxFrameDuration) {
2165                     Log.w(TAG, String.format(
2166                         "Frame drop at %d: expectation %f, observed %f",
2167                         i, expectedFrameDurationMinMs, frameDurationMs));
2168                     frameDropCount++;
2169                 }
2170                 prevSampleUs = currentSampleUs;
2171             }
2172             float frameDropRate = 100.f * frameDropCount / timestamps.size();
2173             Log.i(TAG, String.format("Frame drop rate %d/%d (%f%%)",
2174                 frameDropCount, timestamps.size(), frameDropRate));
2175             assertTrue(String.format(
2176                     "Camera %s: Video frame drop rate too high: %f%%, tolerance %f%%. " +
2177                     "Video size: %s, expectedDuration [%f,%f], expectedFrameDuration %f, " +
2178                     "frameDropCnt %d, frameCount %d",
2179                     mCamera.getId(), frameDropRate, frameDropTolerance,
2180                     sz.toString(), expectedDurationMinMs, expectedDurationMaxMs,
2181                     expectedFrameDurationMinMs, frameDropCount, timestamps.size()),
2182                     frameDropRate < frameDropTolerance);
2183         } finally {
2184             extractor.release();
2185             if (!DEBUG_DUMP) {
2186                 outFile.delete();
2187             }
2188         }
2189     }
2190 
2191     /**
2192      * Validate video snapshot capture image object validity and test.
2193      *
2194      * <p> Check for size, format and jpeg decoding</p>
2195      *
2196      * @param image The JPEG image to be verified.
2197      * @param size The JPEG capture size to be verified against.
2198      */
2199     private void validateVideoSnapshotCapture(Image image, Size size) {
2200         CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(),
2201                 ImageFormat.JPEG, /*filePath*/null);
2202     }
2203 
2204     /**
2205      * Validate if video snapshot causes frame drop.
2206      * Here frame drop is defined as frame duration >= 2 * expected frame duration.
2207      * Return the estimated number of frames dropped during video snapshot
2208      */
2209     private int validateFrameDropAroundVideoSnapshot(
2210             SimpleCaptureCallback resultListener, long imageTimeStamp) {
2211         double expectedDurationMs = 1000.0 / mVideoFrameRate;
2212         CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2213         long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP);
2214         while (resultListener.hasMoreResults()) {
2215             CaptureResult currentResult =
2216                     resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2217             long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP);
2218             if (currentTS == imageTimeStamp) {
2219                 // validate the timestamp before and after, then return
2220                 CaptureResult nextResult =
2221                         resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2222                 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP);
2223                 double durationMs = (currentTS - prevTS) / 1000000.0;
2224                 int totalFramesDropped = 0;
2225 
2226                 // Snapshots in legacy mode pause the preview briefly.  Skip the duration
2227                 // requirements for legacy mode unless this is fixed.
2228                 if (!mStaticInfo.isHardwareLevelLegacy()) {
2229                     mCollector.expectTrue(
2230                             String.format(
2231                                     "Video %dx%d Frame drop detected before video snapshot: " +
2232                                             "duration %.2fms (expected %.2fms)",
2233                                     mVideoSize.getWidth(), mVideoSize.getHeight(),
2234                                     durationMs, expectedDurationMs
2235                             ),
2236                             durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED)
2237                     );
2238                     // Log a warning is there is any frame drop detected.
2239                     if (durationMs >= expectedDurationMs * 2) {
2240                         Log.w(TAG, String.format(
2241                                 "Video %dx%d Frame drop detected before video snapshot: " +
2242                                         "duration %.2fms (expected %.2fms)",
2243                                 mVideoSize.getWidth(), mVideoSize.getHeight(),
2244                                 durationMs, expectedDurationMs
2245                         ));
2246                     }
2247 
2248                     durationMs = (nextTS - currentTS) / 1000000.0;
2249                     mCollector.expectTrue(
2250                             String.format(
2251                                     "Video %dx%d Frame drop detected after video snapshot: " +
2252                                             "duration %.2fms (expected %.2fms)",
2253                                     mVideoSize.getWidth(), mVideoSize.getHeight(),
2254                                     durationMs, expectedDurationMs
2255                             ),
2256                             durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED)
2257                     );
2258                     // Log a warning is there is any frame drop detected.
2259                     if (durationMs >= expectedDurationMs * 2) {
2260                         Log.w(TAG, String.format(
2261                                 "Video %dx%d Frame drop detected after video snapshot: " +
2262                                         "duration %fms (expected %fms)",
2263                                 mVideoSize.getWidth(), mVideoSize.getHeight(),
2264                                 durationMs, expectedDurationMs
2265                         ));
2266                     }
2267 
2268                     double totalDurationMs = (nextTS - prevTS) / 1000000.0;
2269                     // Minus 2 for the expected 2 frames interval
2270                     totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2;
2271                     if (totalFramesDropped < 0) {
2272                         Log.w(TAG, "totalFrameDropped is " + totalFramesDropped +
2273                                 ". Video frame rate might be too fast.");
2274                     }
2275                     totalFramesDropped = Math.max(0, totalFramesDropped);
2276                 }
2277                 return totalFramesDropped;
2278             }
2279             prevTS = currentTS;
2280         }
2281         throw new AssertionFailedError(
2282                 "Video snapshot timestamp does not match any of capture results!");
2283     }
2284 
2285     /**
2286      * Validate frame jittering from the input simple listener's buffered results
2287      */
2288     private void validateJittering(SimpleCaptureCallback resultListener) {
2289         double expectedDurationMs = 1000.0 / mVideoFrameRate;
2290         CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2291         long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP);
2292         while (resultListener.hasMoreResults()) {
2293             CaptureResult currentResult =
2294                     resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2295             long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP);
2296             double durationMs = (currentTS - prevTS) / 1000000.0;
2297             double durationError = Math.abs(durationMs - expectedDurationMs);
2298             long frameNumber = currentResult.getFrameNumber();
2299             mCollector.expectTrue(
2300                     String.format(
2301                             "Resolution %dx%d Frame %d: jittering (%.2fms) exceeds bound [%.2fms,%.2fms]",
2302                             mVideoSize.getWidth(), mVideoSize.getHeight(),
2303                             frameNumber, durationMs,
2304                             expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS,
2305                             expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS),
2306                     durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS);
2307             prevTS = currentTS;
2308         }
2309     }
2310 
2311     /**
2312      * Calculate a video bit rate based on the size. The bit rate is scaled
2313      * based on ratio of video size to 1080p size.
2314      */
2315     private int getVideoBitRate(Size sz) {
2316         int rate = BIT_RATE_1080P;
2317         float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080);
2318         rate = (int)(rate * scaleFactor);
2319 
2320         // Clamp to the MIN, MAX range.
2321         return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate));
2322     }
2323 
2324     /**
2325      * Check if the encoder and camera are able to support this size and frame rate.
2326      * Assume the video compression format is AVC.
2327      */
2328     private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception {
2329         // Check camera capability.
2330         if (!isSupportedByCamera(sz, captureRate)) {
2331             return false;
2332         }
2333 
2334         // Check encode capability.
2335         if (!isSupportedByAVCEncoder(sz, encodingRate)){
2336             return false;
2337         }
2338 
2339         if(VERBOSE) {
2340             Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@"
2341                     + getVideoBitRate(sz) / 1000 + "Kbps");
2342         }
2343 
2344         return true;
2345     }
2346 
2347     private boolean isSupportedByCamera(Size sz, int frameRate) {
2348         // Check if camera can support this sz and frame rate combination.
2349         StreamConfigurationMap config = mStaticInfo.
2350                 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
2351 
2352         long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz);
2353         if (minDuration == 0) {
2354             return false;
2355         }
2356 
2357         int maxFrameRate = (int) (1e9f / minDuration);
2358         return maxFrameRate >= frameRate;
2359     }
2360 
2361     /**
2362      * Check if encoder can support this size and frame rate combination by querying
2363      * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate
2364      * as the bit rates targeted in this test are well below the bit rate max value specified
2365      * by AVC specification for certain level.
2366      */
2367     private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) {
2368         MediaFormat format = MediaFormat.createVideoFormat(
2369                 MediaFormat.MIMETYPE_VIDEO_AVC, sz.getWidth(), sz.getHeight());
2370         format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
2371         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2372         return mcl.findEncoderForFormat(format) != null;
2373     }
2374 
2375     private static class ImageWriterQueuer implements ImageReader.OnImageAvailableListener {
2376         public ImageWriterQueuer(ImageWriter writer) {
2377             mWriter = writer;
2378         }
2379 
2380         public void resetInvalidSurfaceFlag() {
2381             synchronized (mLock) {
2382                 mExpectInvalidSurface = false;
2383             }
2384         }
2385 
2386         // Indicate that the writer surface is about to get released
2387         // and become invalid.
2388         public void expectInvalidSurface() {
2389             // If we sync on 'mLock', we risk a possible deadlock
2390             // during 'mWriter.queueInputImage(image)' which is
2391             // called while the lock is held.
2392             mExpectInvalidSurface = true;
2393         }
2394 
2395         @Override
2396         public void onImageAvailable(ImageReader reader) {
2397             Image image = null;
2398             try {
2399                 image = reader.acquireNextImage();
2400             } finally {
2401                 synchronized (mLock) {
2402                     if (image != null && mWriter != null) {
2403                         try {
2404                             mWriter.queueInputImage(image);
2405                             mQueuedCount++;
2406                         } catch (IllegalStateException e) {
2407                             // Per API documentation ISE are possible
2408                             // in case the writer surface is not valid.
2409                             // Re-throw in case we have some other
2410                             // unexpected ISE.
2411                             if (mExpectInvalidSurface) {
2412                                 Log.d(TAG, "Invalid writer surface");
2413                                 image.close();
2414                             } else {
2415                                 throw e;
2416                             }
2417                         }
2418                     } else if (image != null) {
2419                         image.close();
2420                     }
2421                 }
2422             }
2423         }
2424 
2425         public int getQueuedCount() {
2426             synchronized (mLock) {
2427                 return mQueuedCount;
2428             }
2429         }
2430 
2431         public void close() {
2432             synchronized (mLock) {
2433                 mWriter = null;
2434             }
2435         }
2436 
2437         private Object      mLock = new Object();
2438         private ImageWriter mWriter = null;
2439         private int         mQueuedCount = 0;
2440         private boolean     mExpectInvalidSurface = false;
2441     }
2442 }
2443