1 /*
2  * Copyright (C) 2014 The Android Open Source Project Licensed under the Apache
3  * License, Version 2.0 (the "License"); you may not use this file except in
4  * compliance with the License. You may obtain a copy of the License at
5  * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law
6  * or agreed to in writing, software distributed under the License is
7  * distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
8  * KIND, either express or implied. See the License for the specific language
9  * governing permissions and limitations under the License.
10  */
11 
12 package android.hardware.camera2.cts;
13 
14 import static android.hardware.camera2.cts.CameraTestUtils.*;
15 import static com.android.ex.camera2.blocking.BlockingSessionCallback.*;
16 
17 import android.graphics.ImageFormat;
18 import android.graphics.SurfaceTexture;
19 import android.hardware.camera2.CameraCharacteristics;
20 import android.hardware.camera2.CameraCaptureSession;
21 import android.hardware.camera2.CameraConstrainedHighSpeedCaptureSession;
22 import android.hardware.camera2.CameraDevice;
23 import android.hardware.camera2.CaptureRequest;
24 import android.hardware.camera2.CaptureResult;
25 import android.hardware.camera2.cts.helpers.StaticMetadata;
26 import android.hardware.camera2.params.OutputConfiguration;
27 import android.hardware.camera2.params.SessionConfiguration;
28 import android.hardware.camera2.params.StreamConfigurationMap;
29 import android.hardware.HardwareBuffer;
30 import android.util.Size;
31 import android.hardware.camera2.cts.testcases.Camera2SurfaceViewTestCase;
32 import android.media.CamcorderProfile;
33 import android.media.EncoderProfiles;
34 import android.media.MediaCodec;
35 import android.media.MediaCodecInfo;
36 import android.media.MediaCodecInfo.CodecCapabilities;
37 import android.media.MediaCodecInfo.CodecProfileLevel;
38 import android.media.Image;
39 import android.media.ImageReader;
40 import android.media.ImageWriter;
41 import android.media.MediaCodecList;
42 import android.media.MediaExtractor;
43 import android.media.MediaFormat;
44 import android.media.MediaRecorder;
45 import android.os.Environment;
46 import android.os.Handler;
47 import android.os.HandlerThread;
48 import android.os.SystemClock;
49 import android.test.suitebuilder.annotation.LargeTest;
50 import android.util.Log;
51 import android.util.Range;
52 import android.view.Surface;
53 
54 import com.android.compatibility.common.util.MediaUtils;
55 import com.android.ex.camera2.blocking.BlockingSessionCallback;
56 
57 import junit.framework.AssertionFailedError;
58 
59 import org.junit.runners.Parameterized;
60 import org.junit.runner.RunWith;
61 import org.junit.Test;
62 
63 import java.io.File;
64 import java.util.ArrayList;
65 import java.util.Arrays;
66 import java.util.Collections;
67 import java.util.List;
68 import java.util.HashMap;
69 
70 /**
71  * CameraDevice video recording use case tests by using MediaRecorder and
72  * MediaCodec.
73  */
74 @LargeTest
75 @RunWith(Parameterized.class)
76 public class RecordingTest extends Camera2SurfaceViewTestCase {
77     private static final String TAG = "RecordingTest";
78     private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE);
79     private static final boolean DEBUG_DUMP = Log.isLoggable(TAG, Log.DEBUG);
80     private static final int RECORDING_DURATION_MS = 3000;
81     private static final int PREVIEW_DURATION_MS = 3000;
82     private static final float DURATION_MARGIN = 0.2f;
83     private static final double FRAME_DURATION_ERROR_TOLERANCE_MS = 3.0;
84     private static final float FRAMEDURATION_MARGIN = 0.2f;
85     private static final float VID_SNPSHT_FRMDRP_RATE_TOLERANCE = 10.0f;
86     private static final float FRMDRP_RATE_TOLERANCE = 5.0f;
87     private static final int BIT_RATE_1080P = 16000000;
88     private static final int BIT_RATE_MIN = 64000;
89     private static final int BIT_RATE_MAX = 40000000;
90     private static final int VIDEO_FRAME_RATE = 30;
91     private static final int[] mCamcorderProfileList = {
92             CamcorderProfile.QUALITY_HIGH,
93             CamcorderProfile.QUALITY_2160P,
94             CamcorderProfile.QUALITY_1080P,
95             CamcorderProfile.QUALITY_720P,
96             CamcorderProfile.QUALITY_480P,
97             CamcorderProfile.QUALITY_CIF,
98             CamcorderProfile.QUALITY_QCIF,
99             CamcorderProfile.QUALITY_QVGA,
100             CamcorderProfile.QUALITY_LOW,
101     };
102     private static final int MAX_VIDEO_SNAPSHOT_IMAGES = 5;
103     private static final int BURST_VIDEO_SNAPSHOT_NUM = 3;
104     private static final int SLOWMO_SLOW_FACTOR = 4;
105     private static final int MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED = 4;
106     private List<Size> mSupportedVideoSizes;
107     private Surface mRecordingSurface;
108     private Surface mPersistentSurface;
109     private MediaRecorder mMediaRecorder;
110     private String mOutMediaFileName;
111     private int mVideoFrameRate;
112     private Size mVideoSize;
113     private long mRecordingStartTime;
114 
115     private Surface mIntermediateSurface;
116     private ImageReader mIntermediateReader;
117     private ImageWriter mIntermediateWriter;
118     private ImageWriterQueuer mQueuer;
119     private HandlerThread mIntermediateThread;
120     private Handler mIntermediateHandler;
121 
122     @Override
setUp()123     public void setUp() throws Exception {
124         super.setUp();
125     }
126 
127     @Override
tearDown()128     public void tearDown() throws Exception {
129         super.tearDown();
130     }
131 
doBasicRecording(boolean useVideoStab)132     private void doBasicRecording(boolean useVideoStab) throws Exception {
133         doBasicRecording(useVideoStab, false);
134     }
135 
doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface)136     private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface)
137             throws Exception {
138         doBasicRecording(useVideoStab, useIntermediateSurface, false);
139     }
140 
doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)141     private void doBasicRecording(boolean useVideoStab, boolean useIntermediateSurface,
142             boolean useEncoderProfiles) throws Exception {
143         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
144             try {
145                 Log.i(TAG, "Testing basic recording for camera " + mCameraIdsUnderTest[i]);
146                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
147                 if (!staticInfo.isColorOutputSupported()) {
148                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
149                             " does not support color outputs, skipping");
150                     continue;
151                 }
152 
153                 // External camera doesn't support CamcorderProfile recording
154                 if (staticInfo.isExternalCamera()) {
155                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
156                             " does not support CamcorderProfile, skipping");
157                     continue;
158                 }
159 
160                 if (!staticInfo.isVideoStabilizationSupported() && useVideoStab) {
161                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
162                             " does not support video stabilization, skipping the stabilization"
163                             + " test");
164                     continue;
165                 }
166 
167                 // Re-use the MediaRecorder object for the same camera device.
168                 mMediaRecorder = new MediaRecorder();
169                 openDevice(mCameraIdsUnderTest[i]);
170                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
171 
172                 basicRecordingTestByCamera(mCamcorderProfileList, useVideoStab,
173                         useIntermediateSurface, useEncoderProfiles);
174             } finally {
175                 closeDevice();
176                 releaseRecorder();
177             }
178         }
179     }
180 
181     /**
182      * <p>
183      * Test basic video stabilitzation camera recording.
184      * </p>
185      * <p>
186      * This test covers the typical basic use case of camera recording with video
187      * stabilization is enabled, if video stabilization is supported.
188      * MediaRecorder is used to record the audio and video, CamcorderProfile is
189      * used to configure the MediaRecorder. It goes through the pre-defined
190      * CamcorderProfile list, test each profile configuration and validate the
191      * recorded video. Preview is set to the video size.
192      * </p>
193      */
194     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicVideoStabilizationRecording()195     public void testBasicVideoStabilizationRecording() throws Exception {
196         doBasicRecording(/*useVideoStab*/true);
197     }
198 
199     /**
200      * <p>
201      * Test basic camera recording.
202      * </p>
203      * <p>
204      * This test covers the typical basic use case of camera recording.
205      * MediaRecorder is used to record the audio and video, CamcorderProfile is
206      * used to configure the MediaRecorder. It goes through the pre-defined
207      * CamcorderProfile list, test each profile configuration and validate the
208      * recorded video. Preview is set to the video size.
209      * </p>
210      */
211     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicRecording()212     public void testBasicRecording() throws Exception {
213         doBasicRecording(/*useVideoStab*/false);
214     }
215 
216     /**
217      * <p>
218      * Test camera recording with intermediate surface.
219      * </p>
220      * <p>
221      * This test is similar to testBasicRecording with a tweak where an intermediate
222      * surface is setup between camera and MediaRecorder, giving application a chance
223      * to decide whether to send a frame to recorder or not.
224      * </p>
225      */
226     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testIntermediateSurfaceRecording()227     public void testIntermediateSurfaceRecording() throws Exception {
228         doBasicRecording(/*useVideoStab*/false, /*useIntermediateSurface*/true);
229     }
230 
231     /**
232      * <p>
233      * Test basic camera recording using encoder profiles.
234      * </p>
235      * <p>
236      * This test covers the typical basic use case of camera recording.
237      * MediaRecorder is used to record the audio and video,
238      * EncoderProfiles are used to configure the MediaRecorder. It
239      * goes through the pre-defined CamcorderProfile list, test each
240      * encoder profile combination and validate the recorded video.
241      * Preview is set to the video size.
242      * </p>
243      */
244     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBasicEncoderProfilesRecording()245     public void testBasicEncoderProfilesRecording() throws Exception {
246         doBasicRecording(/*useVideoStab*/false,  /*useIntermediateSurface*/false,
247                 /*useEncoderProfiles*/true);
248     }
249 
250     /**
251      * <p>
252      * Test basic camera recording from a persistent input surface.
253      * </p>
254      * <p>
255      * This test is similar to testBasicRecording except that MediaRecorder records
256      * from a persistent input surface that's used across multiple recording sessions.
257      * </p>
258      */
259     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testRecordingFromPersistentSurface()260     public void testRecordingFromPersistentSurface() throws Exception {
261         if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) {
262             return; // skipped
263         }
264         mPersistentSurface = MediaCodec.createPersistentInputSurface();
265         assertNotNull("Failed to create persistent input surface!", mPersistentSurface);
266 
267         try {
268             doBasicRecording(/*useVideoStab*/false);
269         } finally {
270             mPersistentSurface.release();
271             mPersistentSurface = null;
272         }
273     }
274 
275     /**
276      * <p>
277      * Test camera recording for all supported sizes by using MediaRecorder.
278      * </p>
279      * <p>
280      * This test covers camera recording for all supported sizes by camera. MediaRecorder
281      * is used to encode the video. Preview is set to the video size. Recorded videos are
282      * validated according to the recording configuration.
283      * </p>
284      */
285     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testSupportedVideoSizes()286     public void testSupportedVideoSizes() throws Exception {
287         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
288             try {
289                 Log.i(TAG, "Testing supported video size recording for camera " + mCameraIdsUnderTest[i]);
290                 if (!mAllStaticInfo.get(mCameraIdsUnderTest[i]).isColorOutputSupported()) {
291                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
292                             " does not support color outputs, skipping");
293                     continue;
294                 }
295                 // Re-use the MediaRecorder object for the same camera device.
296                 mMediaRecorder = new MediaRecorder();
297                 openDevice(mCameraIdsUnderTest[i]);
298 
299                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
300 
301                 recordingSizeTestByCamera();
302             } finally {
303                 closeDevice();
304                 releaseRecorder();
305             }
306         }
307     }
308 
309     /**
310      * Test different start/stop orders of Camera and Recorder.
311      *
312      * <p>The recording should be working fine for any kind of start/stop orders.</p>
313      */
314     @Test
testCameraRecorderOrdering()315     public void testCameraRecorderOrdering() {
316         // TODO: need implement
317     }
318 
319     /**
320      * <p>
321      * Test camera recording for all supported sizes by using MediaCodec.
322      * </p>
323      * <p>
324      * This test covers video only recording for all supported sizes (camera and
325      * encoder). MediaCodec is used to encode the video. The recorded videos are
326      * validated according to the recording configuration.
327      * </p>
328      */
329     @Test
testMediaCodecRecording()330     public void testMediaCodecRecording() throws Exception {
331         // TODO. Need implement.
332     }
333 
334     /**
335      * <p>
336      * Test video snapshot for each camera.
337      * </p>
338      * <p>
339      * This test covers video snapshot typical use case. The MediaRecorder is used to record the
340      * video for each available video size. The largest still capture size is selected to
341      * capture the JPEG image. The still capture images are validated according to the capture
342      * configuration. The timestamp of capture result before and after video snapshot is also
343      * checked to make sure no frame drop caused by video snapshot.
344      * </p>
345      */
346     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testVideoSnapshot()347     public void testVideoSnapshot() throws Exception {
348         videoSnapshotHelper(/*burstTest*/false);
349     }
350 
351     /**
352      * <p>
353      * Test burst video snapshot for each camera.
354      * </p>
355      * <p>
356      * This test covers burst video snapshot capture. The MediaRecorder is used to record the
357      * video for each available video size. The largest still capture size is selected to
358      * capture the JPEG image. {@value #BURST_VIDEO_SNAPSHOT_NUM} video snapshot requests will be
359      * sent during the test. The still capture images are validated according to the capture
360      * configuration.
361      * </p>
362      */
363     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testBurstVideoSnapshot()364     public void testBurstVideoSnapshot() throws Exception {
365         videoSnapshotHelper(/*burstTest*/true);
366     }
367 
368     /**
369      * Test timelapse recording, where capture rate is slower than video (playback) frame rate.
370      */
371     @Test
testTimelapseRecording()372     public void testTimelapseRecording() throws Exception {
373         // TODO. Need implement.
374     }
375 
376     @Test
testSlowMotionRecording()377     public void testSlowMotionRecording() throws Exception {
378         slowMotionRecording();
379     }
380 
381     @Test(timeout=60*60*1000) // timeout = 60 mins for long running tests
testConstrainedHighSpeedRecording()382     public void testConstrainedHighSpeedRecording() throws Exception {
383         constrainedHighSpeedRecording();
384     }
385 
386     @Test
testAbandonedHighSpeedRequest()387     public void testAbandonedHighSpeedRequest() throws Exception {
388         for (String id : mCameraIdsUnderTest) {
389             try {
390                 Log.i(TAG, "Testing bad suface for createHighSpeedRequestList for camera " + id);
391                 StaticMetadata staticInfo = mAllStaticInfo.get(id);
392                 if (!staticInfo.isColorOutputSupported()) {
393                     Log.i(TAG, "Camera " + id +
394                             " does not support color outputs, skipping");
395                     continue;
396                 }
397                 if (!staticInfo.isConstrainedHighSpeedVideoSupported()) {
398                     Log.i(TAG, "Camera " + id +
399                             " does not support constrained high speed video, skipping");
400                     continue;
401                 }
402 
403                 // Re-use the MediaRecorder object for the same camera device.
404                 mMediaRecorder = new MediaRecorder();
405                 openDevice(id);
406 
407                 StreamConfigurationMap config =
408                         mStaticInfo.getValueFromKeyNonNull(
409                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
410                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
411                 Size size = highSpeedVideoSizes[0];
412                 Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size);
413                 mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " +
414                         "size " + size, fpsRange);
415                 if (fpsRange == null) {
416                     continue;
417                 }
418 
419                 int captureRate = fpsRange.getLower();
420                 int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR;
421                 // Skip the test if the highest recording FPS supported by CamcorderProfile
422                 if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
423                     Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
424                             + " is not supported by CamcorderProfile");
425                     continue;
426                 }
427 
428                 mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
429                 prepareRecording(size, videoFramerate, captureRate);
430                 updatePreviewSurfaceWithVideo(size, captureRate);
431 
432                 List<Surface> outputSurfaces = new ArrayList<Surface>(2);
433                 assertTrue("Both preview and recording surfaces should be valid",
434                         mPreviewSurface.isValid() && mRecordingSurface.isValid());
435 
436                 outputSurfaces.add(mPreviewSurface);
437                 outputSurfaces.add(mRecordingSurface);
438 
439                 mSessionListener = new BlockingSessionCallback();
440                 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true,
441                         mSessionListener, mHandler);
442 
443                 CaptureRequest.Builder requestBuilder =
444                         mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
445                 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
446                 requestBuilder.addTarget(mPreviewSurface);
447                 requestBuilder.addTarget(mRecordingSurface);
448 
449                 // 1. Test abandoned MediaRecorder
450                 releaseRecorder();
451                 try {
452                     List<CaptureRequest> slowMoRequests =
453                             ((CameraConstrainedHighSpeedCaptureSession) mSession).
454                             createHighSpeedRequestList(requestBuilder.build());
455                     fail("Create high speed request on abandoned surface must fail!");
456                 } catch (IllegalArgumentException e) {
457                     Log.i(TAG, "Release recording surface test passed");
458                     // expected
459                 }
460 
461                 // 2. Test abandoned preview surface
462                 mMediaRecorder = new MediaRecorder();
463                 SurfaceTexture preview = new SurfaceTexture(/*random int*/ 1);
464                 Surface previewSurface = new Surface(preview);
465                 preview.setDefaultBufferSize(size.getWidth(), size.getHeight());
466 
467                 outputSurfaces = new ArrayList<Surface>();
468                 outputSurfaces.add(previewSurface);
469 
470                 prepareRecording(size, videoFramerate, captureRate);
471                 updatePreviewSurfaceWithVideo(size, captureRate);
472 
473                 mSession = configureCameraSession(mCamera, outputSurfaces, /*highSpeed*/true,
474                         mSessionListener, mHandler);
475 
476                 requestBuilder = mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
477                 requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
478                 requestBuilder.addTarget(previewSurface);
479 
480                 // Abandon preview surface.
481                 previewSurface.release();
482 
483                 try {
484                     List<CaptureRequest> slowMoRequests =
485                             ((CameraConstrainedHighSpeedCaptureSession) mSession).
486                             createHighSpeedRequestList(requestBuilder.build());
487                     fail("Create high speed request on abandoned preview surface must fail!");
488                 } catch (IllegalArgumentException e) {
489                     Log.i(TAG, "Release preview surface test passed");
490                     // expected
491                 }
492             } finally {
493                 closeDevice();
494                 releaseRecorder();
495             }
496         }
497     }
498 
499     /**
500      * <p>
501      * Test recording framerate accuracy when switching from low FPS to high FPS.
502      * </p>
503      * <p>
504      * This test first record a video with profile of lowest framerate then record a video with
505      * profile of highest framerate. Make sure that the video framerate are still accurate.
506      * </p>
507      */
508     @Test
testRecordingFramerateLowToHigh()509     public void testRecordingFramerateLowToHigh() throws Exception {
510         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
511             try {
512                 Log.i(TAG, "Testing recording framerate low to high for camera " + mCameraIdsUnderTest[i]);
513                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
514                 if (!staticInfo.isColorOutputSupported()) {
515                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
516                             " does not support color outputs, skipping");
517                     continue;
518                 }
519                 if (staticInfo.isExternalCamera()) {
520                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
521                             " does not support CamcorderProfile, skipping");
522                     continue;
523                 }
524                 // Re-use the MediaRecorder object for the same camera device.
525                 mMediaRecorder = new MediaRecorder();
526                 openDevice(mCameraIdsUnderTest[i]);
527 
528                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
529 
530                 int minFpsProfileId = -1, minFps = 1000;
531                 int maxFpsProfileId = -1, maxFps = 0;
532                 int cameraId = Integer.valueOf(mCamera.getId());
533 
534                 for (int profileId : mCamcorderProfileList) {
535                     if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
536                         continue;
537                     }
538                     CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
539                     if (profile.videoFrameRate < minFps) {
540                         minFpsProfileId = profileId;
541                         minFps = profile.videoFrameRate;
542                     }
543                     if (profile.videoFrameRate > maxFps) {
544                         maxFpsProfileId = profileId;
545                         maxFps = profile.videoFrameRate;
546                     }
547                 }
548 
549                 int camcorderProfileList[] = new int[] {minFpsProfileId, maxFpsProfileId};
550                 basicRecordingTestByCamera(camcorderProfileList, /*useVideoStab*/false);
551             } finally {
552                 closeDevice();
553                 releaseRecorder();
554             }
555         }
556     }
557 
558     /**
559      * <p>
560      * Test preview and video surfaces sharing the same camera stream.
561      * </p>
562      */
563     @Test
testVideoPreviewSurfaceSharing()564     public void testVideoPreviewSurfaceSharing() throws Exception {
565         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
566             try {
567                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
568                 if (staticInfo.isHardwareLevelLegacy()) {
569                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] + " is legacy, skipping");
570                     continue;
571                 }
572                 if (!staticInfo.isColorOutputSupported()) {
573                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
574                             " does not support color outputs, skipping");
575                     continue;
576                 }
577                 // Re-use the MediaRecorder object for the same camera device.
578                 mMediaRecorder = new MediaRecorder();
579                 openDevice(mCameraIdsUnderTest[i]);
580 
581                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
582 
583                 videoPreviewSurfaceSharingTestByCamera();
584             } finally {
585                 closeDevice();
586                 releaseRecorder();
587             }
588         }
589     }
590 
591     /**
592      * <p>
593      * Test recording with same recording surface and different preview surfaces.
594      * </p>
595      * <p>
596      * This test maintains persistent video surface while changing preview surface.
597      * This exercises format/dataspace override behavior of the camera device.
598      * </p>
599      */
600     @Test
testRecordingWithDifferentPreviewSizes()601     public void testRecordingWithDifferentPreviewSizes() throws Exception {
602         if (!MediaUtils.checkCodecForDomain(true /* encoder */, "video")) {
603             return; // skipped
604         }
605         mPersistentSurface = MediaCodec.createPersistentInputSurface();
606         assertNotNull("Failed to create persistent input surface!", mPersistentSurface);
607 
608         try {
609             doRecordingWithDifferentPreviewSizes();
610         } finally {
611             mPersistentSurface.release();
612             mPersistentSurface = null;
613         }
614     }
615 
doRecordingWithDifferentPreviewSizes()616     public void doRecordingWithDifferentPreviewSizes() throws Exception {
617         for (int i = 0; i < mCameraIdsUnderTest.length; i++) {
618             try {
619                 Log.i(TAG, "Testing recording with different preview sizes for camera " +
620                         mCameraIdsUnderTest[i]);
621                 StaticMetadata staticInfo = mAllStaticInfo.get(mCameraIdsUnderTest[i]);
622                 if (!staticInfo.isColorOutputSupported()) {
623                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
624                             " does not support color outputs, skipping");
625                     continue;
626                 }
627                 if (staticInfo.isExternalCamera()) {
628                     Log.i(TAG, "Camera " + mCameraIdsUnderTest[i] +
629                             " does not support CamcorderProfile, skipping");
630                     continue;
631                 }
632                 // Re-use the MediaRecorder object for the same camera device.
633                 mMediaRecorder = new MediaRecorder();
634                 openDevice(mCameraIdsUnderTest[i]);
635 
636                 initSupportedVideoSize(mCameraIdsUnderTest[i]);
637 
638                 Size maxPreviewSize = mOrderedPreviewSizes.get(0);
639                 List<Range<Integer> > fpsRanges = Arrays.asList(
640                         mStaticInfo.getAeAvailableTargetFpsRangesChecked());
641                 int cameraId = Integer.valueOf(mCamera.getId());
642                 int maxVideoFrameRate = -1;
643                 for (int profileId : mCamcorderProfileList) {
644                     if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
645                         continue;
646                     }
647                     CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
648 
649                     Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
650                     Range<Integer> fpsRange = new Range(
651                             profile.videoFrameRate, profile.videoFrameRate);
652                     if (maxVideoFrameRate < profile.videoFrameRate) {
653                         maxVideoFrameRate = profile.videoFrameRate;
654                     }
655 
656                     if (allowedUnsupported(cameraId, profileId)) {
657                         continue;
658                     }
659 
660                     if (mStaticInfo.isHardwareLevelLegacy() &&
661                             (videoSz.getWidth() > maxPreviewSize.getWidth() ||
662                              videoSz.getHeight() > maxPreviewSize.getHeight())) {
663                         // Skip. Legacy mode can only do recording up to max preview size
664                         continue;
665                     }
666                     assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId +
667                                     " must be one of the camera device supported video size!",
668                                     mSupportedVideoSizes.contains(videoSz));
669                     assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId +
670                             ") must be one of the camera device available FPS range!",
671                             fpsRanges.contains(fpsRange));
672 
673                     // Configure preview and recording surfaces.
674                     mOutMediaFileName = mDebugFileNameBase + "/test_video_surface_reconfig.mp4";
675 
676                     // prepare preview surface by using video size.
677                     List<Size> previewSizes = getPreviewSizesForVideo(videoSz,
678                             profile.videoFrameRate);
679                     if (previewSizes.size() <= 1) {
680                         continue;
681                     }
682 
683                     // 1. Do video recording using largest compatbile preview sizes
684                     prepareRecordingWithProfile(profile);
685                     updatePreviewSurface(previewSizes.get(0));
686                     SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
687                     startRecording(
688                             /* useMediaRecorder */true, resultListener,
689                             /*useVideoStab*/false, fpsRange, false);
690                     SystemClock.sleep(RECORDING_DURATION_MS);
691                     stopRecording(/* useMediaRecorder */true, /* useIntermediateSurface */false,
692                             /* stopStreaming */false);
693 
694                     // 2. Reconfigure with the same recording surface, but switch to a smaller
695                     // preview size.
696                     prepareRecordingWithProfile(profile);
697                     updatePreviewSurface(previewSizes.get(1));
698                     SimpleCaptureCallback resultListener2 = new SimpleCaptureCallback();
699                     startRecording(
700                             /* useMediaRecorder */true, resultListener2,
701                             /*useVideoStab*/false, fpsRange, false);
702                     SystemClock.sleep(RECORDING_DURATION_MS);
703                     stopRecording(/* useMediaRecorder */true);
704                     break;
705                 }
706             } finally {
707                 closeDevice();
708                 releaseRecorder();
709             }
710         }
711     }
712 
713     /**
714      * Test camera preview and video surface sharing for maximum supported size.
715      */
videoPreviewSurfaceSharingTestByCamera()716     private void videoPreviewSurfaceSharingTestByCamera() throws Exception {
717         for (Size sz : mOrderedPreviewSizes) {
718             if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) {
719                 continue;
720             }
721 
722             if (VERBOSE) {
723                 Log.v(TAG, "Testing camera recording with video size " + sz.toString());
724             }
725 
726             // Configure preview and recording surfaces.
727             mOutMediaFileName = mDebugFileNameBase + "/test_video_share.mp4";
728             if (DEBUG_DUMP) {
729                 mOutMediaFileName = mDebugFileNameBase + "/test_video_share_" + mCamera.getId() +
730                     "_" + sz.toString() + ".mp4";
731             }
732 
733             // Allow external camera to use variable fps range
734             Range<Integer> fpsRange = null;
735             if (mStaticInfo.isExternalCamera()) {
736                 Range<Integer>[] availableFpsRange =
737                         mStaticInfo.getAeAvailableTargetFpsRangesChecked();
738 
739                 boolean foundRange = false;
740                 int minFps = 0;
741                 for (int i = 0; i < availableFpsRange.length; i += 1) {
742                     if (minFps < availableFpsRange[i].getLower()
743                             && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) {
744                         minFps = availableFpsRange[i].getLower();
745                         foundRange = true;
746                     }
747                 }
748                 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange);
749                 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE);
750             }
751 
752             // Use AVC and AAC a/v compression format.
753             prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE);
754 
755             // prepare preview surface by using video size.
756             updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE);
757 
758             // Start recording
759             SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
760             if (!startSharedRecording(/* useMediaRecorder */true, resultListener,
761                     /*useVideoStab*/false, fpsRange)) {
762                 mMediaRecorder.reset();
763                 continue;
764             }
765 
766             // Record certain duration.
767             SystemClock.sleep(RECORDING_DURATION_MS);
768 
769             // Stop recording and preview
770             stopRecording(/* useMediaRecorder */true);
771             // Convert number of frames camera produced into the duration in unit of ms.
772             float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE;
773             float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs;
774             float durationMaxMs = durationMinMs;
775             float frameDurationMaxMs = 0.f;
776             if (fpsRange != null) {
777                 frameDurationMaxMs = 1000.0f / fpsRange.getLower();
778                 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs;
779             }
780 
781             // Validation.
782             validateRecording(sz, durationMinMs, durationMaxMs,
783                     frameDurationMinMs, frameDurationMaxMs,
784                     FRMDRP_RATE_TOLERANCE);
785 
786             break;
787         }
788     }
789 
790     /**
791      * Test slow motion recording where capture rate (camera output) is different with
792      * video (playback) frame rate for each camera if high speed recording is supported
793      * by both camera and encoder.
794      *
795      * <p>
796      * Normal recording use cases make the capture rate (camera output frame
797      * rate) the same as the video (playback) frame rate. This guarantees that
798      * the motions in the scene play at the normal speed. If the capture rate is
799      * faster than video frame rate, for a given time duration, more number of
800      * frames are captured than it can be played in the same time duration. This
801      * generates "slow motion" effect during playback.
802      * </p>
803      */
slowMotionRecording()804     private void slowMotionRecording() throws Exception {
805         for (String id : mCameraIdsUnderTest) {
806             try {
807                 Log.i(TAG, "Testing slow motion recording for camera " + id);
808                 StaticMetadata staticInfo = mAllStaticInfo.get(id);
809                 if (!staticInfo.isColorOutputSupported()) {
810                     Log.i(TAG, "Camera " + id +
811                             " does not support color outputs, skipping");
812                     continue;
813                 }
814                 if (!staticInfo.isHighSpeedVideoSupported()) {
815                     continue;
816                 }
817 
818                 // Re-use the MediaRecorder object for the same camera device.
819                 mMediaRecorder = new MediaRecorder();
820                 openDevice(id);
821 
822                 StreamConfigurationMap config =
823                         mStaticInfo.getValueFromKeyNonNull(
824                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
825                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
826                 for (Size size : highSpeedVideoSizes) {
827                     Range<Integer> fpsRange = getHighestHighSpeedFixedFpsRangeForSize(config, size);
828                     mCollector.expectNotNull("Unable to find the fixed frame rate fps range for " +
829                             "size " + size, fpsRange);
830                     if (fpsRange == null) {
831                         continue;
832                     }
833 
834                     int captureRate = fpsRange.getLower();
835                     int videoFramerate = captureRate / SLOWMO_SLOW_FACTOR;
836                     // Skip the test if the highest recording FPS supported by CamcorderProfile
837                     if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
838                         Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
839                                 + " is not supported by CamcorderProfile");
840                         continue;
841                     }
842 
843                     mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video.mp4";
844                     if (DEBUG_DUMP) {
845                         mOutMediaFileName = mDebugFileNameBase + "/test_slowMo_video_" + id + "_"
846                                 + size.toString() + ".mp4";
847                     }
848 
849                     prepareRecording(size, videoFramerate, captureRate);
850 
851                     // prepare preview surface by using video size.
852                     updatePreviewSurfaceWithVideo(size, captureRate);
853 
854                     // Start recording
855                     SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
856                     startSlowMotionRecording(/*useMediaRecorder*/true, videoFramerate, captureRate,
857                             fpsRange, resultListener, /*useHighSpeedSession*/false);
858 
859                     // Record certain duration.
860                     SystemClock.sleep(RECORDING_DURATION_MS);
861 
862                     // Stop recording and preview
863                     stopRecording(/*useMediaRecorder*/true);
864                     // Convert number of frames camera produced into the duration in unit of ms.
865                     float frameDurationMs = 1000.0f / videoFramerate;
866                     float durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
867 
868                     // Validation.
869                     validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
870                 }
871 
872             } finally {
873                 closeDevice();
874                 releaseRecorder();
875             }
876         }
877     }
878 
constrainedHighSpeedRecording()879     private void constrainedHighSpeedRecording() throws Exception {
880         for (String id : mCameraIdsUnderTest) {
881             try {
882                 Log.i(TAG, "Testing constrained high speed recording for camera " + id);
883 
884                 if (!mAllStaticInfo.get(id).isConstrainedHighSpeedVideoSupported()) {
885                     Log.i(TAG, "Camera " + id + " doesn't support high speed recording, skipping.");
886                     continue;
887                 }
888 
889                 // Re-use the MediaRecorder object for the same camera device.
890                 mMediaRecorder = new MediaRecorder();
891                 openDevice(id);
892 
893                 StreamConfigurationMap config =
894                         mStaticInfo.getValueFromKeyNonNull(
895                                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
896                 Size[] highSpeedVideoSizes = config.getHighSpeedVideoSizes();
897                 for (Size size : highSpeedVideoSizes) {
898                     List<Range<Integer>> fixedFpsRanges =
899                             getHighSpeedFixedFpsRangeForSize(config, size);
900                     mCollector.expectTrue("Unable to find the fixed frame rate fps range for " +
901                             "size " + size, fixedFpsRanges.size() > 0);
902                     // Test recording for each FPS range
903                     for (Range<Integer> fpsRange : fixedFpsRanges) {
904                         int captureRate = fpsRange.getLower();
905                         final int VIDEO_FRAME_RATE = 30;
906                         // Skip the test if the highest recording FPS supported by CamcorderProfile
907                         if (fpsRange.getUpper() > getFpsFromHighSpeedProfileForSize(size)) {
908                             Log.w(TAG, "high speed recording " + size + "@" + captureRate + "fps"
909                                     + " is not supported by CamcorderProfile");
910                             continue;
911                         }
912 
913                         SimpleCaptureCallback previewResultListener = new SimpleCaptureCallback();
914 
915                         // prepare preview surface by using video size.
916                         updatePreviewSurfaceWithVideo(size, captureRate);
917 
918                         startConstrainedPreview(fpsRange, previewResultListener);
919 
920                         mOutMediaFileName = mDebugFileNameBase + "/test_cslowMo_video_" +
921                             captureRate + "fps_" + id + "_" + size.toString() + ".mp4";
922 
923                         prepareRecording(size, VIDEO_FRAME_RATE, captureRate);
924 
925                         SystemClock.sleep(PREVIEW_DURATION_MS);
926 
927                         stopCameraStreaming();
928 
929                         SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
930                         // Start recording
931                         startSlowMotionRecording(/*useMediaRecorder*/true, VIDEO_FRAME_RATE,
932                                 captureRate, fpsRange, resultListener,
933                                 /*useHighSpeedSession*/true);
934 
935                         // Record certain duration.
936                         SystemClock.sleep(RECORDING_DURATION_MS);
937 
938                         // Stop recording and preview
939                         stopRecording(/*useMediaRecorder*/true);
940 
941                         startConstrainedPreview(fpsRange, previewResultListener);
942 
943                         // Convert number of frames camera produced into the duration in unit of ms.
944                         float frameDurationMs = 1000.0f / VIDEO_FRAME_RATE;
945                         float durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
946 
947                         // Validation.
948                         validateRecording(size, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
949 
950                         SystemClock.sleep(PREVIEW_DURATION_MS);
951 
952                         stopCameraStreaming();
953                     }
954                 }
955 
956             } finally {
957                 closeDevice();
958                 releaseRecorder();
959             }
960         }
961     }
962 
963     /**
964      * Get high speed FPS from CamcorderProfiles for a given size.
965      *
966      * @param size The size used to search the CamcorderProfiles for the FPS.
967      * @return high speed video FPS, 0 if the given size is not supported by the CamcorderProfiles.
968      */
getFpsFromHighSpeedProfileForSize(Size size)969     private int getFpsFromHighSpeedProfileForSize(Size size) {
970         for (int quality = CamcorderProfile.QUALITY_HIGH_SPEED_480P;
971                 quality <= CamcorderProfile.QUALITY_HIGH_SPEED_2160P; quality++) {
972             if (CamcorderProfile.hasProfile(quality)) {
973                 CamcorderProfile profile = CamcorderProfile.get(quality);
974                 if (size.equals(new Size(profile.videoFrameWidth, profile.videoFrameHeight))){
975                     return profile.videoFrameRate;
976                 }
977             }
978         }
979 
980         return 0;
981     }
982 
getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)983     private Range<Integer> getHighestHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config,
984             Size size) {
985         Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size);
986         Range<Integer> maxRange = availableFpsRanges[0];
987         boolean foundRange = false;
988         for (Range<Integer> range : availableFpsRanges) {
989             if (range.getLower().equals(range.getUpper()) && range.getLower() >= maxRange.getLower()) {
990                 foundRange = true;
991                 maxRange = range;
992             }
993         }
994 
995         if (!foundRange) {
996             return null;
997         }
998         return maxRange;
999     }
1000 
getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config, Size size)1001     private List<Range<Integer>> getHighSpeedFixedFpsRangeForSize(StreamConfigurationMap config,
1002             Size size) {
1003         Range<Integer>[] availableFpsRanges = config.getHighSpeedVideoFpsRangesFor(size);
1004         List<Range<Integer>> fixedRanges = new ArrayList<Range<Integer>>();
1005         for (Range<Integer> range : availableFpsRanges) {
1006             if (range.getLower().equals(range.getUpper())) {
1007                 fixedRanges.add(range);
1008             }
1009         }
1010         return fixedRanges;
1011     }
1012 
startConstrainedPreview(Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener)1013     private void startConstrainedPreview(Range<Integer> fpsRange,
1014             CameraCaptureSession.CaptureCallback listener) throws Exception {
1015         List<Surface> outputSurfaces = new ArrayList<Surface>(1);
1016         assertTrue("Preview surface should be valid", mPreviewSurface.isValid());
1017         outputSurfaces.add(mPreviewSurface);
1018         mSessionListener = new BlockingSessionCallback();
1019 
1020         List<CaptureRequest> slowMoRequests = null;
1021         CaptureRequest.Builder requestBuilder =
1022             mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1023         requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1024         requestBuilder.addTarget(mPreviewSurface);
1025         CaptureRequest initialRequest = requestBuilder.build();
1026         CameraTestUtils.checkSessionConfigurationWithSurfaces(mCamera, mHandler,
1027                 outputSurfaces, /*inputConfig*/ null, SessionConfiguration.SESSION_HIGH_SPEED,
1028                 /*defaultSupport*/ true, "Constrained session configuration query failed");
1029         mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener,
1030                 mHandler, initialRequest);
1031         slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession).
1032             createHighSpeedRequestList(initialRequest);
1033 
1034         mSession.setRepeatingBurst(slowMoRequests, listener, mHandler);
1035     }
1036 
startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate, int captureRate, Range<Integer> fpsRange, CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession)1037     private void startSlowMotionRecording(boolean useMediaRecorder, int videoFrameRate,
1038             int captureRate, Range<Integer> fpsRange,
1039             CameraCaptureSession.CaptureCallback listener, boolean useHighSpeedSession)
1040             throws Exception {
1041         List<Surface> outputSurfaces = new ArrayList<Surface>(2);
1042         assertTrue("Both preview and recording surfaces should be valid",
1043                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1044         outputSurfaces.add(mPreviewSurface);
1045         outputSurfaces.add(mRecordingSurface);
1046         // Video snapshot surface
1047         if (mReaderSurface != null) {
1048             outputSurfaces.add(mReaderSurface);
1049         }
1050         mSessionListener = new BlockingSessionCallback();
1051 
1052         // Create slow motion request list
1053         List<CaptureRequest> slowMoRequests = null;
1054         if (useHighSpeedSession) {
1055             CaptureRequest.Builder requestBuilder =
1056                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1057             requestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1058             requestBuilder.addTarget(mPreviewSurface);
1059             requestBuilder.addTarget(mRecordingSurface);
1060             CaptureRequest initialRequest = requestBuilder.build();
1061             mSession = buildConstrainedCameraSession(mCamera, outputSurfaces, mSessionListener,
1062                     mHandler, initialRequest);
1063             slowMoRequests = ((CameraConstrainedHighSpeedCaptureSession) mSession).
1064                     createHighSpeedRequestList(initialRequest);
1065         } else {
1066             CaptureRequest.Builder recordingRequestBuilder =
1067                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1068             recordingRequestBuilder.set(CaptureRequest.CONTROL_MODE,
1069                     CaptureRequest.CONTROL_MODE_USE_SCENE_MODE);
1070             recordingRequestBuilder.set(CaptureRequest.CONTROL_SCENE_MODE,
1071                     CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO);
1072 
1073             CaptureRequest.Builder recordingOnlyBuilder =
1074                     mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1075             recordingOnlyBuilder.set(CaptureRequest.CONTROL_MODE,
1076                     CaptureRequest.CONTROL_MODE_USE_SCENE_MODE);
1077             recordingOnlyBuilder.set(CaptureRequest.CONTROL_SCENE_MODE,
1078                     CaptureRequest.CONTROL_SCENE_MODE_HIGH_SPEED_VIDEO);
1079             int slowMotionFactor = captureRate / videoFrameRate;
1080 
1081             // Make sure camera output frame rate is set to correct value.
1082             recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1083             recordingRequestBuilder.addTarget(mRecordingSurface);
1084             recordingRequestBuilder.addTarget(mPreviewSurface);
1085             recordingOnlyBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1086             recordingOnlyBuilder.addTarget(mRecordingSurface);
1087 
1088             CaptureRequest initialRequest = recordingRequestBuilder.build();
1089             mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces,
1090                     mSessionListener, mHandler, initialRequest);
1091 
1092             slowMoRequests = new ArrayList<CaptureRequest>();
1093             slowMoRequests.add(initialRequest);// Preview + recording.
1094 
1095             for (int i = 0; i < slowMotionFactor - 1; i++) {
1096                 slowMoRequests.add(recordingOnlyBuilder.build()); // Recording only.
1097             }
1098         }
1099 
1100         mSession.setRepeatingBurst(slowMoRequests, listener, mHandler);
1101 
1102         if (useMediaRecorder) {
1103             mMediaRecorder.start();
1104         } else {
1105             // TODO: need implement MediaCodec path.
1106         }
1107 
1108     }
1109 
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)1110     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab)
1111             throws Exception {
1112         basicRecordingTestByCamera(camcorderProfileList, useVideoStab, false);
1113     }
1114 
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface)1115     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab,
1116             boolean useIntermediateSurface) throws Exception {
1117         basicRecordingTestByCamera(camcorderProfileList, useVideoStab,
1118                 useIntermediateSurface, false);
1119     }
1120 
1121     /**
1122      * Test camera recording by using each available CamcorderProfile for a
1123      * given camera. preview size is set to the video size.
1124      */
basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab, boolean useIntermediateSurface, boolean useEncoderProfiles)1125     private void basicRecordingTestByCamera(int[] camcorderProfileList, boolean useVideoStab,
1126             boolean useIntermediateSurface, boolean useEncoderProfiles) throws Exception {
1127         Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1128         List<Range<Integer> > fpsRanges = Arrays.asList(
1129                 mStaticInfo.getAeAvailableTargetFpsRangesChecked());
1130         int cameraId = Integer.valueOf(mCamera.getId());
1131         int maxVideoFrameRate = -1;
1132 
1133         // only validate recording for non-perf measurement runs
1134         boolean validateRecording = !isPerfMeasure();
1135         for (int profileId : camcorderProfileList) {
1136             if (!CamcorderProfile.hasProfile(cameraId, profileId)) {
1137                 continue;
1138             }
1139 
1140             CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
1141             Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1142 
1143             Range<Integer> fpsRange = new Range(profile.videoFrameRate, profile.videoFrameRate);
1144             if (maxVideoFrameRate < profile.videoFrameRate) {
1145                 maxVideoFrameRate = profile.videoFrameRate;
1146             }
1147 
1148             if (allowedUnsupported(cameraId, profileId)) {
1149                 continue;
1150             }
1151 
1152             if (mStaticInfo.isHardwareLevelLegacy() &&
1153                     (videoSz.getWidth() > maxPreviewSize.getWidth() ||
1154                      videoSz.getHeight() > maxPreviewSize.getHeight())) {
1155                 // Skip. Legacy mode can only do recording up to max preview size
1156                 continue;
1157             }
1158             assertTrue("Video size " + videoSz.toString() + " for profile ID " + profileId +
1159                             " must be one of the camera device supported video size!",
1160                             mSupportedVideoSizes.contains(videoSz));
1161             assertTrue("Frame rate range " + fpsRange + " (for profile ID " + profileId +
1162                     ") must be one of the camera device available FPS range!",
1163                     fpsRanges.contains(fpsRange));
1164 
1165 
1166             if (useEncoderProfiles) {
1167                 // Iterate through all video-audio codec combination
1168                 EncoderProfiles profiles = CamcorderProfile.getAll(mCamera.getId(), profileId);
1169                 for (EncoderProfiles.VideoProfile videoProfile : profiles.getVideoProfiles()) {
1170                     boolean hasAudioProfile = false;
1171                     for (EncoderProfiles.AudioProfile audioProfile : profiles.getAudioProfiles()) {
1172                         hasAudioProfile = true;
1173                         doBasicRecordingByProfile(profiles, videoProfile, audioProfile,
1174                                 useVideoStab, useIntermediateSurface, validateRecording);
1175                         // Only measure the default video profile of the largest video
1176                         // recording size when measuring perf
1177                         if (isPerfMeasure()) {
1178                             break;
1179                         }
1180                     }
1181                     // Timelapse profiles do not have audio track
1182                     if (!hasAudioProfile) {
1183                         doBasicRecordingByProfile(profiles, videoProfile, /* audioProfile */null,
1184                                 useVideoStab, useIntermediateSurface, validateRecording);
1185                     }
1186                 }
1187             } else {
1188                 doBasicRecordingByProfile(
1189                         profile, useVideoStab, useIntermediateSurface, validateRecording);
1190             }
1191 
1192             if (isPerfMeasure()) {
1193                 // Only measure the largest video recording size when measuring perf
1194                 break;
1195             }
1196         }
1197         if (maxVideoFrameRate != -1) {
1198             // At least one CamcorderProfile is present, check FPS
1199             assertTrue("At least one CamcorderProfile must support >= 24 FPS",
1200                     maxVideoFrameRate >= 24);
1201         }
1202     }
1203 
doBasicRecordingByProfile( CamcorderProfile profile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1204     private void doBasicRecordingByProfile(
1205             CamcorderProfile profile, boolean userVideoStab,
1206             boolean useIntermediateSurface, boolean validate) throws Exception {
1207         Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1208         int frameRate = profile.videoFrameRate;
1209 
1210         if (VERBOSE) {
1211             Log.v(TAG, "Testing camera recording with video size " + videoSz.toString());
1212         }
1213 
1214         // Configure preview and recording surfaces.
1215         mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1216         if (DEBUG_DUMP) {
1217             mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1218                     + videoSz.toString() + ".mp4";
1219         }
1220 
1221         setupMediaRecorder(profile);
1222         completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate);
1223     }
1224 
doBasicRecordingByProfile( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile, boolean userVideoStab, boolean useIntermediateSurface, boolean validate)1225     private void doBasicRecordingByProfile(
1226             EncoderProfiles profiles,
1227             EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile,
1228             boolean userVideoStab, boolean useIntermediateSurface, boolean validate)
1229                     throws Exception {
1230         Size videoSz = new Size(videoProfile.getWidth(), videoProfile.getHeight());
1231         int frameRate = videoProfile.getFrameRate();
1232 
1233         if (VERBOSE) {
1234             Log.v(TAG, "Testing camera recording with video size " + videoSz.toString() +
1235                   ", video codec " + videoProfile.getMediaType() + ", and audio codec " +
1236                   (audioProfile == null ? "(null)" : audioProfile.getMediaType()));
1237         }
1238 
1239         // Configure preview and recording surfaces.
1240         mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1241         if (DEBUG_DUMP) {
1242             mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1243                     + videoSz.toString() + "_" + videoProfile.getCodec();
1244             if (audioProfile != null) {
1245                 mOutMediaFileName += "_" + audioProfile.getCodec();
1246             }
1247             mOutMediaFileName += ".mp4";
1248         }
1249 
1250         setupMediaRecorder(profiles, videoProfile, audioProfile);
1251         completeBasicRecording(videoSz, frameRate, userVideoStab, useIntermediateSurface, validate);
1252     }
1253 
completeBasicRecording( Size videoSz, int frameRate, boolean useVideoStab, boolean useIntermediateSurface, boolean validate)1254     private void completeBasicRecording(
1255             Size videoSz, int frameRate, boolean useVideoStab,
1256             boolean useIntermediateSurface, boolean validate) throws Exception {
1257         prepareRecording(useIntermediateSurface);
1258 
1259         // prepare preview surface by using video size.
1260         updatePreviewSurfaceWithVideo(videoSz, frameRate);
1261 
1262         // Start recording
1263         SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1264         startRecording(/* useMediaRecorder */true, resultListener, useVideoStab,
1265                 useIntermediateSurface);
1266 
1267         // Record certain duration.
1268         SystemClock.sleep(RECORDING_DURATION_MS);
1269 
1270         // Stop recording and preview
1271         stopRecording(/* useMediaRecorder */true, useIntermediateSurface,
1272                 /* stopCameraStreaming */true);
1273         // Convert number of frames camera produced into the duration in unit of ms.
1274         float frameDurationMs = 1000.0f / frameRate;
1275         float durationMs = 0.f;
1276         if (useIntermediateSurface) {
1277             durationMs = mQueuer.getQueuedCount() * frameDurationMs;
1278         } else {
1279             durationMs = resultListener.getTotalNumFrames() * frameDurationMs;
1280         }
1281 
1282         if (VERBOSE) {
1283             Log.v(TAG, "video frame rate: " + frameRate +
1284                             ", num of frames produced: " + resultListener.getTotalNumFrames());
1285         }
1286 
1287         if (validate) {
1288             validateRecording(videoSz, durationMs, frameDurationMs, FRMDRP_RATE_TOLERANCE);
1289         }
1290     }
1291 
1292     /**
1293      * Test camera recording for each supported video size by camera, preview
1294      * size is set to the video size.
1295      */
recordingSizeTestByCamera()1296     private void recordingSizeTestByCamera() throws Exception {
1297         for (Size sz : mSupportedVideoSizes) {
1298             if (!isSupported(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE)) {
1299                 continue;
1300             }
1301 
1302             if (VERBOSE) {
1303                 Log.v(TAG, "Testing camera recording with video size " + sz.toString());
1304             }
1305 
1306             // Configure preview and recording surfaces.
1307             mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1308             if (DEBUG_DUMP) {
1309                 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + mCamera.getId() + "_"
1310                         + sz.toString() + ".mp4";
1311             }
1312 
1313             // Allow external camera to use variable fps range
1314             Range<Integer> fpsRange = null;
1315             if (mStaticInfo.isExternalCamera()) {
1316                 Range<Integer>[] availableFpsRange =
1317                         mStaticInfo.getAeAvailableTargetFpsRangesChecked();
1318 
1319                 boolean foundRange = false;
1320                 int minFps = 0;
1321                 for (int i = 0; i < availableFpsRange.length; i += 1) {
1322                     if (minFps < availableFpsRange[i].getLower()
1323                             && VIDEO_FRAME_RATE == availableFpsRange[i].getUpper()) {
1324                         minFps = availableFpsRange[i].getLower();
1325                         foundRange = true;
1326                     }
1327                 }
1328                 assertTrue("Cannot find FPS range for maxFps " + VIDEO_FRAME_RATE, foundRange);
1329                 fpsRange = Range.create(minFps, VIDEO_FRAME_RATE);
1330             }
1331 
1332             // Use AVC and AAC a/v compression format.
1333             prepareRecording(sz, VIDEO_FRAME_RATE, VIDEO_FRAME_RATE);
1334 
1335             // prepare preview surface by using video size.
1336             updatePreviewSurfaceWithVideo(sz, VIDEO_FRAME_RATE);
1337 
1338             // Start recording
1339             SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1340             startRecording(
1341                     /* useMediaRecorder */true, resultListener,
1342                     /*useVideoStab*/false, fpsRange, false);
1343 
1344             // Record certain duration.
1345             SystemClock.sleep(RECORDING_DURATION_MS);
1346 
1347             // Stop recording and preview
1348             stopRecording(/* useMediaRecorder */true);
1349             // Convert number of frames camera produced into the duration in unit of ms.
1350             float frameDurationMinMs = 1000.0f / VIDEO_FRAME_RATE;
1351             float durationMinMs = resultListener.getTotalNumFrames() * frameDurationMinMs;
1352             float durationMaxMs = durationMinMs;
1353             float frameDurationMaxMs = 0.f;
1354             if (fpsRange != null) {
1355                 frameDurationMaxMs = 1000.0f / fpsRange.getLower();
1356                 durationMaxMs = resultListener.getTotalNumFrames() * frameDurationMaxMs;
1357             }
1358 
1359             // Validation.
1360             validateRecording(sz, durationMinMs, durationMaxMs,
1361                     frameDurationMinMs, frameDurationMaxMs,
1362                     FRMDRP_RATE_TOLERANCE);
1363         }
1364     }
1365 
1366     /**
1367      * Initialize the supported video sizes.
1368      */
initSupportedVideoSize(String cameraId)1369     private void initSupportedVideoSize(String cameraId)  throws Exception {
1370         Size maxVideoSize = SIZE_BOUND_1080P;
1371         if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2160P)) {
1372             maxVideoSize = SIZE_BOUND_2160P;
1373         } else if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_QHD)) {
1374             maxVideoSize = SIZE_BOUND_QHD;
1375         } else if (CamcorderProfile.hasProfile(CamcorderProfile.QUALITY_2K)) {
1376             maxVideoSize = SIZE_BOUND_2K;
1377         }
1378 
1379         mSupportedVideoSizes =
1380                 getSupportedVideoSizes(cameraId, mCameraManager, maxVideoSize);
1381     }
1382 
1383     /**
1384      * Simple wrapper to wrap normal/burst video snapshot tests
1385      */
videoSnapshotHelper(boolean burstTest)1386     private void videoSnapshotHelper(boolean burstTest) throws Exception {
1387             for (String id : mCameraIdsUnderTest) {
1388                 try {
1389                     Log.i(TAG, "Testing video snapshot for camera " + id);
1390 
1391                     StaticMetadata staticInfo = mAllStaticInfo.get(id);
1392                     if (!staticInfo.isColorOutputSupported()) {
1393                         Log.i(TAG, "Camera " + id +
1394                                 " does not support color outputs, skipping");
1395                         continue;
1396                     }
1397 
1398                     if (staticInfo.isExternalCamera()) {
1399                         Log.i(TAG, "Camera " + id +
1400                                 " does not support CamcorderProfile, skipping");
1401                         continue;
1402                     }
1403 
1404                     // Re-use the MediaRecorder object for the same camera device.
1405                     mMediaRecorder = new MediaRecorder();
1406 
1407                     openDevice(id);
1408 
1409                     initSupportedVideoSize(id);
1410 
1411                     videoSnapshotTestByCamera(burstTest);
1412                 } finally {
1413                     closeDevice();
1414                     releaseRecorder();
1415                 }
1416             }
1417     }
1418 
1419     /**
1420      * Returns {@code true} if the {@link CamcorderProfile} ID is allowed to be unsupported.
1421      *
1422      * <p>This only allows unsupported profiles when using the LEGACY mode of the Camera API.</p>
1423      *
1424      * @param profileId a {@link CamcorderProfile} ID to check.
1425      * @return {@code true} if supported.
1426      */
allowedUnsupported(int cameraId, int profileId)1427     private boolean allowedUnsupported(int cameraId, int profileId) {
1428         if (!mStaticInfo.isHardwareLevelLegacy()) {
1429             return false;
1430         }
1431 
1432         switch(profileId) {
1433             case CamcorderProfile.QUALITY_2160P:
1434             case CamcorderProfile.QUALITY_1080P:
1435             case CamcorderProfile.QUALITY_HIGH:
1436                 return !CamcorderProfile.hasProfile(cameraId, profileId) ||
1437                         CamcorderProfile.get(cameraId, profileId).videoFrameWidth >= 1080;
1438         }
1439         return false;
1440     }
1441 
1442     /**
1443      * Test video snapshot for each  available CamcorderProfile for a given camera.
1444      *
1445      * <p>
1446      * Preview size is set to the video size. For the burst test, frame drop and jittering
1447      * is not checked.
1448      * </p>
1449      *
1450      * @param burstTest Perform burst capture or single capture. For burst capture
1451      *                  {@value #BURST_VIDEO_SNAPSHOT_NUM} capture requests will be sent.
1452      */
videoSnapshotTestByCamera(boolean burstTest)1453     private void videoSnapshotTestByCamera(boolean burstTest)
1454             throws Exception {
1455         final int NUM_SINGLE_SHOT_TEST = 5;
1456         final int FRAMEDROP_TOLERANCE = 8;
1457         final int FRAME_SIZE_15M = 15000000;
1458         final float FRAME_DROP_TOLERENCE_FACTOR = 1.5f;
1459         int kFrameDrop_Tolerence = FRAMEDROP_TOLERANCE;
1460 
1461         for (int profileId : mCamcorderProfileList) {
1462             int cameraId = Integer.valueOf(mCamera.getId());
1463             if (!CamcorderProfile.hasProfile(cameraId, profileId) ||
1464                     allowedUnsupported(cameraId, profileId)) {
1465                 continue;
1466             }
1467 
1468             CamcorderProfile profile = CamcorderProfile.get(cameraId, profileId);
1469             Size QCIF = new Size(176, 144);
1470             Size FULL_HD = new Size(1920, 1080);
1471             Size videoSz = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1472             Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1473 
1474             if (mStaticInfo.isHardwareLevelLegacy() &&
1475                     (videoSz.getWidth() > maxPreviewSize.getWidth() ||
1476                      videoSz.getHeight() > maxPreviewSize.getHeight())) {
1477                 // Skip. Legacy mode can only do recording up to max preview size
1478                 continue;
1479             }
1480 
1481             if (!mSupportedVideoSizes.contains(videoSz)) {
1482                 mCollector.addMessage("Video size " + videoSz.toString() + " for profile ID " +
1483                         profileId + " must be one of the camera device supported video size!");
1484                 continue;
1485             }
1486 
1487             // For LEGACY, find closest supported smaller or equal JPEG size to the current video
1488             // size; if no size is smaller than the video, pick the smallest JPEG size.  The assert
1489             // for video size above guarantees that for LIMITED or FULL, we select videoSz here.
1490             // Also check for minFrameDuration here to make sure jpeg stream won't slow down
1491             // video capture
1492             Size videoSnapshotSz = mOrderedStillSizes.get(mOrderedStillSizes.size() - 1);
1493             // Allow a bit tolerance so we don't fail for a few nano seconds of difference
1494             final float FRAME_DURATION_TOLERANCE = 0.01f;
1495             long videoFrameDuration = (long) (1e9 / profile.videoFrameRate *
1496                     (1.0 + FRAME_DURATION_TOLERANCE));
1497             HashMap<Size, Long> minFrameDurationMap = mStaticInfo.
1498                     getAvailableMinFrameDurationsForFormatChecked(ImageFormat.JPEG);
1499             for (int i = mOrderedStillSizes.size() - 2; i >= 0; i--) {
1500                 Size candidateSize = mOrderedStillSizes.get(i);
1501                 if (mStaticInfo.isHardwareLevelLegacy()) {
1502                     // Legacy level doesn't report min frame duration
1503                     if (candidateSize.getWidth() <= videoSz.getWidth() &&
1504                             candidateSize.getHeight() <= videoSz.getHeight()) {
1505                         videoSnapshotSz = candidateSize;
1506                     }
1507                 } else {
1508                     Long jpegFrameDuration = minFrameDurationMap.get(candidateSize);
1509                     assertTrue("Cannot find minimum frame duration for jpeg size " + candidateSize,
1510                             jpegFrameDuration != null);
1511                     if (candidateSize.getWidth() <= videoSz.getWidth() &&
1512                             candidateSize.getHeight() <= videoSz.getHeight() &&
1513                             jpegFrameDuration <= videoFrameDuration) {
1514                         videoSnapshotSz = candidateSize;
1515                     }
1516                 }
1517             }
1518             Size defaultvideoSnapshotSz = videoSnapshotSz;
1519 
1520             /**
1521              * Only test full res snapshot when below conditions are all true.
1522              * 1. Camera is at least a LIMITED device.
1523              * 2. video size is up to max preview size, which will be bounded by 1080p.
1524              * 3. Full resolution jpeg stream can keep up to video stream speed.
1525              *    When full res jpeg stream cannot keep up to video stream speed, search
1526              *    the largest jpeg size that can susptain video speed instead.
1527              */
1528             if (mStaticInfo.isHardwareLevelAtLeastLimited() &&
1529                     videoSz.getWidth() <= maxPreviewSize.getWidth() &&
1530                     videoSz.getHeight() <= maxPreviewSize.getHeight()) {
1531                 for (Size jpegSize : mOrderedStillSizes) {
1532                     Long jpegFrameDuration = minFrameDurationMap.get(jpegSize);
1533                     assertTrue("Cannot find minimum frame duration for jpeg size " + jpegSize,
1534                             jpegFrameDuration != null);
1535                     if (jpegFrameDuration <= videoFrameDuration) {
1536                         videoSnapshotSz = jpegSize;
1537                         break;
1538                     }
1539                     if (jpegSize.equals(videoSz)) {
1540                         throw new AssertionFailedError(
1541                                 "Cannot find adequate video snapshot size for video size" +
1542                                         videoSz);
1543                     }
1544                 }
1545             }
1546 
1547             if (videoSnapshotSz.getWidth() * videoSnapshotSz.getHeight() > FRAME_SIZE_15M)
1548                 kFrameDrop_Tolerence = (int)(FRAMEDROP_TOLERANCE * FRAME_DROP_TOLERENCE_FACTOR);
1549 
1550             createImageReader(
1551                     videoSnapshotSz, ImageFormat.JPEG,
1552                     MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1553 
1554             // Full or better devices should support whatever video snapshot size calculated above.
1555             // Limited devices may only be able to support the default one.
1556             if (mStaticInfo.isHardwareLevelLimited()) {
1557                 List<Surface> outputs = new ArrayList<Surface>();
1558                 outputs.add(mPreviewSurface);
1559                 outputs.add(mRecordingSurface);
1560                 outputs.add(mReaderSurface);
1561                 boolean isSupported = isStreamConfigurationSupported(
1562                         mCamera, outputs, mSessionListener, mHandler);
1563                 if (!isSupported) {
1564                     videoSnapshotSz = defaultvideoSnapshotSz;
1565                     createImageReader(
1566                             videoSnapshotSz, ImageFormat.JPEG,
1567                             MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1568                 }
1569             }
1570 
1571             if (videoSz.equals(QCIF) &&
1572                     ((videoSnapshotSz.getWidth() > FULL_HD.getWidth()) ||
1573                      (videoSnapshotSz.getHeight() > FULL_HD.getHeight()))) {
1574                 List<Surface> outputs = new ArrayList<Surface>();
1575                 outputs.add(mPreviewSurface);
1576                 outputs.add(mRecordingSurface);
1577                 outputs.add(mReaderSurface);
1578                 boolean isSupported = isStreamConfigurationSupported(
1579                         mCamera, outputs, mSessionListener, mHandler);
1580                 if (!isSupported) {
1581                     videoSnapshotSz = defaultvideoSnapshotSz;
1582                     createImageReader(
1583                             videoSnapshotSz, ImageFormat.JPEG,
1584                             MAX_VIDEO_SNAPSHOT_IMAGES, /*listener*/null);
1585                 }
1586             }
1587 
1588             Log.i(TAG, "Testing video snapshot size " + videoSnapshotSz +
1589                     " for video size " + videoSz);
1590 
1591             if (VERBOSE) {
1592                 Log.v(TAG, "Testing camera recording with video size " + videoSz.toString());
1593             }
1594 
1595             // Configure preview and recording surfaces.
1596             mOutMediaFileName = mDebugFileNameBase + "/test_video.mp4";
1597             if (DEBUG_DUMP) {
1598                 mOutMediaFileName = mDebugFileNameBase + "/test_video_" + cameraId + "_"
1599                         + videoSz.toString() + ".mp4";
1600             }
1601 
1602             int numTestIterations = burstTest ? 1 : NUM_SINGLE_SHOT_TEST;
1603             int totalDroppedFrames = 0;
1604 
1605             for (int numTested = 0; numTested < numTestIterations; numTested++) {
1606                 prepareRecordingWithProfile(profile);
1607 
1608                 // prepare video snapshot
1609                 SimpleCaptureCallback resultListener = new SimpleCaptureCallback();
1610                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener();
1611                 CaptureRequest.Builder videoSnapshotRequestBuilder =
1612                         mCamera.createCaptureRequest((mStaticInfo.isHardwareLevelLegacy()) ?
1613                                 CameraDevice.TEMPLATE_RECORD :
1614                                 CameraDevice.TEMPLATE_VIDEO_SNAPSHOT);
1615 
1616                 // prepare preview surface by using video size.
1617                 updatePreviewSurfaceWithVideo(videoSz, profile.videoFrameRate);
1618 
1619                 prepareVideoSnapshot(videoSnapshotRequestBuilder, imageListener);
1620                 Range<Integer> fpsRange = Range.create(profile.videoFrameRate,
1621                         profile.videoFrameRate);
1622                 videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE,
1623                         fpsRange);
1624                 if (mStaticInfo.isVideoStabilizationSupported()) {
1625                     videoSnapshotRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1626                             CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1627                 }
1628                 CaptureRequest request = videoSnapshotRequestBuilder.build();
1629 
1630                 // Start recording
1631                 startRecording(/* useMediaRecorder */true, resultListener,
1632                         /*useVideoStab*/mStaticInfo.isVideoStabilizationSupported());
1633                 long startTime = SystemClock.elapsedRealtime();
1634 
1635                 // Record certain duration.
1636                 SystemClock.sleep(RECORDING_DURATION_MS / 2);
1637 
1638                 // take video snapshot
1639                 if (burstTest) {
1640                     List<CaptureRequest> requests =
1641                             new ArrayList<CaptureRequest>(BURST_VIDEO_SNAPSHOT_NUM);
1642                     for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) {
1643                         requests.add(request);
1644                     }
1645                     mSession.captureBurst(requests, resultListener, mHandler);
1646                 } else {
1647                     mSession.capture(request, resultListener, mHandler);
1648                 }
1649 
1650                 // make sure recording is still going after video snapshot
1651                 SystemClock.sleep(RECORDING_DURATION_MS / 2);
1652 
1653                 // Stop recording and preview
1654                 float durationMs = (float) stopRecording(/* useMediaRecorder */true);
1655                 // For non-burst test, use number of frames to also double check video frame rate.
1656                 // Burst video snapshot is allowed to cause frame rate drop, so do not use number
1657                 // of frames to estimate duration
1658                 if (!burstTest) {
1659                     durationMs = resultListener.getTotalNumFrames() * 1000.0f /
1660                         profile.videoFrameRate;
1661                 }
1662 
1663                 float frameDurationMs = 1000.0f / profile.videoFrameRate;
1664                 // Validation recorded video
1665                 validateRecording(videoSz, durationMs,
1666                         frameDurationMs, VID_SNPSHT_FRMDRP_RATE_TOLERANCE);
1667 
1668                 if (burstTest) {
1669                     for (int i = 0; i < BURST_VIDEO_SNAPSHOT_NUM; i++) {
1670                         Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS);
1671                         validateVideoSnapshotCapture(image, videoSnapshotSz);
1672                         image.close();
1673                     }
1674                 } else {
1675                     // validate video snapshot image
1676                     Image image = imageListener.getImage(CAPTURE_IMAGE_TIMEOUT_MS);
1677                     validateVideoSnapshotCapture(image, videoSnapshotSz);
1678 
1679                     // validate if there is framedrop around video snapshot
1680                     totalDroppedFrames +=  validateFrameDropAroundVideoSnapshot(
1681                             resultListener, image.getTimestamp());
1682 
1683                     //TODO: validate jittering. Should move to PTS
1684                     //validateJittering(resultListener);
1685 
1686                     image.close();
1687                 }
1688             }
1689 
1690             if (!burstTest) {
1691                 Log.w(TAG, String.format("Camera %d Video size %s: Number of dropped frames " +
1692                         "detected in %d trials is %d frames.", cameraId, videoSz.toString(),
1693                         numTestIterations, totalDroppedFrames));
1694                 mCollector.expectLessOrEqual(
1695                         String.format(
1696                                 "Camera %d Video size %s: Number of dropped frames %d must not"
1697                                 + " be larger than %d",
1698                                 cameraId, videoSz.toString(), totalDroppedFrames,
1699                                 kFrameDrop_Tolerence),
1700                         kFrameDrop_Tolerence, totalDroppedFrames);
1701             }
1702             closeImageReader();
1703         }
1704     }
1705 
1706     /**
1707      * Configure video snapshot request according to the still capture size
1708      */
prepareVideoSnapshot( CaptureRequest.Builder requestBuilder, ImageReader.OnImageAvailableListener imageListener)1709     private void prepareVideoSnapshot(
1710             CaptureRequest.Builder requestBuilder,
1711             ImageReader.OnImageAvailableListener imageListener)
1712             throws Exception {
1713         mReader.setOnImageAvailableListener(imageListener, mHandler);
1714         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1715         requestBuilder.addTarget(mRecordingSurface);
1716         assertNotNull("Preview surface must be non-null!", mPreviewSurface);
1717         requestBuilder.addTarget(mPreviewSurface);
1718         assertNotNull("Reader surface must be non-null!", mReaderSurface);
1719         requestBuilder.addTarget(mReaderSurface);
1720     }
1721 
1722     /**
1723      * Find compatible preview sizes for video size and framerate.
1724      *
1725      * <p>Preview size will be capped with max preview size.</p>
1726      *
1727      * @param videoSize The video size used for preview.
1728      * @param videoFrameRate The video frame rate
1729      */
getPreviewSizesForVideo(Size videoSize, int videoFrameRate)1730     private List<Size> getPreviewSizesForVideo(Size videoSize, int videoFrameRate) {
1731         if (mOrderedPreviewSizes == null) {
1732             throw new IllegalStateException("supported preview size list is not initialized yet");
1733         }
1734         final float FRAME_DURATION_TOLERANCE = 0.01f;
1735         long videoFrameDuration = (long) (1e9 / videoFrameRate *
1736                 (1.0 + FRAME_DURATION_TOLERANCE));
1737         HashMap<Size, Long> minFrameDurationMap = mStaticInfo.
1738                 getAvailableMinFrameDurationsForFormatChecked(ImageFormat.PRIVATE);
1739         Size maxPreviewSize = mOrderedPreviewSizes.get(0);
1740         ArrayList<Size> previewSizes = new ArrayList<>();
1741         if (videoSize.getWidth() > maxPreviewSize.getWidth() ||
1742                 videoSize.getHeight() > maxPreviewSize.getHeight()) {
1743             for (Size s : mOrderedPreviewSizes) {
1744                 Long frameDuration = minFrameDurationMap.get(s);
1745                 if (mStaticInfo.isHardwareLevelLegacy()) {
1746                     // Legacy doesn't report min frame duration
1747                     frameDuration = new Long(0);
1748                 }
1749                 assertTrue("Cannot find minimum frame duration for private size" + s,
1750                         frameDuration != null);
1751                 if (frameDuration <= videoFrameDuration &&
1752                         s.getWidth() <= videoSize.getWidth() &&
1753                         s.getHeight() <= videoSize.getHeight()) {
1754                     Log.v(TAG, "Add preview size " + s.toString() + " for video size " +
1755                             videoSize.toString());
1756                     previewSizes.add(s);
1757                 }
1758             }
1759         }
1760 
1761         if (previewSizes.isEmpty()) {
1762             previewSizes.add(videoSize);
1763         }
1764 
1765         return previewSizes;
1766     }
1767 
1768     /**
1769      * Update preview size with video size.
1770      *
1771      * <p>Preview size will be capped with max preview size.</p>
1772      *
1773      * @param videoSize The video size used for preview.
1774      * @param videoFrameRate The video frame rate
1775      *
1776      */
updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate)1777     private void updatePreviewSurfaceWithVideo(Size videoSize, int videoFrameRate) {
1778         List<Size> previewSizes = getPreviewSizesForVideo(videoSize, videoFrameRate);
1779         updatePreviewSurface(previewSizes.get(0));
1780     }
1781 
prepareRecordingWithProfile(CamcorderProfile profile)1782     private void prepareRecordingWithProfile(CamcorderProfile profile) throws Exception {
1783         prepareRecordingWithProfile(profile, false);
1784     }
1785 
1786     /**
1787      * Configure MediaRecorder recording session with CamcorderProfile, prepare
1788      * the recording surface.
1789      */
prepareRecordingWithProfile(CamcorderProfile profile, boolean useIntermediateSurface)1790     private void prepareRecordingWithProfile(CamcorderProfile profile,
1791             boolean useIntermediateSurface) throws Exception {
1792         // Prepare MediaRecorder.
1793         setupMediaRecorder(profile);
1794         prepareRecording(useIntermediateSurface);
1795     }
1796 
setupMediaRecorder(CamcorderProfile profile)1797     private void setupMediaRecorder(CamcorderProfile profile) throws Exception {
1798         // Set-up MediaRecorder.
1799         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1800         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1801         mMediaRecorder.setProfile(profile);
1802 
1803         mVideoFrameRate = profile.videoFrameRate;
1804         mVideoSize = new Size(profile.videoFrameWidth, profile.videoFrameHeight);
1805     }
1806 
setupMediaRecorder( EncoderProfiles profiles, EncoderProfiles.VideoProfile videoProfile, EncoderProfiles.AudioProfile audioProfile)1807     private void setupMediaRecorder(
1808             EncoderProfiles profiles,
1809             EncoderProfiles.VideoProfile videoProfile,
1810             EncoderProfiles.AudioProfile audioProfile) throws Exception {
1811         // Set-up MediaRecorder.
1812         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1813         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1814         mMediaRecorder.setOutputFormat(profiles.getRecommendedFileFormat());
1815         mMediaRecorder.setVideoProfile(videoProfile);
1816         if (audioProfile != null) {
1817             mMediaRecorder.setAudioProfile(audioProfile);
1818         }
1819 
1820         mVideoFrameRate = videoProfile.getFrameRate();
1821         mVideoSize = new Size(videoProfile.getWidth(), videoProfile.getHeight());
1822     }
1823 
prepareRecording(boolean useIntermediateSurface)1824     private void prepareRecording(boolean useIntermediateSurface) throws Exception {
1825         // Continue preparing MediaRecorder
1826         mMediaRecorder.setOutputFile(mOutMediaFileName);
1827         if (mPersistentSurface != null) {
1828             mMediaRecorder.setInputSurface(mPersistentSurface);
1829             mRecordingSurface = mPersistentSurface;
1830         }
1831         mMediaRecorder.prepare();
1832         if (mPersistentSurface == null) {
1833             mRecordingSurface = mMediaRecorder.getSurface();
1834         }
1835         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1836 
1837         if (useIntermediateSurface) {
1838             mIntermediateReader = ImageReader.newInstance(
1839                     mVideoSize.getWidth(), mVideoSize.getHeight(),
1840                     ImageFormat.PRIVATE, /*maxImages*/3, HardwareBuffer.USAGE_VIDEO_ENCODE);
1841 
1842             mIntermediateSurface = mIntermediateReader.getSurface();
1843             mIntermediateWriter = ImageWriter.newInstance(mRecordingSurface, /*maxImages*/3,
1844                     ImageFormat.PRIVATE);
1845             mQueuer = new ImageWriterQueuer(mIntermediateWriter);
1846 
1847             mIntermediateThread = new HandlerThread(TAG);
1848             mIntermediateThread.start();
1849             mIntermediateHandler = new Handler(mIntermediateThread.getLooper());
1850             mIntermediateReader.setOnImageAvailableListener(mQueuer, mIntermediateHandler);
1851         }
1852     }
1853 
1854     /**
1855      * Configure MediaRecorder recording session with CamcorderProfile, prepare
1856      * the recording surface. Use AVC for video compression, AAC for audio compression.
1857      * Both are required for android devices by android CDD.
1858      */
prepareRecording(Size sz, int videoFrameRate, int captureRate)1859     private void prepareRecording(Size sz, int videoFrameRate, int captureRate)
1860             throws Exception {
1861         // Prepare MediaRecorder.
1862         mMediaRecorder.setAudioSource(MediaRecorder.AudioSource.CAMCORDER);
1863         mMediaRecorder.setVideoSource(MediaRecorder.VideoSource.SURFACE);
1864         mMediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP);
1865         mMediaRecorder.setOutputFile(mOutMediaFileName);
1866         mMediaRecorder.setVideoEncodingBitRate(getVideoBitRate(sz));
1867         mMediaRecorder.setVideoFrameRate(videoFrameRate);
1868         mMediaRecorder.setCaptureRate(captureRate);
1869         mMediaRecorder.setVideoSize(sz.getWidth(), sz.getHeight());
1870         mMediaRecorder.setVideoEncoder(MediaRecorder.VideoEncoder.H264);
1871         mMediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC);
1872         if (mPersistentSurface != null) {
1873             mMediaRecorder.setInputSurface(mPersistentSurface);
1874             mRecordingSurface = mPersistentSurface;
1875         }
1876         mMediaRecorder.prepare();
1877         if (mPersistentSurface == null) {
1878             mRecordingSurface = mMediaRecorder.getSurface();
1879         }
1880         assertNotNull("Recording surface must be non-null!", mRecordingSurface);
1881         mVideoFrameRate = videoFrameRate;
1882         mVideoSize = sz;
1883     }
1884 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab)1885     private void startRecording(boolean useMediaRecorder,
1886             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab) throws Exception {
1887         startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null,
1888                 /*useIntermediateSurface*/false);
1889     }
1890 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, boolean useIntermediateSurface)1891     private void startRecording(boolean useMediaRecorder,
1892             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1893             boolean useIntermediateSurface) throws Exception {
1894         startRecording(useMediaRecorder, listener, useVideoStab, /*variableFpsRange*/null,
1895                 useIntermediateSurface);
1896     }
1897 
startRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange, boolean useIntermediateSurface)1898     private void startRecording(boolean useMediaRecorder,
1899             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1900             Range<Integer> variableFpsRange, boolean useIntermediateSurface) throws Exception {
1901         if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) {
1902             throw new IllegalArgumentException("Video stabilization is not supported");
1903         }
1904 
1905         List<Surface> outputSurfaces = new ArrayList<Surface>(2);
1906         assertTrue("Both preview and recording surfaces should be valid",
1907                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1908         outputSurfaces.add(mPreviewSurface);
1909         if (useIntermediateSurface) {
1910             outputSurfaces.add(mIntermediateSurface);
1911         } else {
1912             outputSurfaces.add(mRecordingSurface);
1913         }
1914 
1915         // Video snapshot surface
1916         if (mReaderSurface != null) {
1917             outputSurfaces.add(mReaderSurface);
1918         }
1919         mSessionListener = new BlockingSessionCallback();
1920 
1921         CaptureRequest.Builder recordingRequestBuilder =
1922                 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1923         // Make sure camera output frame rate is set to correct value.
1924         Range<Integer> fpsRange = (variableFpsRange == null) ?
1925                 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange;
1926 
1927         recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1928         if (useVideoStab) {
1929             recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1930                     CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1931         }
1932         if (useIntermediateSurface) {
1933             recordingRequestBuilder.addTarget(mIntermediateSurface);
1934             if (mQueuer != null) {
1935                 mQueuer.resetInvalidSurfaceFlag();
1936             }
1937         } else {
1938             recordingRequestBuilder.addTarget(mRecordingSurface);
1939         }
1940         recordingRequestBuilder.addTarget(mPreviewSurface);
1941         CaptureRequest recordingRequest = recordingRequestBuilder.build();
1942         mSession = configureCameraSessionWithParameters(mCamera, outputSurfaces, mSessionListener,
1943                 mHandler, recordingRequest);
1944         mSession.setRepeatingRequest(recordingRequest, listener, mHandler);
1945 
1946         if (useMediaRecorder) {
1947             mMediaRecorder.start();
1948         } else {
1949             // TODO: need implement MediaCodec path.
1950         }
1951         mRecordingStartTime = SystemClock.elapsedRealtime();
1952     }
1953 
1954     /**
1955      * Start video recording with preview and video surfaces sharing the same
1956      * camera stream.
1957      *
1958      * @return true if success, false if sharing is not supported.
1959      */
startSharedRecording(boolean useMediaRecorder, CameraCaptureSession.CaptureCallback listener, boolean useVideoStab, Range<Integer> variableFpsRange)1960     private boolean startSharedRecording(boolean useMediaRecorder,
1961             CameraCaptureSession.CaptureCallback listener, boolean useVideoStab,
1962             Range<Integer> variableFpsRange) throws Exception {
1963         if (!mStaticInfo.isVideoStabilizationSupported() && useVideoStab) {
1964             throw new IllegalArgumentException("Video stabilization is not supported");
1965         }
1966 
1967         List<OutputConfiguration> outputConfigs = new ArrayList<OutputConfiguration>(2);
1968         assertTrue("Both preview and recording surfaces should be valid",
1969                 mPreviewSurface.isValid() && mRecordingSurface.isValid());
1970         OutputConfiguration sharedConfig = new OutputConfiguration(mPreviewSurface);
1971         sharedConfig.enableSurfaceSharing();
1972         sharedConfig.addSurface(mRecordingSurface);
1973         outputConfigs.add(sharedConfig);
1974 
1975         CaptureRequest.Builder recordingRequestBuilder =
1976                 mCamera.createCaptureRequest(CameraDevice.TEMPLATE_RECORD);
1977         // Make sure camera output frame rate is set to correct value.
1978         Range<Integer> fpsRange = (variableFpsRange == null) ?
1979                 Range.create(mVideoFrameRate, mVideoFrameRate) : variableFpsRange;
1980         recordingRequestBuilder.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange);
1981         if (useVideoStab) {
1982             recordingRequestBuilder.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE,
1983                     CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE_ON);
1984         }
1985         CaptureRequest recordingRequest = recordingRequestBuilder.build();
1986 
1987         mSessionListener = new BlockingSessionCallback();
1988         mSession = tryConfigureCameraSessionWithConfig(mCamera, outputConfigs, recordingRequest,
1989                 mSessionListener, mHandler);
1990 
1991         if (mSession == null) {
1992             Log.i(TAG, "Sharing between preview and video is not supported");
1993             return false;
1994         }
1995 
1996         recordingRequestBuilder.addTarget(mRecordingSurface);
1997         recordingRequestBuilder.addTarget(mPreviewSurface);
1998         mSession.setRepeatingRequest(recordingRequestBuilder.build(), listener, mHandler);
1999 
2000         if (useMediaRecorder) {
2001             mMediaRecorder.start();
2002         } else {
2003             // TODO: need implement MediaCodec path.
2004         }
2005         mRecordingStartTime = SystemClock.elapsedRealtime();
2006         return true;
2007     }
2008 
2009 
stopCameraStreaming()2010     private void stopCameraStreaming() throws Exception {
2011         if (VERBOSE) {
2012             Log.v(TAG, "Stopping camera streaming and waiting for idle");
2013         }
2014         // Stop repeating, wait for captures to complete, and disconnect from
2015         // surfaces
2016         mSession.close();
2017         mSessionListener.getStateWaiter().waitForState(SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS);
2018     }
2019 
stopRecording(boolean useMediaRecorder)2020     private int stopRecording(boolean useMediaRecorder) throws Exception {
2021         return stopRecording(useMediaRecorder, /*useIntermediateSurface*/false,
2022                 /*stopStreaming*/true);
2023     }
2024 
2025     // Stop recording and return the estimated video duration in milliseconds.
stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface, boolean stopStreaming)2026     private int stopRecording(boolean useMediaRecorder, boolean useIntermediateSurface,
2027             boolean stopStreaming) throws Exception {
2028         long stopRecordingTime = SystemClock.elapsedRealtime();
2029         if (useMediaRecorder) {
2030             if (stopStreaming) {
2031                 stopCameraStreaming();
2032             }
2033             if (useIntermediateSurface) {
2034                 mIntermediateReader.setOnImageAvailableListener(null, null);
2035                 mQueuer.expectInvalidSurface();
2036             }
2037 
2038             mMediaRecorder.stop();
2039             // Can reuse the MediaRecorder object after reset.
2040             mMediaRecorder.reset();
2041         } else {
2042             // TODO: need implement MediaCodec path.
2043         }
2044 
2045         if (useIntermediateSurface) {
2046             mIntermediateReader.close();
2047             mQueuer.close();
2048             mIntermediateWriter.close();
2049             mIntermediateSurface.release();
2050             mIntermediateReader = null;
2051             mIntermediateSurface = null;
2052             mIntermediateWriter = null;
2053             mIntermediateThread.quitSafely();
2054             mIntermediateHandler = null;
2055         }
2056 
2057         if (mPersistentSurface == null && mRecordingSurface != null) {
2058             mRecordingSurface.release();
2059             mRecordingSurface = null;
2060         }
2061         return (int) (stopRecordingTime - mRecordingStartTime);
2062     }
2063 
releaseRecorder()2064     private void releaseRecorder() {
2065         if (mMediaRecorder != null) {
2066             mMediaRecorder.release();
2067             mMediaRecorder = null;
2068         }
2069     }
2070 
validateRecording( Size sz, float expectedDurationMs, float expectedFrameDurationMs, float frameDropTolerance)2071     private void validateRecording(
2072             Size sz, float expectedDurationMs, float expectedFrameDurationMs,
2073             float frameDropTolerance) throws Exception {
2074         validateRecording(sz,
2075                 expectedDurationMs,  /*fixed FPS recording*/0.f,
2076                 expectedFrameDurationMs, /*fixed FPS recording*/0.f,
2077                 frameDropTolerance);
2078     }
2079 
validateRecording( Size sz, float expectedDurationMinMs, float expectedDurationMaxMs, float expectedFrameDurationMinMs, float expectedFrameDurationMaxMs, float frameDropTolerance)2080     private void validateRecording(
2081             Size sz,
2082             float expectedDurationMinMs,      // Min duration (maxFps)
2083             float expectedDurationMaxMs,      // Max duration (minFps). 0.f for fixed fps recording
2084             float expectedFrameDurationMinMs, // maxFps
2085             float expectedFrameDurationMaxMs, // minFps. 0.f for fixed fps recording
2086             float frameDropTolerance) throws Exception {
2087         File outFile = new File(mOutMediaFileName);
2088         assertTrue("No video is recorded", outFile.exists());
2089         float maxFrameDuration = expectedFrameDurationMinMs * (1.0f + FRAMEDURATION_MARGIN);
2090         if (expectedFrameDurationMaxMs > 0.f) {
2091             maxFrameDuration = expectedFrameDurationMaxMs * (1.0f + FRAMEDURATION_MARGIN);
2092         }
2093 
2094         if (expectedDurationMaxMs == 0.f) {
2095             expectedDurationMaxMs = expectedDurationMinMs;
2096         }
2097 
2098         MediaExtractor extractor = new MediaExtractor();
2099         try {
2100             extractor.setDataSource(mOutMediaFileName);
2101             long durationUs = 0;
2102             int width = -1, height = -1;
2103             int numTracks = extractor.getTrackCount();
2104             int selectedTrack = -1;
2105             final String VIDEO_MIME_TYPE = "video";
2106             for (int i = 0; i < numTracks; i++) {
2107                 MediaFormat format = extractor.getTrackFormat(i);
2108                 String mime = format.getString(MediaFormat.KEY_MIME);
2109                 if (mime.contains(VIDEO_MIME_TYPE)) {
2110                     Log.i(TAG, "video format is: " + format.toString());
2111                     durationUs = format.getLong(MediaFormat.KEY_DURATION);
2112                     width = format.getInteger(MediaFormat.KEY_WIDTH);
2113                     height = format.getInteger(MediaFormat.KEY_HEIGHT);
2114                     selectedTrack = i;
2115                     extractor.selectTrack(i);
2116                     break;
2117                 }
2118             }
2119             if (selectedTrack < 0) {
2120                 throw new AssertionFailedError(
2121                         "Cannot find video track!");
2122             }
2123 
2124             Size videoSz = new Size(width, height);
2125             assertTrue("Video size doesn't match, expected " + sz.toString() +
2126                     " got " + videoSz.toString(), videoSz.equals(sz));
2127             float duration = (float) (durationUs / 1000);
2128             if (VERBOSE) {
2129                 Log.v(TAG, String.format("Video duration: recorded %fms, expected [%f,%f]ms",
2130                                          duration, expectedDurationMinMs, expectedDurationMaxMs));
2131             }
2132 
2133             // Do rest of validation only for better-than-LEGACY devices
2134             if (mStaticInfo.isHardwareLevelLegacy()) return;
2135 
2136             // TODO: Don't skip this one for video snapshot on LEGACY
2137             assertTrue(String.format(
2138                     "Camera %s: Video duration doesn't match: recorded %fms, expected [%f,%f]ms.",
2139                     mCamera.getId(), duration,
2140                     expectedDurationMinMs * (1.f - DURATION_MARGIN),
2141                     expectedDurationMaxMs * (1.f + DURATION_MARGIN)),
2142                     duration > expectedDurationMinMs * (1.f - DURATION_MARGIN) &&
2143                             duration < expectedDurationMaxMs * (1.f + DURATION_MARGIN));
2144 
2145             // Check for framedrop
2146             long lastSampleUs = 0;
2147             int frameDropCount = 0;
2148             int expectedFrameCount = (int) (expectedDurationMinMs / expectedFrameDurationMinMs);
2149             ArrayList<Long> timestamps = new ArrayList<Long>(expectedFrameCount);
2150             while (true) {
2151                 timestamps.add(extractor.getSampleTime());
2152                 if (!extractor.advance()) {
2153                     break;
2154                 }
2155             }
2156             Collections.sort(timestamps);
2157             long prevSampleUs = timestamps.get(0);
2158             for (int i = 1; i < timestamps.size(); i++) {
2159                 long currentSampleUs = timestamps.get(i);
2160                 float frameDurationMs = (float) (currentSampleUs - prevSampleUs) / 1000;
2161                 if (frameDurationMs > maxFrameDuration) {
2162                     Log.w(TAG, String.format(
2163                         "Frame drop at %d: expectation %f, observed %f",
2164                         i, expectedFrameDurationMinMs, frameDurationMs));
2165                     frameDropCount++;
2166                 }
2167                 prevSampleUs = currentSampleUs;
2168             }
2169             float frameDropRate = 100.f * frameDropCount / timestamps.size();
2170             Log.i(TAG, String.format("Frame drop rate %d/%d (%f%%)",
2171                 frameDropCount, timestamps.size(), frameDropRate));
2172             assertTrue(String.format(
2173                     "Camera %s: Video frame drop rate too high: %f%%, tolerance %f%%. " +
2174                     "Video size: %s, expectedDuration [%f,%f], expectedFrameDuration %f, " +
2175                     "frameDropCnt %d, frameCount %d",
2176                     mCamera.getId(), frameDropRate, frameDropTolerance,
2177                     sz.toString(), expectedDurationMinMs, expectedDurationMaxMs,
2178                     expectedFrameDurationMinMs, frameDropCount, timestamps.size()),
2179                     frameDropRate < frameDropTolerance);
2180         } finally {
2181             extractor.release();
2182             if (!DEBUG_DUMP) {
2183                 outFile.delete();
2184             }
2185         }
2186     }
2187 
2188     /**
2189      * Validate video snapshot capture image object validity and test.
2190      *
2191      * <p> Check for size, format and jpeg decoding</p>
2192      *
2193      * @param image The JPEG image to be verified.
2194      * @param size The JPEG capture size to be verified against.
2195      */
2196     private void validateVideoSnapshotCapture(Image image, Size size) {
2197         CameraTestUtils.validateImage(image, size.getWidth(), size.getHeight(),
2198                 ImageFormat.JPEG, /*filePath*/null);
2199     }
2200 
2201     /**
2202      * Validate if video snapshot causes frame drop.
2203      * Here frame drop is defined as frame duration >= 2 * expected frame duration.
2204      * Return the estimated number of frames dropped during video snapshot
2205      */
2206     private int validateFrameDropAroundVideoSnapshot(
2207             SimpleCaptureCallback resultListener, long imageTimeStamp) {
2208         double expectedDurationMs = 1000.0 / mVideoFrameRate;
2209         CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2210         long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP);
2211         while (resultListener.hasMoreResults()) {
2212             CaptureResult currentResult =
2213                     resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2214             long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP);
2215             if (currentTS == imageTimeStamp) {
2216                 // validate the timestamp before and after, then return
2217                 CaptureResult nextResult =
2218                         resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2219                 long nextTS = getValueNotNull(nextResult, CaptureResult.SENSOR_TIMESTAMP);
2220                 double durationMs = (currentTS - prevTS) / 1000000.0;
2221                 int totalFramesDropped = 0;
2222 
2223                 // Snapshots in legacy mode pause the preview briefly.  Skip the duration
2224                 // requirements for legacy mode unless this is fixed.
2225                 if (!mStaticInfo.isHardwareLevelLegacy()) {
2226                     mCollector.expectTrue(
2227                             String.format(
2228                                     "Video %dx%d Frame drop detected before video snapshot: " +
2229                                             "duration %.2fms (expected %.2fms)",
2230                                     mVideoSize.getWidth(), mVideoSize.getHeight(),
2231                                     durationMs, expectedDurationMs
2232                             ),
2233                             durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED)
2234                     );
2235                     // Log a warning is there is any frame drop detected.
2236                     if (durationMs >= expectedDurationMs * 2) {
2237                         Log.w(TAG, String.format(
2238                                 "Video %dx%d Frame drop detected before video snapshot: " +
2239                                         "duration %.2fms (expected %.2fms)",
2240                                 mVideoSize.getWidth(), mVideoSize.getHeight(),
2241                                 durationMs, expectedDurationMs
2242                         ));
2243                     }
2244 
2245                     durationMs = (nextTS - currentTS) / 1000000.0;
2246                     mCollector.expectTrue(
2247                             String.format(
2248                                     "Video %dx%d Frame drop detected after video snapshot: " +
2249                                             "duration %.2fms (expected %.2fms)",
2250                                     mVideoSize.getWidth(), mVideoSize.getHeight(),
2251                                     durationMs, expectedDurationMs
2252                             ),
2253                             durationMs <= (expectedDurationMs * MAX_NUM_FRAME_DROP_INTERVAL_ALLOWED)
2254                     );
2255                     // Log a warning is there is any frame drop detected.
2256                     if (durationMs >= expectedDurationMs * 2) {
2257                         Log.w(TAG, String.format(
2258                                 "Video %dx%d Frame drop detected after video snapshot: " +
2259                                         "duration %fms (expected %fms)",
2260                                 mVideoSize.getWidth(), mVideoSize.getHeight(),
2261                                 durationMs, expectedDurationMs
2262                         ));
2263                     }
2264 
2265                     double totalDurationMs = (nextTS - prevTS) / 1000000.0;
2266                     // Minus 2 for the expected 2 frames interval
2267                     totalFramesDropped = (int) (totalDurationMs / expectedDurationMs) - 2;
2268                     if (totalFramesDropped < 0) {
2269                         Log.w(TAG, "totalFrameDropped is " + totalFramesDropped +
2270                                 ". Video frame rate might be too fast.");
2271                     }
2272                     totalFramesDropped = Math.max(0, totalFramesDropped);
2273                 }
2274                 return totalFramesDropped;
2275             }
2276             prevTS = currentTS;
2277         }
2278         throw new AssertionFailedError(
2279                 "Video snapshot timestamp does not match any of capture results!");
2280     }
2281 
2282     /**
2283      * Validate frame jittering from the input simple listener's buffered results
2284      */
2285     private void validateJittering(SimpleCaptureCallback resultListener) {
2286         double expectedDurationMs = 1000.0 / mVideoFrameRate;
2287         CaptureResult prevResult = resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2288         long prevTS = getValueNotNull(prevResult, CaptureResult.SENSOR_TIMESTAMP);
2289         while (resultListener.hasMoreResults()) {
2290             CaptureResult currentResult =
2291                     resultListener.getCaptureResult(WAIT_FOR_RESULT_TIMEOUT_MS);
2292             long currentTS = getValueNotNull(currentResult, CaptureResult.SENSOR_TIMESTAMP);
2293             double durationMs = (currentTS - prevTS) / 1000000.0;
2294             double durationError = Math.abs(durationMs - expectedDurationMs);
2295             long frameNumber = currentResult.getFrameNumber();
2296             mCollector.expectTrue(
2297                     String.format(
2298                             "Resolution %dx%d Frame %d: jittering (%.2fms) exceeds bound [%.2fms,%.2fms]",
2299                             mVideoSize.getWidth(), mVideoSize.getHeight(),
2300                             frameNumber, durationMs,
2301                             expectedDurationMs - FRAME_DURATION_ERROR_TOLERANCE_MS,
2302                             expectedDurationMs + FRAME_DURATION_ERROR_TOLERANCE_MS),
2303                     durationError <= FRAME_DURATION_ERROR_TOLERANCE_MS);
2304             prevTS = currentTS;
2305         }
2306     }
2307 
2308     /**
2309      * Calculate a video bit rate based on the size. The bit rate is scaled
2310      * based on ratio of video size to 1080p size.
2311      */
2312     private int getVideoBitRate(Size sz) {
2313         int rate = BIT_RATE_1080P;
2314         float scaleFactor = sz.getHeight() * sz.getWidth() / (float)(1920 * 1080);
2315         rate = (int)(rate * scaleFactor);
2316 
2317         // Clamp to the MIN, MAX range.
2318         return Math.max(BIT_RATE_MIN, Math.min(BIT_RATE_MAX, rate));
2319     }
2320 
2321     /**
2322      * Check if the encoder and camera are able to support this size and frame rate.
2323      * Assume the video compression format is AVC.
2324      */
2325     private boolean isSupported(Size sz, int captureRate, int encodingRate) throws Exception {
2326         // Check camera capability.
2327         if (!isSupportedByCamera(sz, captureRate)) {
2328             return false;
2329         }
2330 
2331         // Check encode capability.
2332         if (!isSupportedByAVCEncoder(sz, encodingRate)){
2333             return false;
2334         }
2335 
2336         if(VERBOSE) {
2337             Log.v(TAG, "Both encoder and camera support " + sz.toString() + "@" + encodingRate + "@"
2338                     + getVideoBitRate(sz) / 1000 + "Kbps");
2339         }
2340 
2341         return true;
2342     }
2343 
2344     private boolean isSupportedByCamera(Size sz, int frameRate) {
2345         // Check if camera can support this sz and frame rate combination.
2346         StreamConfigurationMap config = mStaticInfo.
2347                 getValueFromKeyNonNull(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
2348 
2349         long minDuration = config.getOutputMinFrameDuration(MediaRecorder.class, sz);
2350         if (minDuration == 0) {
2351             return false;
2352         }
2353 
2354         int maxFrameRate = (int) (1e9f / minDuration);
2355         return maxFrameRate >= frameRate;
2356     }
2357 
2358     /**
2359      * Check if encoder can support this size and frame rate combination by querying
2360      * MediaCodec capability. Check is based on size and frame rate. Ignore the bit rate
2361      * as the bit rates targeted in this test are well below the bit rate max value specified
2362      * by AVC specification for certain level.
2363      */
2364     private static boolean isSupportedByAVCEncoder(Size sz, int frameRate) {
2365         MediaFormat format = MediaFormat.createVideoFormat(
2366                 MediaFormat.MIMETYPE_VIDEO_AVC, sz.getWidth(), sz.getHeight());
2367         format.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
2368         MediaCodecList mcl = new MediaCodecList(MediaCodecList.REGULAR_CODECS);
2369         return mcl.findEncoderForFormat(format) != null;
2370     }
2371 
2372     private static class ImageWriterQueuer implements ImageReader.OnImageAvailableListener {
2373         public ImageWriterQueuer(ImageWriter writer) {
2374             mWriter = writer;
2375         }
2376 
2377         public void resetInvalidSurfaceFlag() {
2378             synchronized (mLock) {
2379                 mExpectInvalidSurface = false;
2380             }
2381         }
2382 
2383         // Indicate that the writer surface is about to get released
2384         // and become invalid.
2385         public void expectInvalidSurface() {
2386             // If we sync on 'mLock', we risk a possible deadlock
2387             // during 'mWriter.queueInputImage(image)' which is
2388             // called while the lock is held.
2389             mExpectInvalidSurface = true;
2390         }
2391 
2392         @Override
2393         public void onImageAvailable(ImageReader reader) {
2394             Image image = null;
2395             try {
2396                 image = reader.acquireNextImage();
2397             } finally {
2398                 synchronized (mLock) {
2399                     if (image != null && mWriter != null) {
2400                         try {
2401                             mWriter.queueInputImage(image);
2402                             mQueuedCount++;
2403                         } catch (IllegalStateException e) {
2404                             // Per API documentation ISE are possible
2405                             // in case the writer surface is not valid.
2406                             // Re-throw in case we have some other
2407                             // unexpected ISE.
2408                             if (mExpectInvalidSurface) {
2409                                 Log.d(TAG, "Invalid writer surface");
2410                                 image.close();
2411                             } else {
2412                                 throw e;
2413                             }
2414                         }
2415                     } else if (image != null) {
2416                         image.close();
2417                     }
2418                 }
2419             }
2420         }
2421 
2422         public int getQueuedCount() {
2423             synchronized (mLock) {
2424                 return mQueuedCount;
2425             }
2426         }
2427 
2428         public void close() {
2429             synchronized (mLock) {
2430                 mWriter = null;
2431             }
2432         }
2433 
2434         private Object      mLock = new Object();
2435         private ImageWriter mWriter = null;
2436         private int         mQueuedCount = 0;
2437         private boolean     mExpectInvalidSurface = false;
2438     }
2439 }
2440