1 /*
2  * Copyright 2020 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.cts;
18 
19 import static android.hardware.camera2.cts.helpers.AssertHelpers.assertArrayContains;
20 import static org.mockito.ArgumentMatchers.anyBoolean;
21 import static org.mockito.Matchers.any;
22 import static org.mockito.Matchers.anyInt;
23 import static org.mockito.Matchers.anyLong;
24 import static org.mockito.Matchers.eq;
25 import static org.mockito.Mockito.atLeastOnce;
26 import static org.mockito.Mockito.mock;
27 import static org.mockito.Mockito.reset;
28 import static org.mockito.Mockito.timeout;
29 import static org.mockito.Mockito.times;
30 import static org.mockito.Mockito.verify;
31 
32 import com.android.compatibility.common.util.DeviceReportLog;
33 import com.android.compatibility.common.util.ResultType;
34 import com.android.compatibility.common.util.ResultUnit;
35 import com.android.compatibility.common.util.Stat;
36 import com.android.internal.camera.flags.Flags;
37 import com.android.ex.camera2.blocking.BlockingSessionCallback;
38 import com.android.ex.camera2.blocking.BlockingExtensionSessionCallback;
39 import com.android.ex.camera2.blocking.BlockingStateCallback;
40 import com.android.ex.camera2.exceptions.TimeoutRuntimeException;
41 import com.android.ex.camera2.pos.AutoFocusStateMachine;
42 
43 import android.graphics.ColorSpace;
44 import android.graphics.ImageFormat;
45 import android.graphics.PointF;
46 import android.graphics.Rect;
47 import android.graphics.SurfaceTexture;
48 import android.hardware.HardwareBuffer;
49 import android.hardware.camera2.CameraCaptureSession;
50 import android.hardware.camera2.CameraCharacteristics;
51 import android.hardware.camera2.CameraDevice;
52 import android.hardware.camera2.CameraExtensionCharacteristics;
53 import android.hardware.camera2.CameraExtensionSession;
54 import android.hardware.camera2.CameraMetadata;
55 import android.hardware.camera2.CaptureRequest;
56 import android.hardware.camera2.CaptureResult;
57 import android.hardware.camera2.ExtensionCaptureRequest;
58 import android.hardware.camera2.ExtensionCaptureResult;
59 import android.hardware.camera2.TotalCaptureResult;
60 import android.hardware.camera2.cts.helpers.CameraErrorCollector;
61 import android.hardware.camera2.cts.helpers.StaticMetadata;
62 import android.hardware.camera2.cts.testcases.Camera2AndroidTestRule;
63 import android.hardware.camera2.params.ColorSpaceProfiles;
64 import android.hardware.camera2.params.DynamicRangeProfiles;
65 import android.hardware.camera2.params.ExtensionSessionConfiguration;
66 import android.hardware.camera2.params.MeteringRectangle;
67 import android.hardware.camera2.params.OutputConfiguration;
68 import android.hardware.camera2.params.SessionConfiguration;
69 import android.media.ExifInterface;
70 import android.media.Image;
71 import android.media.ImageReader;
72 import android.os.Build;
73 import android.os.SystemClock;
74 import android.platform.test.annotations.RequiresFlagsEnabled;
75 import android.util.Range;
76 import android.util.Size;
77 
78 import static android.hardware.camera2.cts.CameraTestUtils.*;
79 import static android.hardware.cts.helpers.CameraUtils.*;
80 
81 import android.util.Log;
82 import android.view.Surface;
83 import android.view.TextureView;
84 
85 import androidx.test.platform.app.InstrumentationRegistry;
86 import androidx.test.rule.ActivityTestRule;
87 
88 import org.junit.Rule;
89 import org.junit.Test;
90 import org.junit.runner.RunWith;
91 import org.junit.runners.Parameterized;
92 
93 import java.io.IOException;
94 import java.util.ArrayList;
95 import java.util.Arrays;
96 import java.util.HashSet;
97 import java.util.List;
98 import java.util.Set;
99 import java.util.stream.Collectors;
100 
101 @RunWith(Parameterized.class)
102 public class CameraExtensionSessionTest extends Camera2ParameterizedTestCase {
103     private static final String TAG = "CameraExtensionSessionTest";
104     private static final boolean VERBOSE = Log.isLoggable(TAG, Log.VERBOSE);
105     private static final long WAIT_FOR_COMMAND_TO_COMPLETE_MS = 5000;
106     private static final long REPEATING_REQUEST_TIMEOUT_MS = 5000;
107     private static final int MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS = 10000;
108     private static final float ZOOM_ERROR_MARGIN = 0.05f;
109 
110     private static final int WAIT_FOR_FOCUS_DONE_TIMEOUT_MS = 6000;
111 
112     private static final CaptureRequest.Key[] FOCUS_CAPTURE_REQUEST_SET = {
113             CaptureRequest.CONTROL_AF_MODE,
114             CaptureRequest.CONTROL_AF_REGIONS,
115             CaptureRequest.CONTROL_AF_TRIGGER};
116     private static final CaptureResult.Key[] FOCUS_CAPTURE_RESULT_SET = {
117             CaptureResult.CONTROL_AF_MODE,
118             CaptureResult.CONTROL_AF_REGIONS,
119             CaptureResult.CONTROL_AF_TRIGGER,
120             CaptureResult.CONTROL_AF_STATE};
121     private static final CaptureRequest.Key[] ZOOM_CAPTURE_REQUEST_SET = {
122             CaptureRequest.CONTROL_ZOOM_RATIO};
123     private static final CaptureResult.Key[] ZOOM_CAPTURE_RESULT_SET = {
124             CaptureResult.CONTROL_ZOOM_RATIO};
125     private CaptureRequest.Key[] EYES_FREE_AUTO_ZOOM_REQUEST_SET = new CaptureRequest.Key[0];
126     private CaptureResult.Key[] EYES_FREE_AUTO_ZOOM_RESULT_SET = new CaptureResult.Key[0];
127     private CaptureRequest.Key[] EYES_FREE_REQUEST_SET = new CaptureRequest.Key[0];
128     private CaptureResult.Key[] EYES_FREE_RESULT_SET = new CaptureResult.Key[0];
129 
130     private static final boolean EFV_API_SUPPORTED =
131             Build.VERSION.SDK_INT > Build.VERSION_CODES.UPSIDE_DOWN_CAKE && Flags.concertModeApi();
132 
133     private SurfaceTexture mSurfaceTexture = null;
134     private Camera2AndroidTestRule mTestRule = null;
135     private CameraErrorCollector mCollector =  null;
136 
137     private DeviceReportLog mReportLog;
138 
139     @Override
setUp()140     public void setUp() throws Exception {
141         super.setUp();
142         mTestRule = new Camera2AndroidTestRule(mContext);
143         mTestRule.before();
144         mCollector = new CameraErrorCollector();
145         if (EFV_API_SUPPORTED) {
146             EYES_FREE_AUTO_ZOOM_REQUEST_SET = new CaptureRequest.Key[] {
147                     ExtensionCaptureRequest.EFV_AUTO_ZOOM,
148                     ExtensionCaptureRequest.EFV_MAX_PADDING_ZOOM_FACTOR};
149             EYES_FREE_AUTO_ZOOM_RESULT_SET = new CaptureResult.Key[] {
150                     ExtensionCaptureResult.EFV_AUTO_ZOOM,
151                     ExtensionCaptureResult.EFV_AUTO_ZOOM_PADDING_REGION};
152             EYES_FREE_REQUEST_SET = new CaptureRequest.Key[] {
153                     ExtensionCaptureRequest.EFV_PADDING_ZOOM_FACTOR,
154                     ExtensionCaptureRequest.EFV_STABILIZATION_MODE,
155                     ExtensionCaptureRequest.EFV_TRANSLATE_VIEWPORT,
156                     ExtensionCaptureRequest.EFV_ROTATE_VIEWPORT};
157             EYES_FREE_RESULT_SET = new CaptureResult.Key[] {
158                     ExtensionCaptureResult.EFV_PADDING_REGION,
159                     ExtensionCaptureResult.EFV_TARGET_COORDINATES,
160                     ExtensionCaptureResult.EFV_PADDING_ZOOM_FACTOR,
161                     ExtensionCaptureResult.EFV_STABILIZATION_MODE,
162                     ExtensionCaptureResult.EFV_ROTATE_VIEWPORT,
163                     ExtensionCaptureResult.EFV_TRANSLATE_VIEWPORT};
164         }
165     }
166 
167     @Override
tearDown()168     public void tearDown() throws Exception {
169         if (mTestRule != null) {
170             mTestRule.after();
171         }
172         if (mSurfaceTexture != null) {
173             mSurfaceTexture.release();
174             mSurfaceTexture = null;
175         }
176         if (mCollector != null) {
177             try {
178                 mCollector.verify();
179             } catch (Throwable e) {
180                 throw new Exception(e.getMessage());
181             }
182         }
183         super.tearDown();
184     }
185 
186     @Rule
187     public ActivityTestRule<CameraExtensionTestActivity> mActivityRule =
188             new ActivityTestRule<>(CameraExtensionTestActivity.class);
189 
updatePreviewSurfaceTexture()190     private void updatePreviewSurfaceTexture() {
191         if (mSurfaceTexture != null) {
192             return;
193         }
194 
195         TextureView textureView = mActivityRule.getActivity().getTextureView();
196         mSurfaceTexture = getAvailableSurfaceTexture(WAIT_FOR_COMMAND_TO_COMPLETE_MS, textureView);
197         assertNotNull("Failed to acquire valid preview surface texture!", mSurfaceTexture);
198     }
199 
200     // Verify that camera extension sessions can be created and closed as expected.
201     @Test
testBasicExtensionLifecycle()202     public void testBasicExtensionLifecycle() throws Exception {
203         for (String id : getCameraIdsUnderTest()) {
204             StaticMetadata staticMeta =
205                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
206             if (!staticMeta.isColorOutputSupported()) {
207                 continue;
208             }
209             updatePreviewSurfaceTexture();
210             CameraExtensionCharacteristics extensionChars =
211                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
212             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
213             for (Integer extension : supportedExtensions) {
214                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
215                         mSurfaceTexture.getClass());
216                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
217                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
218                 OutputConfiguration outputConfig = new OutputConfiguration(
219                         new Surface(mSurfaceTexture));
220                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
221                 outputConfigs.add(outputConfig);
222 
223                 BlockingExtensionSessionCallback sessionListener =
224                         new BlockingExtensionSessionCallback(
225                                 mock(CameraExtensionSession.StateCallback.class));
226                 ExtensionSessionConfiguration configuration =
227                         new ExtensionSessionConfiguration(extension, outputConfigs,
228                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
229 
230                 try {
231                     mTestRule.openDevice(id);
232                     CameraDevice camera = mTestRule.getCamera();
233                     camera.createExtensionSession(configuration);
234                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
235                             SESSION_CONFIGURE_TIMEOUT_MS);
236 
237                     extensionSession.close();
238                     sessionListener.getStateWaiter().waitForState(
239                             BlockingExtensionSessionCallback.SESSION_CLOSED,
240                             SESSION_CLOSE_TIMEOUT_MS);
241                 } finally {
242                     mTestRule.closeDevice(id);
243                 }
244             }
245         }
246     }
247 
248     // Verify that regular camera sessions close as expected after creating a camera extension
249     // session.
250     @Test
testCloseCaptureSession()251     public void testCloseCaptureSession() throws Exception {
252         for (String id : getCameraIdsUnderTest()) {
253             StaticMetadata staticMeta =
254                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
255             if (!staticMeta.isColorOutputSupported()) {
256                 continue;
257             }
258             updatePreviewSurfaceTexture();
259             CameraExtensionCharacteristics extensionChars =
260                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
261             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
262             for (Integer extension : supportedExtensions) {
263                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
264                         mSurfaceTexture.getClass());
265                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
266                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
267                 Surface repeatingSurface = new Surface(mSurfaceTexture);
268                 OutputConfiguration textureOutput = new OutputConfiguration(repeatingSurface);
269                 List<OutputConfiguration> outputs = new ArrayList<>();
270                 outputs.add(textureOutput);
271                 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback(
272                         mock(CameraCaptureSession.StateCallback.class));
273                 SessionConfiguration regularConfiguration = new SessionConfiguration(
274                         SessionConfiguration.SESSION_REGULAR, outputs,
275                         new HandlerExecutor(mTestRule.getHandler()), regularSessionListener);
276 
277                 BlockingExtensionSessionCallback sessionListener =
278                         new BlockingExtensionSessionCallback(mock(
279                                 CameraExtensionSession.StateCallback.class));
280                 ExtensionSessionConfiguration configuration =
281                         new ExtensionSessionConfiguration(extension, outputs,
282                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
283 
284                 try {
285                     mTestRule.openDevice(id);
286                     mTestRule.getCamera().createCaptureSession(regularConfiguration);
287 
288                     CameraCaptureSession session =
289                             regularSessionListener
290                                     .waitAndGetSession(SESSION_CONFIGURE_TIMEOUT_MS);
291                     assertNotNull(session);
292 
293                     CameraDevice camera = mTestRule.getCamera();
294                     camera.createExtensionSession(configuration);
295                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
296                             SESSION_CONFIGURE_TIMEOUT_MS);
297                     assertNotNull(extensionSession);
298 
299                     regularSessionListener.getStateWaiter().waitForState(
300                             BlockingExtensionSessionCallback.SESSION_CLOSED,
301                             SESSION_CLOSE_TIMEOUT_MS);
302 
303                     extensionSession.close();
304                     sessionListener.getStateWaiter().waitForState(
305                             BlockingExtensionSessionCallback.SESSION_CLOSED,
306                             SESSION_CLOSE_TIMEOUT_MS);
307                 } finally {
308                     mTestRule.closeDevice(id);
309                 }
310             }
311         }
312     }
313 
314     // Verify that camera extension sessions close as expected when creating a regular capture
315     // session.
316     @Test
testCloseExtensionSession()317     public void testCloseExtensionSession() throws Exception {
318         for (String id : getCameraIdsUnderTest()) {
319             StaticMetadata staticMeta =
320                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
321             if (!staticMeta.isColorOutputSupported()) {
322                 continue;
323             }
324             updatePreviewSurfaceTexture();
325             CameraExtensionCharacteristics extensionChars =
326                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
327             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
328             for (Integer extension : supportedExtensions) {
329                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
330                         mSurfaceTexture.getClass());
331                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
332                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
333                 Surface surface = new Surface(mSurfaceTexture);
334                 OutputConfiguration textureOutput = new OutputConfiguration(surface);
335                 List<OutputConfiguration> outputs = new ArrayList<>();
336                 outputs.add(textureOutput);
337                 BlockingSessionCallback regularSessionListener = new BlockingSessionCallback(
338                         mock(CameraCaptureSession.StateCallback.class));
339                 SessionConfiguration regularConfiguration = new SessionConfiguration(
340                         SessionConfiguration.SESSION_REGULAR, outputs,
341                         new HandlerExecutor(mTestRule.getHandler()), regularSessionListener);
342 
343                 BlockingExtensionSessionCallback sessionListener =
344                         new BlockingExtensionSessionCallback(mock(
345                                 CameraExtensionSession.StateCallback.class));
346                 ExtensionSessionConfiguration configuration =
347                         new ExtensionSessionConfiguration(extension, outputs,
348                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
349 
350                 try {
351                     mTestRule.openDevice(id);
352                     CameraDevice camera = mTestRule.getCamera();
353                     camera.createExtensionSession(configuration);
354                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
355                             SESSION_CONFIGURE_TIMEOUT_MS);
356                     assertNotNull(extensionSession);
357 
358                     mTestRule.getCamera().createCaptureSession(regularConfiguration);
359                     sessionListener.getStateWaiter().waitForState(
360                             BlockingExtensionSessionCallback.SESSION_CLOSED,
361                             SESSION_CLOSE_TIMEOUT_MS);
362 
363                     CameraCaptureSession session =
364                             regularSessionListener.waitAndGetSession(
365                                     SESSION_CONFIGURE_TIMEOUT_MS);
366                     session.close();
367                     regularSessionListener.getStateWaiter().waitForState(
368                             BlockingSessionCallback.SESSION_CLOSED, SESSION_CLOSE_TIMEOUT_MS);
369                 } finally {
370                     mTestRule.closeDevice(id);
371                 }
372             }
373         }
374     }
375 
376     // Verify camera device query
377     @Test
testGetDevice()378     public void testGetDevice() throws Exception {
379         for (String id : getCameraIdsUnderTest()) {
380             StaticMetadata staticMeta =
381                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
382             if (!staticMeta.isColorOutputSupported()) {
383                 continue;
384             }
385             updatePreviewSurfaceTexture();
386             CameraExtensionCharacteristics extensionChars =
387                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
388             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
389             for (Integer extension : supportedExtensions) {
390                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
391                         mSurfaceTexture.getClass());
392                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
393                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
394                 OutputConfiguration privateOutput = new OutputConfiguration(
395                         new Surface(mSurfaceTexture));
396                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
397                 outputConfigs.add(privateOutput);
398 
399                 BlockingExtensionSessionCallback sessionListener =
400                         new BlockingExtensionSessionCallback(
401                                 mock(CameraExtensionSession.StateCallback.class));
402                 ExtensionSessionConfiguration configuration =
403                         new ExtensionSessionConfiguration(extension, outputConfigs,
404                                 new HandlerExecutor(mTestRule.getHandler()), sessionListener);
405 
406                 try {
407                     mTestRule.openDevice(id);
408                     CameraDevice camera = mTestRule.getCamera();
409                     camera.createExtensionSession(configuration);
410                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
411                             SESSION_CONFIGURE_TIMEOUT_MS);
412 
413                     assertEquals("Unexpected/Invalid camera device", mTestRule.getCamera(),
414                             extensionSession.getDevice());
415                 } finally {
416                     mTestRule.closeDevice(id);
417                 }
418 
419                 try {
420                     sessionListener.getStateWaiter().waitForState(
421                             BlockingExtensionSessionCallback.SESSION_CLOSED,
422                             SESSION_CLOSE_TIMEOUT_MS);
423                     fail("should get TimeoutRuntimeException due to previously closed camera "
424                             + "device");
425                 } catch (TimeoutRuntimeException e) {
426                     // Expected, per API spec we should not receive any further session callbacks
427                     // besides the device state 'onClosed' callback.
428                 }
429             }
430         }
431     }
432 
433     // Test case for repeating/stopRepeating on all supported extensions and expected state/capture
434     // callbacks.
435     @Test
testRepeatingCapture()436     public void testRepeatingCapture() throws Exception {
437         for (String id : getCameraIdsUnderTest()) {
438             StaticMetadata staticMeta =
439                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
440             if (!staticMeta.isColorOutputSupported()) {
441                 continue;
442             }
443             updatePreviewSurfaceTexture();
444             CameraExtensionCharacteristics extensionChars =
445                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
446             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
447             for (Integer extension : supportedExtensions) {
448                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
449                         mSurfaceTexture.getClass());
450                 Size maxSize =
451                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
452                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
453                         maxSize.getHeight());
454                 Surface texturedSurface = new Surface(mSurfaceTexture);
455 
456                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
457                 outputConfigs.add(new OutputConfiguration(texturedSurface));
458 
459                 BlockingExtensionSessionCallback sessionListener =
460                         new BlockingExtensionSessionCallback(mock(
461                                 CameraExtensionSession.StateCallback.class));
462                 ExtensionSessionConfiguration configuration =
463                         new ExtensionSessionConfiguration(extension, outputConfigs,
464                                 new HandlerExecutor(mTestRule.getHandler()),
465                                 sessionListener);
466 
467                 boolean captureResultsSupported =
468                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
469 
470                 try {
471                     mTestRule.openDevice(id);
472                     CameraDevice camera = mTestRule.getCamera();
473                     camera.createExtensionSession(configuration);
474                     CameraExtensionSession extensionSession =
475                             sessionListener.waitAndGetSession(
476                                     SESSION_CONFIGURE_TIMEOUT_MS);
477                     assertNotNull(extensionSession);
478 
479                     CaptureRequest.Builder captureBuilder =
480                             mTestRule.getCamera().createCaptureRequest(
481                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
482                     captureBuilder.addTarget(texturedSurface);
483                     CameraExtensionSession.ExtensionCaptureCallback captureCallbackMock =
484                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
485                     SimpleCaptureCallback simpleCaptureCallback =
486                             new SimpleCaptureCallback(extension, captureCallbackMock,
487                                     extensionChars.getAvailableCaptureResultKeys(extension),
488                                     mCollector);
489                     CaptureRequest request = captureBuilder.build();
490                     int sequenceId = extensionSession.setRepeatingRequest(request,
491                             new HandlerExecutor(mTestRule.getHandler()), simpleCaptureCallback);
492 
493                     verify(captureCallbackMock,
494                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
495                             .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
496                     verify(captureCallbackMock,
497                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
498                             .onCaptureProcessStarted(extensionSession, request);
499                     if (captureResultsSupported) {
500                         verify(captureCallbackMock,
501                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
502                                 .onCaptureResultAvailable(eq(extensionSession), eq(request),
503                                         any(TotalCaptureResult.class));
504                     }
505 
506                     extensionSession.stopRepeating();
507 
508                     verify(captureCallbackMock,
509                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
510                             .onCaptureSequenceCompleted(extensionSession, sequenceId);
511 
512                     verify(captureCallbackMock, times(0))
513                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
514                                     anyInt());
515 
516                     extensionSession.close();
517 
518                     sessionListener.getStateWaiter().waitForState(
519                             BlockingExtensionSessionCallback.SESSION_CLOSED,
520                             SESSION_CLOSE_TIMEOUT_MS);
521 
522                     assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" +
523                                     " callbacks must be greater or equal than the number of calls" +
524                                     " to onCaptureStarted!",
525                             simpleCaptureCallback.getTotalFramesArrived() +
526                                     simpleCaptureCallback.getTotalFramesFailed() >=
527                                     simpleCaptureCallback.getTotalFramesStarted());
528                     assertTrue(String.format("The last repeating request surface timestamp " +
529                                     "%d must be less than or equal to the last " +
530                                     "onCaptureStarted " +
531                                     "timestamp %d", mSurfaceTexture.getTimestamp(),
532                             simpleCaptureCallback.getLastTimestamp()),
533                             mSurfaceTexture.getTimestamp() <=
534                                     simpleCaptureCallback.getLastTimestamp());
535                 } finally {
536                     mTestRule.closeDevice(id);
537                     texturedSurface.release();
538                 }
539             }
540         }
541     }
542 
543     // Test for postview of still capture on all supported extensions
544     @Test
testPostviewAndCapture()545     public void testPostviewAndCapture() throws Exception {
546         final int IMAGE_COUNT = 10;
547         final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = {
548                 ImageFormat.YUV_420_888,
549                 ImageFormat.JPEG,
550                 ImageFormat.JPEG_R
551         };
552         for (String id : getCameraIdsUnderTest()) {
553             StaticMetadata staticMeta =
554                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
555             if (!staticMeta.isColorOutputSupported()) {
556                 continue;
557             }
558             updatePreviewSurfaceTexture();
559             CameraExtensionCharacteristics extensionChars =
560                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
561             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
562             for (Integer extension : supportedExtensions) {
563                 if (!extensionChars.isPostviewAvailable(extension)) {
564                     continue;
565                 }
566 
567                 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
568                     boolean captureProgressSupported =
569                             extensionChars.isCaptureProcessProgressAvailable(extension);
570 
571                     List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
572                             captureFormat);
573                     if (extensionSizes.isEmpty()) {
574                         continue;
575                     }
576 
577                     Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
578 
579                     for (int postviewFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
580                         List<Size> postviewSizes = extensionChars.getPostviewSupportedSizes(
581                                 extension, maxSize, postviewFormat);
582                         if (postviewSizes.isEmpty()) {
583                             continue;
584                         }
585 
586                         SimpleImageReaderListener imageListener = new SimpleImageReaderListener(
587                                 false, 1);
588                         ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
589                                 captureFormat, /*maxImages*/ 1, imageListener,
590                                 mTestRule.getHandler());
591                         Surface imageReaderSurface = extensionImageReader.getSurface();
592                         OutputConfiguration readerOutput =
593                                 new OutputConfiguration(imageReaderSurface);
594                         List<OutputConfiguration> outputConfigs = new ArrayList<>();
595                         outputConfigs.add(readerOutput);
596 
597                         Size postviewSize =
598                                 CameraTestUtils.getMaxSize(postviewSizes.toArray(new Size[0]));
599                         SimpleImageReaderListener imageListenerPostview =
600                                 new SimpleImageReaderListener(false, 1);
601                         ImageReader postviewImageReader = CameraTestUtils.makeImageReader(
602                                     postviewSize, postviewFormat, /*maxImages*/ 1,
603                                     imageListenerPostview, mTestRule.getHandler());
604 
605                         Surface postviewImageReaderSurface = postviewImageReader.getSurface();
606                         OutputConfiguration postviewReaderOutput =
607                                 new OutputConfiguration(postviewImageReaderSurface);
608                         BlockingExtensionSessionCallback sessionListener =
609                                 new BlockingExtensionSessionCallback(mock(
610                                         CameraExtensionSession.StateCallback.class));
611                         ExtensionSessionConfiguration configuration =
612                                 new ExtensionSessionConfiguration(extension, outputConfigs,
613                                         new HandlerExecutor(mTestRule.getHandler()),
614                                         sessionListener);
615                         configuration.setPostviewOutputConfiguration(postviewReaderOutput);
616                         assertNotNull(configuration.getPostviewOutputConfiguration());
617 
618                         String streamName = "test_extension_postview_capture";
619                         mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
620                         mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
621                         mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
622                                 ResultUnit.NONE);
623                         double[] captureTimes = new double[IMAGE_COUNT];
624                         double[] postviewCaptureTimes = new double[IMAGE_COUNT];
625                         boolean captureResultsSupported =
626                                 !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
627 
628                         try {
629                             mTestRule.openDevice(id);
630                             CameraDevice camera = mTestRule.getCamera();
631                             camera.createExtensionSession(configuration);
632                             CameraExtensionSession extensionSession =
633                                     sessionListener.waitAndGetSession(
634                                             SESSION_CONFIGURE_TIMEOUT_MS);
635                             assertNotNull(extensionSession);
636 
637                             CaptureRequest.Builder captureBuilder =
638                                     mTestRule.getCamera().createCaptureRequest(
639                                             CameraDevice.TEMPLATE_STILL_CAPTURE);
640                             captureBuilder.addTarget(imageReaderSurface);
641                             captureBuilder.addTarget(postviewImageReaderSurface);
642                             CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
643                                     mock(CameraExtensionSession.ExtensionCaptureCallback.class);
644                             SimpleCaptureCallback captureCallback =
645                                     new SimpleCaptureCallback(extension, captureMockCallback,
646                                             extensionChars.getAvailableCaptureResultKeys(
647                                             extension), mCollector);
648 
649                             for (int i = 0; i < IMAGE_COUNT; i++) {
650                                 int jpegOrientation = (i * 90) % 360; // degrees [0..270]
651                                 if (captureFormat == ImageFormat.JPEG
652                                         || captureFormat == ImageFormat.JPEG_R
653                                         || postviewFormat == ImageFormat.JPEG
654                                         || postviewFormat == ImageFormat.JPEG_R) {
655                                     captureBuilder.set(CaptureRequest.JPEG_ORIENTATION,
656                                             jpegOrientation);
657                                 }
658                                 CaptureRequest request = captureBuilder.build();
659                                 long startTimeMs = SystemClock.elapsedRealtime();
660                                 captureCallback.resetCaptureProgress();
661                                 int sequenceId = extensionSession.capture(request,
662                                         new HandlerExecutor(mTestRule.getHandler()),
663                                         captureCallback);
664 
665                                 Image imgPostview  =
666                                         imageListenerPostview
667                                         .getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
668                                 postviewCaptureTimes[i] =
669                                         SystemClock.elapsedRealtime() - startTimeMs;
670 
671                                 if (postviewFormat == ImageFormat.JPEG
672                                         || postviewFormat == ImageFormat.JPEG_R) {
673                                     verifyJpegOrientation(imgPostview, postviewSize,
674                                             jpegOrientation, postviewFormat);
675                                 } else {
676                                     validateImage(imgPostview, postviewSize.getWidth(),
677                                             postviewSize.getHeight(), postviewFormat, null);
678                                 }
679 
680                                 Long imgTsPostview = imgPostview.getTimestamp();
681                                 imgPostview.close();
682 
683                                 Image img =
684                                         imageListener.getImage(
685                                         MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
686                                 captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
687 
688                                 if (captureFormat == ImageFormat.JPEG
689                                         || captureFormat == ImageFormat.JPEG_R) {
690                                     verifyJpegOrientation(img, maxSize,
691                                             jpegOrientation, captureFormat);
692                                 } else {
693                                     validateImage(img, maxSize.getWidth(),
694                                             maxSize.getHeight(), captureFormat, null);
695                                 }
696 
697                                 Long imgTs = img.getTimestamp();
698                                 img.close();
699 
700                                 assertEquals("Still capture timestamp does not match its "
701                                         + "postview timestamp", imgTsPostview, imgTs);
702 
703                                 verify(captureMockCallback,
704                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
705                                         .onCaptureStarted(eq(extensionSession), eq(request),
706                                         eq(imgTs));
707                                 verify(captureMockCallback, times(1))
708                                         .onCaptureStarted(eq(extensionSession), eq(request),
709                                         anyLong());
710                                 verify(captureMockCallback,
711                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
712                                         .onCaptureProcessStarted(extensionSession, request);
713                                 verify(captureMockCallback,
714                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
715                                         .onCaptureSequenceCompleted(extensionSession, sequenceId);
716                                 if (captureResultsSupported) {
717                                     verify(captureMockCallback,
718                                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
719                                             .onCaptureResultAvailable(eq(extensionSession),
720                                             eq(request), any(TotalCaptureResult.class));
721                                 }
722                                 if (captureProgressSupported) {
723                                     verify(captureMockCallback,
724                                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
725                                             .onCaptureProcessProgressed(eq(extensionSession),
726                                             eq(request), eq(100));
727                                 }
728                             }
729 
730                             mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
731                                     ResultUnit.NONE);
732                             mReportLog.addValue("height", maxSize.getHeight(),
733                                     ResultType.NEUTRAL, ResultUnit.NONE);
734                             mReportLog.addValue("captureFormat", captureFormat, ResultType.NEUTRAL,
735                                     ResultUnit.NONE);
736                             mReportLog.addValue("postviewFormat", postviewFormat,
737                                     ResultType.NEUTRAL, ResultUnit.NONE);
738                             long avgPostviewLatency = (long) Stat.getAverage(postviewCaptureTimes);
739                             mReportLog.addValue("avg_postview_latency", avgPostviewLatency,
740                                     ResultType.LOWER_BETTER, ResultUnit.MS);
741                             long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
742                             mReportLog.addValue("avg_capture_latency", avgCaptureLatency,
743                                     ResultType.LOWER_BETTER, ResultUnit.MS);
744 
745                             verify(captureMockCallback, times(0))
746                                     .onCaptureSequenceAborted(any(CameraExtensionSession.class),
747                                             anyInt());
748                             verify(captureMockCallback, times(0))
749                                     .onCaptureFailed(any(CameraExtensionSession.class),
750                                             any(CaptureRequest.class));
751                             verify(captureMockCallback, times(0))
752                                     .onCaptureFailed(any(CameraExtensionSession.class),
753                                             any(CaptureRequest.class), anyInt());
754                             Range<Long> latencyRange =
755                                     extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
756                                             maxSize, captureFormat);
757                             if (latencyRange != null) {
758                                 String msg = String.format("Camera [%s]: The measured average "
759                                         + "capture latency of %d ms. for extension type %d  "
760                                         + "with image format: %d and size: %dx%d must be "
761                                         + "within the advertised range of [%d, %d] ms.",
762                                         id, avgCaptureLatency, extension, captureFormat,
763                                         maxSize.getWidth(), maxSize.getHeight(),
764                                         latencyRange.getLower(), latencyRange.getUpper());
765                                 assertTrue(msg, latencyRange.contains(avgCaptureLatency));
766                             }
767 
768                             extensionSession.close();
769 
770                             sessionListener.getStateWaiter().waitForState(
771                                     BlockingExtensionSessionCallback.SESSION_CLOSED,
772                                     SESSION_CLOSE_TIMEOUT_MS);
773                         } finally {
774                             mTestRule.closeDevice(id);
775                             postviewImageReader.close();
776                             extensionImageReader.close();
777                             mReportLog.submit(InstrumentationRegistry.getInstrumentation());
778                         }
779                     }
780                 }
781             }
782         }
783     }
784 
785     @Test
786     @RequiresFlagsEnabled({Flags.FLAG_EXTENSION_10_BIT,
787         Flags.FLAG_CAMERA_EXTENSIONS_CHARACTERISTICS_GET})
test10bitRepeatingAndCaptureCombined()788     public void test10bitRepeatingAndCaptureCombined() throws Exception {
789         final int IMAGE_COUNT = 5;
790         for (String id : getCameraIdsUnderTest()) {
791             StaticMetadata staticMeta =
792                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
793             if (!staticMeta.isColorOutputSupported()) {
794                 continue;
795             }
796             updatePreviewSurfaceTexture();
797             CameraExtensionCharacteristics extensionChars =
798                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
799             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
800             for (Integer extension : supportedExtensions) {
801                 boolean captureProgressSupported = extensionChars.isCaptureProcessProgressAvailable(
802                         extension);
803                 int captureFormat = ImageFormat.YCBCR_P010;
804                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
805                         captureFormat);
806                 if (extensionSizes.isEmpty()) {
807                     continue;
808                 }
809 
810                 int[] capabilities = extensionChars.get(extension,
811                         CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES);
812                 assertNotNull(capabilities);
813                 assertArrayContains("Supports YCBCR_P010 format but "
814                         + "REQUEST_AVAILABLE_CAPABILITIES does not contain "
815                         + "REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT", capabilities,
816                         CameraCharacteristics
817                         .REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT);
818 
819                 Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
820                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
821                         1);
822                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
823                         captureFormat, /*maxImages*/ 1, imageListener,
824                         mTestRule.getHandler());
825                 Surface imageReaderSurface = extensionImageReader.getSurface();
826                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
827 
828                 DynamicRangeProfiles dynamicRangeProfiles = extensionChars
829                         .get(extension,
830                         CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES);
831                 assertNotNull(dynamicRangeProfiles);
832                 assertTrue(dynamicRangeProfiles.getSupportedProfiles()
833                         .contains(DynamicRangeProfiles.HLG10));
834 
835                 // HLG10 is supported for all 10-bit capable devices
836                 readerOutput.setDynamicRangeProfile(DynamicRangeProfiles.HLG10);
837 
838                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
839                 outputConfigs.add(readerOutput);
840 
841                 // Pick a supported preview/repeating size with aspect ratio close to the
842                 // multi-frame capture size
843                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
844                         mSurfaceTexture.getClass());
845                 Size maxRepeatingSize =
846                         CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
847                 List<Size> previewSizes = getSupportedPreviewSizes(id,
848                         mTestRule.getCameraManager(),
849                         getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
850                 List<Size> supportedPreviewSizes =
851                         previewSizes.stream().filter(repeatingSizes::contains).collect(
852                                 Collectors.toList());
853                 if (!supportedPreviewSizes.isEmpty()) {
854                     float targetAr =
855                             ((float) maxSize.getWidth()) / maxSize.getHeight();
856                     for (Size s : supportedPreviewSizes) {
857                         float currentAr = ((float) s.getWidth()) / s.getHeight();
858                         if (Math.abs(targetAr - currentAr) < 0.01) {
859                             maxRepeatingSize = s;
860                             break;
861                         }
862                     }
863                 }
864 
865                 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
866                         maxRepeatingSize.getHeight());
867                 Surface texturedSurface = new Surface(mSurfaceTexture);
868                 OutputConfiguration previewOutput = new OutputConfiguration(texturedSurface);
869                 previewOutput.setDynamicRangeProfile(DynamicRangeProfiles.HLG10);
870                 outputConfigs.add(previewOutput);
871 
872                 BlockingExtensionSessionCallback sessionListener =
873                         new BlockingExtensionSessionCallback(mock(
874                                 CameraExtensionSession.StateCallback.class));
875                 ExtensionSessionConfiguration configuration =
876                         new ExtensionSessionConfiguration(extension, outputConfigs,
877                                 new HandlerExecutor(mTestRule.getHandler()),
878                                 sessionListener);
879 
880                 ColorSpaceProfiles colorSpaceProfiles = extensionChars
881                         .get(extension,
882                         CameraCharacteristics.REQUEST_AVAILABLE_COLOR_SPACE_PROFILES);
883                 if (colorSpaceProfiles != null) {
884                     Set<ColorSpace.Named> compatibleColorSpaces =
885                             colorSpaceProfiles.getSupportedColorSpacesForDynamicRange(
886                             captureFormat, DynamicRangeProfiles.HLG10);
887                     if (!compatibleColorSpaces.isEmpty()) {
888                         configuration.setColorSpace(compatibleColorSpaces.iterator().next());
889                     }
890                 }
891 
892                 String streamName = "test_extension_10_bit_repeating_and_capture";
893                 mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
894                 mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
895                 mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
896                         ResultUnit.NONE);
897                 double[] captureTimes = new double[IMAGE_COUNT];
898                 boolean captureResultsSupported =
899                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
900 
901                 try {
902                     mTestRule.openDevice(id);
903                     CameraDevice camera = mTestRule.getCamera();
904                     camera.createExtensionSession(configuration);
905                     CameraExtensionSession extensionSession =
906                             sessionListener.waitAndGetSession(
907                                     SESSION_CONFIGURE_TIMEOUT_MS);
908                     assertNotNull(extensionSession);
909 
910                     CaptureRequest.Builder captureBuilder =
911                             mTestRule.getCamera().createCaptureRequest(
912                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
913                     captureBuilder.addTarget(texturedSurface);
914                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
915                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
916                     SimpleCaptureCallback repeatingCaptureCallback =
917                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
918                                     extensionChars.getAvailableCaptureResultKeys(extension),
919                                     mCollector);
920 
921                     CaptureRequest repeatingRequest = captureBuilder.build();
922                     int repeatingSequenceId =
923                             extensionSession.setRepeatingRequest(repeatingRequest,
924                                     new HandlerExecutor(mTestRule.getHandler()),
925                                     repeatingCaptureCallback);
926 
927                     Thread.sleep(REPEATING_REQUEST_TIMEOUT_MS);
928 
929                     verify(repeatingCallbackMock, atLeastOnce())
930                             .onCaptureStarted(eq(extensionSession), eq(repeatingRequest),
931                                     anyLong());
932                     verify(repeatingCallbackMock, atLeastOnce())
933                             .onCaptureProcessStarted(extensionSession, repeatingRequest);
934                     if (captureResultsSupported) {
935                         verify(repeatingCallbackMock,
936                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
937                                 .onCaptureResultAvailable(eq(extensionSession),
938                                         eq(repeatingRequest), any(TotalCaptureResult.class));
939                     }
940                     verify(repeatingCallbackMock, times(0)).onCaptureProcessProgressed(
941                             any(CameraExtensionSession.class), any(CaptureRequest.class), anyInt());
942 
943                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
944                             CameraDevice.TEMPLATE_STILL_CAPTURE);
945                     captureBuilder.addTarget(imageReaderSurface);
946                     CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
947                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
948                     SimpleCaptureCallback captureCallback =
949                             new SimpleCaptureCallback(extension, captureMockCallback,
950                                     extensionChars.getAvailableCaptureResultKeys(extension),
951                                     mCollector);
952 
953                     for (int i = 0; i < IMAGE_COUNT; i++) {
954                         CaptureRequest request = captureBuilder.build();
955                         long startTimeMs = SystemClock.elapsedRealtime();
956                         captureCallback.resetCaptureProgress();
957                         int sequenceId = extensionSession.capture(request,
958                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
959 
960                         Image img =
961                                 imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
962                         captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
963 
964                         validateImage(img, maxSize.getWidth(),
965                                 maxSize.getHeight(), captureFormat, null);
966 
967                         long imgTs = img.getTimestamp();
968                         img.close();
969 
970                         verify(captureMockCallback,
971                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
972                                 .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs));
973                         verify(captureMockCallback, times(1))
974                                 .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
975                         verify(captureMockCallback,
976                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
977                                 .onCaptureProcessStarted(extensionSession, request);
978                         verify(captureMockCallback,
979                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
980                                 .onCaptureSequenceCompleted(extensionSession, sequenceId);
981                         if (captureResultsSupported) {
982                             verify(captureMockCallback,
983                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
984                                     .onCaptureResultAvailable(eq(extensionSession), eq(request),
985                                     any(TotalCaptureResult.class));
986                         }
987                         if (captureProgressSupported) {
988                             verify(captureMockCallback,
989                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
990                                     .onCaptureProcessProgressed(eq(extensionSession),
991                                     eq(request), eq(100));
992                         }
993                     }
994 
995                     mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
996                             ResultUnit.NONE);
997                     mReportLog.addValue("height", maxSize.getHeight(),
998                             ResultType.NEUTRAL, ResultUnit.NONE);
999                     mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL,
1000                             ResultUnit.NONE);
1001                     long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
1002                     mReportLog.addValue("avg_latency", avgCaptureLatency,
1003                             ResultType.LOWER_BETTER, ResultUnit.MS);
1004 
1005                     verify(captureMockCallback, times(0))
1006                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1007                                     anyInt());
1008                     verify(captureMockCallback, times(0))
1009                             .onCaptureFailed(any(CameraExtensionSession.class),
1010                                     any(CaptureRequest.class));
1011                     verify(captureMockCallback, times(0))
1012                             .onCaptureFailed(any(CameraExtensionSession.class),
1013                                     any(CaptureRequest.class), anyInt());
1014                     Range<Long> latencyRange =
1015                             extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
1016                                     maxSize, captureFormat);
1017                     if (latencyRange != null) {
1018                         String msg = String.format("Camera [%s]: The measured average "
1019                                         + "capture latency of %d ms. for extension type %d  "
1020                                         + "with image format: %d and size: %dx%d must be "
1021                                         + "within the advertised range of [%d, %d] ms.",
1022                                 id, avgCaptureLatency, extension, captureFormat,
1023                                 maxSize.getWidth(), maxSize.getHeight(),
1024                                 latencyRange.getLower(), latencyRange.getUpper());
1025                         assertTrue(msg, latencyRange.contains(avgCaptureLatency));
1026                     }
1027 
1028                     extensionSession.stopRepeating();
1029 
1030                     verify(repeatingCallbackMock,
1031                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1032                             .onCaptureSequenceCompleted(extensionSession, repeatingSequenceId);
1033                     verify(repeatingCallbackMock, times(0))
1034                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1035                             anyInt());
1036 
1037                     extensionSession.close();
1038 
1039                     sessionListener.getStateWaiter().waitForState(
1040                             BlockingExtensionSessionCallback.SESSION_CLOSED,
1041                             SESSION_CLOSE_TIMEOUT_MS);
1042                 } finally {
1043                     mTestRule.closeDevice(id);
1044                     extensionImageReader.close();
1045                     mReportLog.submit(InstrumentationRegistry.getInstrumentation());
1046                 }
1047             }
1048         }
1049     }
1050 
1051     // Test case for multi-frame only capture on all supported extensions and expected state
1052     // callbacks. Verify still frame output, measure the average capture latency and if possible
1053     // ensure that the value is within the reported range.
1054     @Test
testMultiFrameCapture()1055     public void testMultiFrameCapture() throws Exception {
1056         final int IMAGE_COUNT = 10;
1057         final int SUPPORTED_CAPTURE_OUTPUT_FORMATS[] = {
1058                 ImageFormat.YUV_420_888,
1059                 ImageFormat.JPEG,
1060                 ImageFormat.JPEG_R
1061         };
1062         for (String id : getCameraIdsUnderTest()) {
1063             StaticMetadata staticMeta =
1064                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1065             if (!staticMeta.isColorOutputSupported()) {
1066                 continue;
1067             }
1068             updatePreviewSurfaceTexture();
1069             CameraExtensionCharacteristics extensionChars =
1070                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1071             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1072             for (Integer extension : supportedExtensions) {
1073                 boolean captureProgressSupported = extensionChars.isCaptureProcessProgressAvailable(
1074                         extension);
1075                 for (int captureFormat : SUPPORTED_CAPTURE_OUTPUT_FORMATS) {
1076                     List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
1077                             captureFormat);
1078                     if (extensionSizes.isEmpty()) {
1079                         continue;
1080                     }
1081                     Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
1082                     SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
1083                             1);
1084                     ImageReader extensionImageReader = CameraTestUtils.makeImageReader(maxSize,
1085                             captureFormat, /*maxImages*/ 1, imageListener,
1086                             mTestRule.getHandler());
1087                     Surface imageReaderSurface = extensionImageReader.getSurface();
1088                     OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1089                     List<OutputConfiguration> outputConfigs = new ArrayList<>();
1090                     outputConfigs.add(readerOutput);
1091 
1092                     BlockingExtensionSessionCallback sessionListener =
1093                             new BlockingExtensionSessionCallback(mock(
1094                                     CameraExtensionSession.StateCallback.class));
1095                     ExtensionSessionConfiguration configuration =
1096                             new ExtensionSessionConfiguration(extension, outputConfigs,
1097                                     new HandlerExecutor(mTestRule.getHandler()),
1098                                     sessionListener);
1099                     String streamName = "test_extension_capture";
1100                     mReportLog = new DeviceReportLog(REPORT_LOG_NAME, streamName);
1101                     mReportLog.addValue("camera_id", id, ResultType.NEUTRAL, ResultUnit.NONE);
1102                     mReportLog.addValue("extension_id", extension, ResultType.NEUTRAL,
1103                             ResultUnit.NONE);
1104                     double[] captureTimes = new double[IMAGE_COUNT];
1105                     boolean captureResultsSupported =
1106                             !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
1107 
1108                     try {
1109                         mTestRule.openDevice(id);
1110                         CameraDevice camera = mTestRule.getCamera();
1111                         camera.createExtensionSession(configuration);
1112                         CameraExtensionSession extensionSession =
1113                                 sessionListener.waitAndGetSession(
1114                                         SESSION_CONFIGURE_TIMEOUT_MS);
1115                         assertNotNull(extensionSession);
1116 
1117                         CaptureRequest.Builder captureBuilder =
1118                                 mTestRule.getCamera().createCaptureRequest(
1119                                         CameraDevice.TEMPLATE_STILL_CAPTURE);
1120                         captureBuilder.addTarget(imageReaderSurface);
1121                         CameraExtensionSession.ExtensionCaptureCallback captureMockCallback =
1122                                 mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1123                         SimpleCaptureCallback captureCallback =
1124                                 new SimpleCaptureCallback(extension, captureMockCallback,
1125                                         extensionChars.getAvailableCaptureResultKeys(extension),
1126                                         mCollector);
1127 
1128                         for (int i = 0; i < IMAGE_COUNT; i++) {
1129                             int jpegOrientation = (i * 90) % 360; // degrees [0..270]
1130                             if (captureFormat == ImageFormat.JPEG
1131                                     || captureFormat == ImageFormat.JPEG_R) {
1132                                 captureBuilder.set(CaptureRequest.JPEG_ORIENTATION,
1133                                         jpegOrientation);
1134                             }
1135                             CaptureRequest request = captureBuilder.build();
1136                             long startTimeMs = SystemClock.elapsedRealtime();
1137                             captureCallback.resetCaptureProgress();
1138                             int sequenceId = extensionSession.capture(request,
1139                                     new HandlerExecutor(mTestRule.getHandler()), captureCallback);
1140 
1141                             Image img =
1142                                     imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
1143                             captureTimes[i] = SystemClock.elapsedRealtime() - startTimeMs;
1144 
1145                             if (captureFormat == ImageFormat.JPEG
1146                                     || captureFormat == ImageFormat.JPEG_R) {
1147                                 verifyJpegOrientation(img, maxSize,
1148                                         jpegOrientation, captureFormat);
1149                             } else {
1150                                 validateImage(img, maxSize.getWidth(),
1151                                         maxSize.getHeight(), captureFormat, null);
1152                             }
1153 
1154                             Long imgTs = img.getTimestamp();
1155                             img.close();
1156 
1157                             verify(captureMockCallback,
1158                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1159                                     .onCaptureStarted(eq(extensionSession), eq(request), eq(imgTs));
1160                             verify(captureMockCallback, times(1))
1161                                     .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
1162                             verify(captureMockCallback,
1163                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1164                                     .onCaptureProcessStarted(extensionSession, request);
1165                             verify(captureMockCallback,
1166                                     timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1167                                     .onCaptureSequenceCompleted(extensionSession, sequenceId);
1168                             if (captureResultsSupported) {
1169                                 verify(captureMockCallback,
1170                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1171                                         .onCaptureResultAvailable(eq(extensionSession), eq(request),
1172                                                 any(TotalCaptureResult.class));
1173                             }
1174                             if (captureProgressSupported) {
1175                                 verify(captureMockCallback,
1176                                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1177                                     .onCaptureProcessProgressed(eq(extensionSession),
1178                                             eq(request), eq(100));
1179                             }
1180                         }
1181 
1182                         mReportLog.addValue("width", maxSize.getWidth(), ResultType.NEUTRAL,
1183                                 ResultUnit.NONE);
1184                         mReportLog.addValue("height", maxSize.getHeight(),
1185                                 ResultType.NEUTRAL, ResultUnit.NONE);
1186                         mReportLog.addValue("format", captureFormat, ResultType.NEUTRAL,
1187                                 ResultUnit.NONE);
1188                         long avgCaptureLatency = (long) Stat.getAverage(captureTimes);
1189                         mReportLog.addValue("avg_latency", avgCaptureLatency,
1190                                 ResultType.LOWER_BETTER, ResultUnit.MS);
1191 
1192                         verify(captureMockCallback, times(0))
1193                                 .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1194                                         anyInt());
1195                         verify(captureMockCallback, times(0))
1196                                 .onCaptureFailed(any(CameraExtensionSession.class),
1197                                         any(CaptureRequest.class));
1198                         verify(captureMockCallback, times(0))
1199                                 .onCaptureFailed(any(CameraExtensionSession.class),
1200                                         any(CaptureRequest.class), anyInt());
1201                         Range<Long> latencyRange =
1202                                 extensionChars.getEstimatedCaptureLatencyRangeMillis(extension,
1203                                         maxSize, captureFormat);
1204                         if (latencyRange != null) {
1205                             String msg = String.format("Camera [%s]: The measured average "
1206                                             + "capture latency of %d ms. for extension type %d  "
1207                                             + "with image format: %d and size: %dx%d must be "
1208                                             + "within the advertised range of [%d, %d] ms.",
1209                                     id, avgCaptureLatency, extension, captureFormat,
1210                                     maxSize.getWidth(), maxSize.getHeight(),
1211                                     latencyRange.getLower(), latencyRange.getUpper());
1212                             assertTrue(msg, latencyRange.contains(avgCaptureLatency));
1213                         }
1214 
1215                         extensionSession.close();
1216 
1217                         sessionListener.getStateWaiter().waitForState(
1218                                 BlockingExtensionSessionCallback.SESSION_CLOSED,
1219                                 SESSION_CLOSE_TIMEOUT_MS);
1220                     } finally {
1221                         mTestRule.closeDevice(id);
1222                         extensionImageReader.close();
1223                         mReportLog.submit(InstrumentationRegistry.getInstrumentation());
1224                     }
1225                 }
1226             }
1227         }
1228     }
1229 
1230     // Verify concurrent extension sessions behavior
1231     @Test
testConcurrentSessions()1232     public void testConcurrentSessions() throws Exception {
1233         Set<Set<String>> concurrentCameraIdSet =
1234                 mTestRule.getCameraManager().getConcurrentCameraIds();
1235         if (concurrentCameraIdSet.isEmpty()) {
1236             return;
1237         }
1238 
1239         for (String id : getCameraIdsUnderTest()) {
1240             StaticMetadata staticMeta =
1241                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1242             if (!staticMeta.isColorOutputSupported()) {
1243                 continue;
1244             }
1245             CameraExtensionCharacteristics extensionChars =
1246                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1247             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1248             if (supportedExtensions.isEmpty()) {
1249                 continue;
1250             }
1251 
1252             Set<String> concurrentCameraIds = null;
1253             for (Set<String> entry : concurrentCameraIdSet) {
1254                 if (entry.contains(id)) {
1255                     concurrentCameraIds = entry;
1256                     break;
1257                 }
1258             }
1259             if (concurrentCameraIds == null) {
1260                 continue;
1261             }
1262 
1263             String concurrentCameraId = null;
1264             CameraExtensionCharacteristics concurrentExtensionChars = null;
1265             for (String entry : concurrentCameraIds) {
1266                 if (entry.equals(id)) {
1267                     continue;
1268                 }
1269                 if (!(new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(
1270                         entry))).isColorOutputSupported()) {
1271                     continue;
1272                 }
1273                 CameraExtensionCharacteristics chars =
1274                         mTestRule.getCameraManager().getCameraExtensionCharacteristics(entry);
1275                 if (chars.getSupportedExtensions().isEmpty()) {
1276                     continue;
1277                 }
1278                 concurrentCameraId = entry;
1279                 concurrentExtensionChars = chars;
1280                 break;
1281             }
1282             if ((concurrentCameraId == null) || (concurrentExtensionChars == null)) {
1283                 continue;
1284             }
1285 
1286             updatePreviewSurfaceTexture();
1287             int extensionId = supportedExtensions.get(0);
1288             List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extensionId,
1289                     mSurfaceTexture.getClass());
1290             Size maxSize = CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
1291             mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(), maxSize.getHeight());
1292             OutputConfiguration outputConfig = new OutputConfiguration(
1293                     new Surface(mSurfaceTexture));
1294             List<OutputConfiguration> outputConfigs = new ArrayList<>();
1295             outputConfigs.add(outputConfig);
1296 
1297             BlockingExtensionSessionCallback sessionListener =
1298                     new BlockingExtensionSessionCallback(
1299                             mock(CameraExtensionSession.StateCallback.class));
1300             ExtensionSessionConfiguration configuration =
1301                     new ExtensionSessionConfiguration(extensionId, outputConfigs,
1302                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
1303 
1304             CameraDevice concurrentCameraDevice = null;
1305             ImageReader extensionImageReader = null;
1306             try {
1307                 mTestRule.openDevice(id);
1308                 concurrentCameraDevice = CameraTestUtils.openCamera(mTestRule.getCameraManager(),
1309                         concurrentCameraId, new BlockingStateCallback(), mTestRule.getHandler());
1310                 CameraDevice camera = mTestRule.getCamera();
1311                 camera.createExtensionSession(configuration);
1312                 CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
1313                         SESSION_CONFIGURE_TIMEOUT_MS);
1314                 assertNotNull(extensionSession);
1315 
1316                 assertNotNull(concurrentCameraDevice);
1317                 int concurrentExtensionId =
1318                         concurrentExtensionChars.getSupportedExtensions().get(0);
1319                 List<Size> captureSizes = concurrentExtensionChars.getExtensionSupportedSizes(
1320                         concurrentExtensionId, mSurfaceTexture.getClass());
1321                 assertFalse("No SurfaceTexture output supported", captureSizes.isEmpty());
1322                 Size captureMaxSize =
1323                         CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
1324 
1325                 extensionImageReader = ImageReader.newInstance(
1326                         captureMaxSize.getWidth(), captureMaxSize.getHeight(), ImageFormat.PRIVATE,
1327                         /*maxImages*/ 1, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE);
1328                 Surface imageReaderSurface = extensionImageReader.getSurface();
1329                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1330                 outputConfigs = new ArrayList<>();
1331                 outputConfigs.add(readerOutput);
1332                 CameraExtensionSession.StateCallback mockSessionListener =
1333                         mock(CameraExtensionSession.StateCallback.class);
1334                 ExtensionSessionConfiguration concurrentConfiguration =
1335                         new ExtensionSessionConfiguration(concurrentExtensionId, outputConfigs,
1336                                 new HandlerExecutor(mTestRule.getHandler()),
1337                                 mockSessionListener);
1338                 concurrentCameraDevice.createExtensionSession(concurrentConfiguration);
1339                 // Trying to initialize multiple concurrent extension sessions is expected to fail
1340                 verify(mockSessionListener, timeout(SESSION_CONFIGURE_TIMEOUT_MS).times(1))
1341                         .onConfigureFailed(any(CameraExtensionSession.class));
1342                 verify(mockSessionListener, times(0)).onConfigured(
1343                         any(CameraExtensionSession.class));
1344 
1345                 extensionSession.close();
1346                 sessionListener.getStateWaiter().waitForState(
1347                         BlockingExtensionSessionCallback.SESSION_CLOSED,
1348                         SESSION_CLOSE_TIMEOUT_MS);
1349 
1350                 // Initialization of another extension session must now be possible
1351                 BlockingExtensionSessionCallback concurrentSessionListener =
1352                         new BlockingExtensionSessionCallback(
1353                                 mock(CameraExtensionSession.StateCallback.class));
1354                 concurrentConfiguration = new ExtensionSessionConfiguration(concurrentExtensionId,
1355                         outputConfigs, new HandlerExecutor(mTestRule.getHandler()),
1356                         concurrentSessionListener);
1357                 concurrentCameraDevice.createExtensionSession(concurrentConfiguration);
1358                 extensionSession = concurrentSessionListener.waitAndGetSession(
1359                         SESSION_CONFIGURE_TIMEOUT_MS);
1360                 assertNotNull(extensionSession);
1361                 extensionSession.close();
1362                 concurrentSessionListener.getStateWaiter().waitForState(
1363                         BlockingExtensionSessionCallback.SESSION_CLOSED,
1364                         SESSION_CLOSE_TIMEOUT_MS);
1365             } finally {
1366                 mTestRule.closeDevice(id);
1367                 if (concurrentCameraDevice != null) {
1368                     concurrentCameraDevice.close();
1369                 }
1370                 if (extensionImageReader != null) {
1371                     extensionImageReader.close();
1372                 }
1373             }
1374         }
1375     }
1376 
1377     // Test case combined repeating with multi frame capture on all supported extensions.
1378     // Verify still frame output.
1379     @Test
testRepeatingAndCaptureCombined()1380     public void testRepeatingAndCaptureCombined() throws Exception {
1381         final double LATENCY_MARGIN = .3f; // Account for system load, capture call duration etc.
1382         for (String id : getCameraIdsUnderTest()) {
1383             StaticMetadata staticMeta =
1384                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1385             if (!staticMeta.isColorOutputSupported()) {
1386                 continue;
1387             }
1388             updatePreviewSurfaceTexture();
1389             CameraExtensionCharacteristics extensionChars =
1390                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1391             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1392             for (Integer extension : supportedExtensions) {
1393 
1394                 Set<CaptureRequest.Key> supportedRequestKeys =
1395                         extensionChars.getAvailableCaptureRequestKeys(extension);
1396                 boolean supportsStrengthControl = supportedRequestKeys.contains(
1397                         CaptureRequest.EXTENSION_STRENGTH);
1398                 boolean supportsAFControlMode = supportedRequestKeys.contains(
1399                         CaptureRequest.CONTROL_AF_MODE);
1400                 Set<CaptureResult.Key> supportedResultKeys =
1401                         extensionChars.getAvailableCaptureResultKeys(extension);
1402                 boolean supportsAFControlState = supportedResultKeys.contains(
1403                         CaptureResult.CONTROL_AF_STATE);
1404                 if (supportsStrengthControl) {
1405                     assertTrue(supportedResultKeys.contains(CaptureResult.EXTENSION_STRENGTH));
1406                 }
1407 
1408                 int captureFormat = ImageFormat.JPEG;
1409                 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
1410                         captureFormat);
1411                 assertFalse("No Jpeg output supported", captureSizes.isEmpty());
1412                 Size captureMaxSize =
1413                         CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
1414 
1415                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false
1416                         , 1);
1417                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
1418                         captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener,
1419                         mTestRule.getHandler());
1420                 Surface imageReaderSurface = extensionImageReader.getSurface();
1421                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
1422                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
1423                 outputConfigs.add(readerOutput);
1424 
1425                 // Pick a supported preview/repeating size with aspect ratio close to the
1426                 // multi-frame capture size
1427                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
1428                         mSurfaceTexture.getClass());
1429                 Size maxRepeatingSize =
1430                         CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
1431                 List<Size> previewSizes = getSupportedPreviewSizes(id,
1432                         mTestRule.getCameraManager(),
1433                         getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
1434                 List<Size> supportedPreviewSizes =
1435                         previewSizes.stream().filter(repeatingSizes::contains).collect(
1436                                 Collectors.toList());
1437                 if (!supportedPreviewSizes.isEmpty()) {
1438                     float targetAr =
1439                             ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight();
1440                     for (Size s : supportedPreviewSizes) {
1441                         float currentAr = ((float) s.getWidth()) / s.getHeight();
1442                         if (Math.abs(targetAr - currentAr) < 0.01) {
1443                             maxRepeatingSize = s;
1444                             break;
1445                         }
1446                     }
1447                 }
1448 
1449                 boolean captureResultsSupported =
1450                         !extensionChars.getAvailableCaptureResultKeys(extension).isEmpty();
1451 
1452                 mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
1453                         maxRepeatingSize.getHeight());
1454                 Surface texturedSurface = new Surface(mSurfaceTexture);
1455                 outputConfigs.add(new OutputConfiguration(texturedSurface));
1456 
1457                 BlockingExtensionSessionCallback sessionListener =
1458                         new BlockingExtensionSessionCallback(mock(
1459                                 CameraExtensionSession.StateCallback.class));
1460                 ExtensionSessionConfiguration configuration =
1461                         new ExtensionSessionConfiguration(extension, outputConfigs,
1462                                 new HandlerExecutor(mTestRule.getHandler()),
1463                                 sessionListener);
1464                 try {
1465                     mTestRule.openDevice(id);
1466                     CameraDevice camera = mTestRule.getCamera();
1467                     camera.createExtensionSession(configuration);
1468                     CameraExtensionSession extensionSession =
1469                             sessionListener.waitAndGetSession(
1470                                     SESSION_CONFIGURE_TIMEOUT_MS);
1471                     assertNotNull(extensionSession);
1472 
1473                     CaptureRequest.Builder captureBuilder =
1474                             mTestRule.getCamera().createCaptureRequest(
1475                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
1476                     captureBuilder.addTarget(texturedSurface);
1477                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
1478                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1479                     // Check passive AF
1480                     AutoFocusStateListener mockAFListener =
1481                             mock(AutoFocusStateListener.class);
1482                     AutoFocusStateMachine afState = new AutoFocusStateMachine(
1483                             new TestAutoFocusProxy(mockAFListener));
1484                     afState.setPassiveAutoFocus(true /*picture*/, captureBuilder);
1485                     SimpleCaptureCallback repeatingCaptureCallback =
1486                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
1487                                     extensionChars.getAvailableCaptureResultKeys(extension),
1488                                     mCollector, afState, null /*flashState*/, null /*aeState*/,
1489                                     false);
1490 
1491                     if (supportsStrengthControl) {
1492                         captureBuilder.set(CaptureRequest.EXTENSION_STRENGTH, 100);
1493                     }
1494 
1495                     CaptureRequest repeatingRequest = captureBuilder.build();
1496                     if (supportsAFControlMode && supportsAFControlState) {
1497                         captureBuilder.set(
1498                                 CaptureRequest.CONTROL_AF_MODE,
1499                                 CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE
1500                         );
1501                     }
1502                     int repeatingSequenceId =
1503                             extensionSession.setRepeatingRequest(repeatingRequest,
1504                                     new HandlerExecutor(mTestRule.getHandler()),
1505                                     repeatingCaptureCallback);
1506 
1507                     Thread.sleep(REPEATING_REQUEST_TIMEOUT_MS);
1508 
1509                     verify(repeatingCallbackMock, atLeastOnce())
1510                             .onCaptureStarted(eq(extensionSession), eq(repeatingRequest),
1511                                     anyLong());
1512                     verify(repeatingCallbackMock, atLeastOnce())
1513                             .onCaptureProcessStarted(extensionSession, repeatingRequest);
1514                     if (captureResultsSupported) {
1515                         verify(repeatingCallbackMock,
1516                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).atLeastOnce())
1517                                 .onCaptureResultAvailable(eq(extensionSession),
1518                                         eq(repeatingRequest), any(TotalCaptureResult.class));
1519                     }
1520                     verify(repeatingCallbackMock, times(0)).onCaptureProcessProgressed(
1521                             any(CameraExtensionSession.class), any(CaptureRequest.class), anyInt());
1522 
1523                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
1524                             android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE);
1525                     captureBuilder.addTarget(imageReaderSurface);
1526                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
1527                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1528 
1529                     CameraExtensionSession.StillCaptureLatency stillCaptureLatency =
1530                             extensionSession.getRealtimeStillCaptureLatency();
1531                     CaptureRequest captureRequest = captureBuilder.build();
1532                     if (supportsAFControlMode && supportsAFControlState) {
1533                         verify(mockAFListener,
1534                                 timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS).atLeastOnce())
1535                                 .onDone(anyBoolean());
1536                     }
1537                     long startTimeMs = SystemClock.elapsedRealtime();
1538                     int captureSequenceId = extensionSession.capture(captureRequest,
1539                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
1540 
1541                     Image img =
1542                             imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
1543                     long captureTime = SystemClock.elapsedRealtime() - startTimeMs;
1544                     validateImage(img, captureMaxSize.getWidth(),
1545                             captureMaxSize.getHeight(), captureFormat, null);
1546                     Long imgTs = img.getTimestamp();
1547                     img.close();
1548 
1549                     if (stillCaptureLatency != null) {
1550                         assertTrue("Still capture frame latency must be positive!",
1551                                 stillCaptureLatency.getCaptureLatency() > 0);
1552                         assertTrue("Processing capture latency must be non-negative!",
1553                                 stillCaptureLatency.getProcessingLatency() >= 0);
1554                         long estimatedTotalLatency = stillCaptureLatency.getCaptureLatency() +
1555                                 stillCaptureLatency.getProcessingLatency();
1556                         long estimatedTotalLatencyMin =
1557                                 (long) (estimatedTotalLatency * (1.f - LATENCY_MARGIN));
1558                         long estimatedTotalLatencyMax =
1559                                 (long) (estimatedTotalLatency * (1.f + LATENCY_MARGIN));
1560                         assertTrue(String.format("Camera %s: Measured still capture latency " +
1561                                                 "doesn't match: %d ms, expected [%d,%d]ms.", id,
1562                                         captureTime, estimatedTotalLatencyMin,
1563                                         estimatedTotalLatencyMax),
1564                                 (captureTime <= estimatedTotalLatencyMax) &&
1565                                         (captureTime >= estimatedTotalLatencyMin));
1566                     }
1567 
1568                     verify(captureCallback, times(1))
1569                             .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs));
1570                     verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1571                             .onCaptureProcessStarted(extensionSession, captureRequest);
1572                     if (captureResultsSupported) {
1573                         verify(captureCallback,
1574                                 timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1575                                 .onCaptureResultAvailable(eq(extensionSession),
1576                                         eq(captureRequest), any(TotalCaptureResult.class));
1577                     }
1578                     verify(captureCallback, timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1579                             .onCaptureSequenceCompleted(extensionSession,
1580                                     captureSequenceId);
1581                     verify(captureCallback, times(0))
1582                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1583                                     anyInt());
1584                     verify(captureCallback, times(0))
1585                             .onCaptureFailed(any(CameraExtensionSession.class),
1586                                     any(CaptureRequest.class));
1587                     verify(captureCallback, times(0))
1588                             .onCaptureFailed(any(CameraExtensionSession.class),
1589                                     any(CaptureRequest.class), anyInt());
1590 
1591                     extensionSession.stopRepeating();
1592 
1593                     verify(repeatingCallbackMock,
1594                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1595                             .onCaptureSequenceCompleted(extensionSession, repeatingSequenceId);
1596 
1597                     verify(repeatingCallbackMock, times(0))
1598                             .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1599                                     anyInt());
1600 
1601                     extensionSession.close();
1602 
1603                     sessionListener.getStateWaiter().waitForState(
1604                             BlockingExtensionSessionCallback.SESSION_CLOSED,
1605                             SESSION_CLOSE_TIMEOUT_MS);
1606 
1607                     assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed" +
1608                                     " callbacks must be greater or equal than the number of calls" +
1609                                     " to onCaptureStarted!",
1610                             repeatingCaptureCallback.getTotalFramesArrived() +
1611                                     repeatingCaptureCallback.getTotalFramesFailed() >=
1612                             repeatingCaptureCallback.getTotalFramesStarted());
1613                     assertTrue(String.format("The last repeating request surface timestamp " +
1614                                     "%d must be less than or equal to the last " +
1615                                     "onCaptureStarted " +
1616                                     "timestamp %d", mSurfaceTexture.getTimestamp(),
1617                             repeatingCaptureCallback.getLastTimestamp()),
1618                             mSurfaceTexture.getTimestamp() <=
1619                                     repeatingCaptureCallback.getLastTimestamp());
1620 
1621                 } finally {
1622                     mTestRule.closeDevice(id);
1623                     texturedSurface.release();
1624                     extensionImageReader.close();
1625                 }
1626             }
1627         }
1628     }
1629 
1630     // Test case for eyes free videography extension mode
1631     @Test
1632     @RequiresFlagsEnabled(com.android.internal.camera.flags.Flags.FLAG_CONCERT_MODE_API)
testEyesFreeExtension()1633     public void testEyesFreeExtension() throws Exception {
1634         for (String id : getCameraIdsUnderTest()) {
1635             StaticMetadata staticMeta =
1636                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
1637             if (!staticMeta.isColorOutputSupported()) {
1638                 continue;
1639             }
1640             updatePreviewSurfaceTexture();
1641             CameraExtensionCharacteristics extensionChars =
1642                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
1643             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
1644 
1645             if (!supportedExtensions.contains(
1646                     CameraExtensionCharacteristics.EXTENSION_EYES_FREE_VIDEOGRAPHY)) {
1647                 continue;
1648             }
1649 
1650             int extension = CameraExtensionCharacteristics.EXTENSION_EYES_FREE_VIDEOGRAPHY;
1651             List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
1652                     mSurfaceTexture.getClass());
1653             Size maxSize =
1654                     CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
1655             mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
1656                     maxSize.getHeight());
1657             Surface texturedSurface = new Surface(mSurfaceTexture);
1658 
1659             List<OutputConfiguration> outputConfigs = new ArrayList<>();
1660             outputConfigs.add(new OutputConfiguration(texturedSurface));
1661 
1662             BlockingExtensionSessionCallback sessionListener =
1663                     new BlockingExtensionSessionCallback(mock(
1664                             CameraExtensionSession.StateCallback.class));
1665             ExtensionSessionConfiguration configuration =
1666                     new ExtensionSessionConfiguration(extension, outputConfigs,
1667                             new HandlerExecutor(mTestRule.getHandler()),
1668                             sessionListener);
1669 
1670             Set<CaptureResult.Key> supportedResultKeys =
1671                     extensionChars.getAvailableCaptureResultKeys(extension);
1672             Set<CaptureRequest.Key> supportedRequestKeys =
1673                     extensionChars.getAvailableCaptureRequestKeys(extension);
1674 
1675             CameraTestUtils.checkKeysAreSupported(Arrays.asList(
1676                     EYES_FREE_REQUEST_SET, FOCUS_CAPTURE_REQUEST_SET,
1677                     ZOOM_CAPTURE_REQUEST_SET), supportedRequestKeys, true);
1678             CameraTestUtils.checkKeysAreSupported(Arrays.asList(
1679                     EYES_FREE_RESULT_SET, FOCUS_CAPTURE_RESULT_SET,
1680                     ZOOM_CAPTURE_RESULT_SET), supportedResultKeys, true);
1681 
1682             if (EFV_API_SUPPORTED &&
1683                     supportedRequestKeys.contains(ExtensionCaptureRequest.EFV_AUTO_ZOOM)) {
1684                 CameraTestUtils.checkKeysAreSupported(EYES_FREE_AUTO_ZOOM_REQUEST_SET,
1685                         supportedRequestKeys, true);
1686                 CameraTestUtils.checkKeysAreSupported(EYES_FREE_AUTO_ZOOM_RESULT_SET,
1687                         supportedResultKeys, true);
1688             }
1689 
1690             try {
1691                 mTestRule.openDevice(id);
1692                 CameraDevice camera = mTestRule.getCamera();
1693                 camera.createExtensionSession(configuration);
1694                 CameraExtensionSession extensionSession =
1695                         sessionListener.waitAndGetSession(
1696                             SESSION_CONFIGURE_TIMEOUT_MS);
1697                 assertNotNull(extensionSession);
1698 
1699                 CameraExtensionSession.ExtensionCaptureCallback captureCallbackMock =
1700                         mock(CameraExtensionSession.ExtensionCaptureCallback.class);
1701                 SimpleCaptureCallback simpleCaptureCallback =
1702                         new SimpleCaptureCallback(extension, captureCallbackMock,
1703                                 extensionChars.getAvailableCaptureResultKeys(extension),
1704                                 mCollector, staticMeta);
1705 
1706                 if (supportedRequestKeys.contains(ExtensionCaptureRequest.EFV_AUTO_ZOOM)) {
1707                     CaptureRequest.Builder captureBuilder =
1708                             mTestRule.getCamera().createCaptureRequest(
1709                                 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
1710                     captureBuilder.addTarget(texturedSurface);
1711                     captureBuilder.set(ExtensionCaptureRequest.EFV_AUTO_ZOOM, true);
1712                     captureBuilder.set(ExtensionCaptureRequest.EFV_STABILIZATION_MODE,
1713                             ExtensionCaptureRequest.EFV_STABILIZATION_MODE_LOCKED);
1714                     CaptureRequest request = captureBuilder.build();
1715 
1716                     int seqId = extensionSession.setRepeatingRequest(request,
1717                             new HandlerExecutor(mTestRule.getHandler()),
1718                             simpleCaptureCallback);
1719                     assertTrue(seqId > 0);
1720 
1721                     verify(captureCallbackMock,
1722                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
1723                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
1724                             any(TotalCaptureResult.class));
1725 
1726                     captureBuilder.set(ExtensionCaptureRequest.EFV_AUTO_ZOOM, false);
1727                     request = captureBuilder.build();
1728 
1729                     seqId = extensionSession.setRepeatingRequest(request,
1730                             new HandlerExecutor(mTestRule.getHandler()),
1731                             simpleCaptureCallback);
1732                     assertTrue(seqId > 0);
1733 
1734                     verify(captureCallbackMock,
1735                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
1736                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
1737                             any(TotalCaptureResult.class));
1738                 }
1739 
1740                 CaptureRequest.Builder captureBuilder =
1741                         mTestRule.getCamera().createCaptureRequest(
1742                                 android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
1743                 captureBuilder.addTarget(texturedSurface);
1744 
1745                 Range<Float> paddingZoomFactorRange =
1746                         extensionChars.get(extension,
1747                         CameraExtensionCharacteristics.EFV_PADDING_ZOOM_FACTOR_RANGE);
1748                 Float paddingZoomFactor = (paddingZoomFactorRange.getUpper()
1749                         + paddingZoomFactorRange.getLower()) / 2;
1750                 captureBuilder.set(ExtensionCaptureRequest.EFV_PADDING_ZOOM_FACTOR,
1751                         paddingZoomFactor);
1752                 captureBuilder.set(ExtensionCaptureRequest.EFV_STABILIZATION_MODE,
1753                         ExtensionCaptureRequest.EFV_STABILIZATION_MODE_LOCKED);
1754                 CaptureRequest request = captureBuilder.build();
1755 
1756                 int seqId = extensionSession.setRepeatingRequest(request,
1757                         new HandlerExecutor(mTestRule.getHandler()),
1758                         simpleCaptureCallback);
1759                 assertTrue(seqId > 0);
1760 
1761                 verify(captureCallbackMock,
1762                         timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
1763                         .onCaptureResultAvailable(eq(extensionSession), eq(request),
1764                                 any(TotalCaptureResult.class));
1765 
1766                 verify(captureCallbackMock,
1767                         timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
1768                         .onCaptureStarted(eq(extensionSession), eq(request), anyLong());
1769                 verify(captureCallbackMock,
1770                         timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
1771                         .onCaptureProcessStarted(extensionSession, request);
1772 
1773                 extensionSession.stopRepeating();
1774 
1775                 verify(captureCallbackMock,
1776                         timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
1777                         .onCaptureSequenceCompleted(extensionSession, seqId);
1778 
1779                 verify(captureCallbackMock, times(0))
1780                         .onCaptureSequenceAborted(any(CameraExtensionSession.class),
1781                         anyInt());
1782 
1783                 extensionSession.close();
1784 
1785                 sessionListener.getStateWaiter().waitForState(
1786                         BlockingExtensionSessionCallback.SESSION_CLOSED,
1787                         SESSION_CLOSE_TIMEOUT_MS);
1788 
1789                 assertTrue("The sum of onCaptureProcessStarted and onCaptureFailed"
1790                                 + " callbacks must be greater or equal than the number of calls"
1791                                 + " to onCaptureStarted!",
1792                         simpleCaptureCallback.getTotalFramesArrived()
1793                         + simpleCaptureCallback.getTotalFramesFailed()
1794                         >= simpleCaptureCallback.getTotalFramesStarted());
1795                 assertTrue(String.format("The last repeating request surface timestamp "
1796                                 + "%d must be less than or equal to the last "
1797                                 + "onCaptureStarted "
1798                                 + "timestamp %d", mSurfaceTexture.getTimestamp(),
1799                         simpleCaptureCallback.getLastTimestamp()),
1800                         mSurfaceTexture.getTimestamp()
1801                         <= simpleCaptureCallback.getLastTimestamp());
1802             } finally {
1803                 mTestRule.closeDevice(id);
1804                 texturedSurface.release();
1805             }
1806         }
1807     }
1808 
verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation, int captureFormat)1809     private void verifyJpegOrientation(Image img, Size jpegSize, int requestedOrientation,
1810                 int captureFormat)
1811             throws IOException {
1812         byte[] blobBuffer = getDataFromImage(img);
1813         String blobFilename = mTestRule.getDebugFileNameBase() + "/verifyJpegKeys.jpeg";
1814         dumpFile(blobFilename, blobBuffer);
1815         ExifInterface exif = new ExifInterface(blobFilename);
1816         int exifWidth = exif.getAttributeInt(ExifInterface.TAG_IMAGE_WIDTH, /*defaultValue*/0);
1817         int exifHeight = exif.getAttributeInt(ExifInterface.TAG_IMAGE_LENGTH, /*defaultValue*/0);
1818         Size exifSize = new Size(exifWidth, exifHeight);
1819         int exifOrientation = exif.getAttributeInt(ExifInterface.TAG_ORIENTATION,
1820                 /*defaultValue*/ ExifInterface.ORIENTATION_UNDEFINED);
1821         final int ORIENTATION_MIN = ExifInterface.ORIENTATION_UNDEFINED;
1822         final int ORIENTATION_MAX = ExifInterface.ORIENTATION_ROTATE_270;
1823         assertTrue(String.format("Exif orientation must be in range of [%d, %d]",
1824                 ORIENTATION_MIN, ORIENTATION_MAX),
1825                 exifOrientation >= ORIENTATION_MIN && exifOrientation <= ORIENTATION_MAX);
1826 
1827         /**
1828          * Device captured image doesn't respect the requested orientation,
1829          * which means it rotates the image buffer physically. Then we
1830          * should swap the jpegSize width/height accordingly to compare.
1831          */
1832         boolean deviceRotatedImage = exifOrientation == ExifInterface.ORIENTATION_UNDEFINED;
1833 
1834         if (deviceRotatedImage) {
1835             // Case 1.
1836             boolean needSwap = (requestedOrientation % 180 == 90);
1837             if (needSwap) {
1838                 jpegSize = new Size(jpegSize.getHeight(), jpegSize.getWidth());
1839             }
1840         } else {
1841             // Case 2.
1842             assertEquals("Exif orientation should match requested orientation",
1843                     requestedOrientation, getExifOrientationInDegree(exifOrientation));
1844         }
1845 
1846         assertEquals("Exif size should match jpeg capture size", jpegSize, exifSize);
1847 
1848         byte[] data = getDataFromImage(img);
1849         assertTrue("Invalid image data", data != null && data.length > 0);
1850 
1851         switch (captureFormat) {
1852             case ImageFormat.JPEG:
1853                 validateJpegData(data, jpegSize.getWidth(), jpegSize.getHeight(),
1854                         null /*filePath*/);
1855                 break;
1856             case ImageFormat.JPEG_R:
1857                 validateJpegData(data, jpegSize.getWidth(), jpegSize.getHeight(),
1858                         null /*filePath*/, null /*colorSpace*/, true /*gainMapPresent*/);
1859                 break;
1860             default:
1861                 throw new UnsupportedOperationException("Unsupported format for JPEG validation: "
1862                         + captureFormat);
1863         }
1864     }
1865 
getExifOrientationInDegree(int exifOrientation)1866     private static int getExifOrientationInDegree(int exifOrientation) {
1867         switch (exifOrientation) {
1868             case ExifInterface.ORIENTATION_NORMAL:
1869                 return 0;
1870             case ExifInterface.ORIENTATION_ROTATE_90:
1871                 return 90;
1872             case ExifInterface.ORIENTATION_ROTATE_180:
1873                 return 180;
1874             case ExifInterface.ORIENTATION_ROTATE_270:
1875                 return 270;
1876             default:
1877                 fail("It is impossible to get non 0, 90, 180, 270 degress exif" +
1878                         "info based on the request orientation range");
1879                 return -1;
1880         }
1881     }
1882 
1883     public static class SimpleCaptureCallback
1884             extends CameraExtensionSession.ExtensionCaptureCallback {
1885         private long mLastTimestamp = -1;
1886         private int mNumFramesArrived = 0;
1887         private int mNumFramesStarted = 0;
1888         private int mNumFramesFailed = 0;
1889         private int mLastProgressValue = -1;
1890         private boolean mNonIncreasingTimestamps = false;
1891         private HashSet<Long> mExpectedResultTimestamps = new HashSet<>();
1892         private StaticMetadata mStaticInfo;
1893         private final CameraExtensionSession.ExtensionCaptureCallback mProxy;
1894         private final AutoFocusStateMachine mAFStateMachine;
1895         private final FlashStateListener mFlashStateListener;
1896         private final AutoExposureStateListener mAEStateListener;
1897         private final HashSet<CaptureResult.Key> mSupportedResultKeys;
1898         private final CameraErrorCollector mCollector;
1899         private final boolean mPerFrameControl;
1900         private final int mExtensionType;
1901 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector)1902         public SimpleCaptureCallback(int extensionType,
1903                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1904                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector) {
1905             this(extensionType, proxy, supportedResultKeys, errorCollector, null /*afListener*/,
1906                     null /*flashState*/, null /*aeState*/, false /*perFrameControl*/);
1907         }
1908 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, StaticMetadata staticInfo)1909         public SimpleCaptureCallback(int extensionType,
1910                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1911                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector,
1912                 StaticMetadata staticInfo) {
1913             this(extensionType, proxy, supportedResultKeys, errorCollector, null /*afListener*/,
1914                     null /*flashState*/, null /*aeState*/, false /*perFrameControl*/);
1915             mStaticInfo = staticInfo;
1916         }
1917 
SimpleCaptureCallback(int extensionType, CameraExtensionSession.ExtensionCaptureCallback proxy, Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector, AutoFocusStateMachine afState, FlashStateListener flashState, AutoExposureStateListener aeState, boolean perFrameControl)1918         public SimpleCaptureCallback(int extensionType,
1919                 CameraExtensionSession.ExtensionCaptureCallback proxy,
1920                 Set<CaptureResult.Key> supportedResultKeys, CameraErrorCollector errorCollector,
1921                 AutoFocusStateMachine afState, FlashStateListener flashState,
1922                 AutoExposureStateListener aeState, boolean perFrameControl) {
1923             mProxy = proxy;
1924             mSupportedResultKeys = new HashSet<>(supportedResultKeys);
1925             mCollector = errorCollector;
1926             mAFStateMachine = afState;
1927             mFlashStateListener = flashState;
1928             mAEStateListener = aeState;
1929             mPerFrameControl = perFrameControl;
1930             mExtensionType = extensionType;
1931         }
1932 
resetCaptureProgress()1933         public void resetCaptureProgress() {
1934             mLastProgressValue = -1;
1935         }
1936 
1937         @Override
onCaptureStarted(CameraExtensionSession session, CaptureRequest request, long timestamp)1938         public void onCaptureStarted(CameraExtensionSession session,
1939                                      CaptureRequest request, long timestamp) {
1940             mExpectedResultTimestamps.add(timestamp);
1941             if (timestamp < mLastTimestamp) {
1942                 mNonIncreasingTimestamps = true;
1943             }
1944             mLastTimestamp = timestamp;
1945             mNumFramesStarted++;
1946             if (mProxy != null) {
1947                 mProxy.onCaptureStarted(session, request, timestamp);
1948             }
1949         }
1950 
1951         @Override
onCaptureProcessStarted(CameraExtensionSession session, CaptureRequest request)1952         public void onCaptureProcessStarted(CameraExtensionSession session,
1953                                             CaptureRequest request) {
1954             mNumFramesArrived++;
1955             if (mProxy != null) {
1956                 mProxy.onCaptureProcessStarted(session, request);
1957             }
1958         }
1959 
1960         @Override
onCaptureFailed(CameraExtensionSession session, CaptureRequest request)1961         public void onCaptureFailed(CameraExtensionSession session,
1962                                     CaptureRequest request) {
1963             mNumFramesFailed++;
1964             if (mProxy != null) {
1965                 mProxy.onCaptureFailed(session, request);
1966             }
1967         }
1968 
1969         @Override
onCaptureFailed(CameraExtensionSession session, CaptureRequest request, int failure)1970         public void onCaptureFailed(CameraExtensionSession session,
1971                 CaptureRequest request, int failure) {
1972             mNumFramesFailed++;
1973             if (mProxy != null) {
1974                 mProxy.onCaptureFailed(session, request, failure);
1975             }
1976         }
1977 
1978         @Override
onCaptureSequenceAborted(CameraExtensionSession session, int sequenceId)1979         public void onCaptureSequenceAborted(CameraExtensionSession session,
1980                                              int sequenceId) {
1981             if (mProxy != null) {
1982                 mProxy.onCaptureSequenceAborted(session, sequenceId);
1983             }
1984         }
1985 
1986         @Override
onCaptureSequenceCompleted(CameraExtensionSession session, int sequenceId)1987         public void onCaptureSequenceCompleted(CameraExtensionSession session,
1988                                                int sequenceId) {
1989             if (mProxy != null) {
1990                 mProxy.onCaptureSequenceCompleted(session, sequenceId);
1991             }
1992         }
1993 
1994         @Override
onCaptureProcessProgressed(CameraExtensionSession session, CaptureRequest request, int progress)1995         public void onCaptureProcessProgressed(CameraExtensionSession session,
1996                 CaptureRequest request, int progress) {
1997             if ((progress < 0) || (progress > 100)) {
1998                 mCollector.addMessage("Capture progress invalid value: " + progress);
1999                 return;
2000             }
2001             if (mLastProgressValue >= progress) {
2002                 mCollector.addMessage("Unexpected progress value: " + progress +
2003                         " last progress value: " + mLastProgressValue);
2004                 return;
2005             }
2006             mLastProgressValue = progress;
2007             if (mProxy != null) {
2008                 mProxy.onCaptureProcessProgressed(session, request, progress);
2009             }
2010         }
2011 
2012         @Override
onCaptureResultAvailable(CameraExtensionSession session, CaptureRequest request, TotalCaptureResult result)2013         public void  onCaptureResultAvailable(CameraExtensionSession session,
2014                 CaptureRequest request, TotalCaptureResult result) {
2015             final float METERING_REGION_ERROR_PERCENT_DELTA = 0.05f;
2016             if (mSupportedResultKeys.isEmpty()) {
2017                 mCollector.addMessage("Capture results not supported, but " +
2018                         "onCaptureResultAvailable still got triggered!");
2019                 return;
2020             }
2021 
2022             List<CaptureResult.Key<?>> resultKeys = result.getKeys();
2023             for (CaptureResult.Key<?> resultKey : resultKeys) {
2024                 mCollector.expectTrue("Capture result " + resultKey + " is not among the"
2025                         + " supported result keys!", mSupportedResultKeys.contains(resultKey));
2026             }
2027 
2028             Long timeStamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
2029             assertNotNull(timeStamp);
2030             assertTrue("Capture result sensor timestamp: " + timeStamp + " must match "
2031                             + " with the timestamp passed to onCaptureStarted!",
2032                     mExpectedResultTimestamps.contains(timeStamp));
2033 
2034             Integer currentType = result.get(CaptureResult.EXTENSION_CURRENT_TYPE);
2035             if (currentType != null) {
2036                 mCollector.expectNotEquals("The reported extension type cannot be set to AUTO!",
2037                         CameraExtensionCharacteristics.EXTENSION_AUTOMATIC, currentType);
2038                 if (mExtensionType == CameraExtensionCharacteristics.EXTENSION_AUTOMATIC) {
2039                     Integer expectedValues[] = {
2040                             CameraExtensionCharacteristics.EXTENSION_BOKEH,
2041                             CameraExtensionCharacteristics.EXTENSION_HDR,
2042                             CameraExtensionCharacteristics.EXTENSION_NIGHT,
2043                             CameraExtensionCharacteristics.EXTENSION_FACE_RETOUCH};
2044                     mCollector.expectContains("Unexpected extension type result: "
2045                             + currentType, expectedValues, currentType);
2046                 } else {
2047                     mCollector.expectEquals("Unexpected extension type result: " + currentType
2048                             + " expected: " + mExtensionType, mExtensionType, currentType);
2049                 }
2050             }
2051 
2052             Integer strength = request.get(CaptureRequest.EXTENSION_STRENGTH);
2053             if (strength != null) {
2054                 Integer resultStrength = result.get(CaptureResult.EXTENSION_STRENGTH);
2055                 mCollector.expectTrue("Request extension strength: " + strength +
2056                                 " doesn't match with result: " + resultStrength,
2057                         strength.equals(resultStrength));
2058             }
2059 
2060             Integer jpegOrientation = request.get(CaptureRequest.JPEG_ORIENTATION);
2061             if (jpegOrientation != null) {
2062                 Integer resultJpegOrientation = result.get(CaptureResult.JPEG_ORIENTATION);
2063                 mCollector.expectTrue("Request Jpeg orientation: " + jpegOrientation +
2064                         " doesn't match with result: " + resultJpegOrientation,
2065                         jpegOrientation.equals(resultJpegOrientation));
2066             }
2067 
2068             Byte jpegQuality = request.get(CaptureRequest.JPEG_QUALITY);
2069             if (jpegQuality != null) {
2070                 Byte resultJpegQuality = result.get(CaptureResult.JPEG_QUALITY);
2071                 mCollector.expectTrue("Request Jpeg quality: " + jpegQuality +
2072                         " doesn't match with result: " + resultJpegQuality,
2073                         jpegQuality.equals(resultJpegQuality));
2074             }
2075 
2076             if (resultKeys.contains(CaptureResult.CONTROL_ZOOM_RATIO) && mPerFrameControl) {
2077                 Float zoomRatio = request.get(CaptureRequest.CONTROL_ZOOM_RATIO);
2078                 if (zoomRatio != null) {
2079                     Float resultZoomRatio = result.get(CaptureResult.CONTROL_ZOOM_RATIO);
2080                     mCollector.expectTrue(
2081                             String.format("Request and result zoom ratio should be similar " +
2082                                     "(requested = %f, result = %f", zoomRatio, resultZoomRatio),
2083                             Math.abs(zoomRatio - resultZoomRatio) / zoomRatio <= ZOOM_ERROR_MARGIN);
2084                 }
2085             }
2086 
2087             if (mFlashStateListener != null) {
2088                 Integer flashMode = request.get(CaptureRequest.FLASH_MODE);
2089                 if ((flashMode != null) && mPerFrameControl) {
2090                     Integer resultFlashMode = result.get(CaptureResult.FLASH_MODE);
2091                     mCollector.expectTrue("Request flash mode: " + flashMode +
2092                                     " doesn't match with result: " + resultFlashMode,
2093                             flashMode.equals(resultFlashMode));
2094                 }
2095 
2096                 Integer flashState = result.get(CaptureResult.FLASH_STATE);
2097                 if (flashState != null) {
2098                     switch (flashState) {
2099                         case CaptureResult.FLASH_STATE_UNAVAILABLE:
2100                             mFlashStateListener.onUnavailable();
2101                             break;
2102                         case CaptureResult.FLASH_STATE_FIRED:
2103                             mFlashStateListener.onFired();
2104                             break;
2105                         case CaptureResult.FLASH_STATE_CHARGING:
2106                             mFlashStateListener.onCharging();
2107                             break;
2108                         case CaptureResult.FLASH_STATE_PARTIAL:
2109                             mFlashStateListener.onPartial();
2110                             break;
2111                         case CaptureResult.FLASH_STATE_READY:
2112                             mFlashStateListener.onReady();
2113                             break;
2114                         default:
2115                             mCollector.addMessage("Unexpected flash state: " + flashState);
2116                     }
2117                 }
2118             }
2119 
2120             if (mAEStateListener != null) {
2121                 Integer aeMode = request.get(CaptureRequest.CONTROL_AE_MODE);
2122                 if ((aeMode != null) && mPerFrameControl) {
2123                     Integer resultAeMode = result.get(CaptureResult.CONTROL_AE_MODE);
2124                     mCollector.expectTrue("Request AE mode: " + aeMode +
2125                                     " doesn't match with result: " + resultAeMode,
2126                             aeMode.equals(resultAeMode));
2127                 }
2128 
2129                 Boolean aeLock = request.get(CaptureRequest.CONTROL_AE_LOCK);
2130                 if ((aeLock != null) && mPerFrameControl) {
2131                     Boolean resultAeLock = result.get(CaptureResult.CONTROL_AE_LOCK);
2132                     mCollector.expectTrue("Request AE lock: " + aeLock +
2133                                     " doesn't match with result: " + resultAeLock,
2134                             aeLock.equals(resultAeLock));
2135                 }
2136 
2137                 Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
2138                 if (aeState != null) {
2139                     switch (aeState) {
2140                         case CaptureResult.CONTROL_AE_STATE_CONVERGED:
2141                             mAEStateListener.onConverged();
2142                             break;
2143                         case CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED:
2144                             mAEStateListener.onFlashRequired();
2145                             break;
2146                         case CaptureResult.CONTROL_AE_STATE_INACTIVE:
2147                             mAEStateListener.onInactive();
2148                             break;
2149                         case CaptureResult.CONTROL_AE_STATE_LOCKED:
2150                             mAEStateListener.onLocked();
2151                             break;
2152                         case CaptureResult.CONTROL_AE_STATE_PRECAPTURE:
2153                             mAEStateListener.onPrecapture();
2154                             break;
2155                         case CaptureResult.CONTROL_AE_STATE_SEARCHING:
2156                             mAEStateListener.onSearching();
2157                             break;
2158                         default:
2159                             mCollector.addMessage("Unexpected AE state: " + aeState);
2160                     }
2161                 }
2162             }
2163 
2164             if (mAFStateMachine != null) {
2165                 Integer afMode = request.get(CaptureRequest.CONTROL_AF_MODE);
2166                 if ((afMode != null) && mPerFrameControl) {
2167                     Integer resultAfMode = result.get(CaptureResult.CONTROL_AF_MODE);
2168                     mCollector.expectTrue("Request AF mode: " + afMode +
2169                                     " doesn't match with result: " + resultAfMode,
2170                             afMode.equals(resultAfMode));
2171                 }
2172 
2173                 MeteringRectangle[] afRegions = request.get(CaptureRequest.CONTROL_AF_REGIONS);
2174                 if ((afRegions != null) && mPerFrameControl) {
2175                     MeteringRectangle[] resultAfRegions = result.get(
2176                             CaptureResult.CONTROL_AF_REGIONS);
2177                     mCollector.expectMeteringRegionsAreSimilar(
2178                             "AF regions in result and request should be similar",
2179                             afRegions, resultAfRegions, METERING_REGION_ERROR_PERCENT_DELTA);
2180                 }
2181 
2182                 Integer afTrigger = request.get(CaptureRequest.CONTROL_AF_TRIGGER);
2183                 if ((afTrigger != null) && mPerFrameControl) {
2184                     Integer resultAfTrigger = result.get(CaptureResult.CONTROL_AF_TRIGGER);
2185                     mCollector.expectTrue("Request AF trigger: " + afTrigger +
2186                                     " doesn't match with result: " + resultAfTrigger,
2187                             afTrigger.equals(resultAfTrigger));
2188                 }
2189 
2190                 mAFStateMachine.onCaptureCompleted(result);
2191             }
2192 
2193             if (EFV_API_SUPPORTED) {
2194                 verifyEFVExtensionSpecificCaptureResults(result, request);
2195             }
2196 
2197             if (mProxy != null) {
2198                 mProxy.onCaptureResultAvailable(session, request, result);
2199             }
2200         }
2201 
getTotalFramesArrived()2202         public int getTotalFramesArrived() {
2203             return mNumFramesArrived;
2204         }
2205 
getTotalFramesStarted()2206         public int getTotalFramesStarted() {
2207             return mNumFramesStarted;
2208         }
2209 
getTotalFramesFailed()2210         public int getTotalFramesFailed() {
2211             return mNumFramesFailed;
2212         }
2213 
getLastTimestamp()2214         public long getLastTimestamp() throws IllegalStateException {
2215             if (mNonIncreasingTimestamps) {
2216                 throw new IllegalStateException("Non-monotonically increasing timestamps!");
2217             }
2218             return mLastTimestamp;
2219         }
2220 
verifyEFVExtensionSpecificCaptureResults(TotalCaptureResult result, CaptureRequest request)2221         private void verifyEFVExtensionSpecificCaptureResults(TotalCaptureResult result,
2222                 CaptureRequest request) {
2223             if (mExtensionType != CameraExtensionCharacteristics.EXTENSION_EYES_FREE_VIDEOGRAPHY) {
2224                 return;
2225             }
2226 
2227             Boolean enableAutoZoomRequest = request.get(ExtensionCaptureRequest.EFV_AUTO_ZOOM);
2228             if (enableAutoZoomRequest != null) {
2229                 Boolean enableAutoZoomResult = result.get(ExtensionCaptureResult.EFV_AUTO_ZOOM);
2230                 mCollector.expectTrue("Request auto zoom doesn't match with "
2231                         + " result auto zoom mode ",
2232                         enableAutoZoomRequest.equals(enableAutoZoomResult));
2233 
2234                 if (!enableAutoZoomRequest) {
2235                     int[] autoZoomPaddingRegion =
2236                             result.get(ExtensionCaptureResult.EFV_AUTO_ZOOM_PADDING_REGION);
2237                     mCollector.expectTrue("Unexpected EFV_AUTO_ZOOM_PADDING_REGION when "
2238                             + "auto zoom is disabled ", autoZoomPaddingRegion == null);
2239                 }
2240             }
2241 
2242             Integer stabilizationModeRequest =
2243                     request.get(ExtensionCaptureRequest.EFV_STABILIZATION_MODE);
2244             if (stabilizationModeRequest != null) {
2245                 Integer stabilizationModeResult =
2246                         result.get(ExtensionCaptureResult.EFV_STABILIZATION_MODE);
2247                 mCollector.expectTrue("Request stabilization mode doesn't match with "
2248                         + " result stabilization mode ",
2249                         stabilizationModeRequest.equals(stabilizationModeResult));
2250 
2251                 if (stabilizationModeRequest
2252                         == ExtensionCaptureRequest.EFV_STABILIZATION_MODE_LOCKED) {
2253                     int[] paddingRegion =
2254                             result.get(ExtensionCaptureResult.EFV_PADDING_REGION);
2255                     mCollector.expectTrue("Expected EFV_PADDING_REGION when "
2256                             + " stabilization is in locked mode ", paddingRegion != null);
2257                     PointF[] targetCoordinates =
2258                             result.get(ExtensionCaptureResult.EFV_TARGET_COORDINATES);
2259                     mCollector.expectTrue("Expected EFV_TARGET_COORDINATES when "
2260                             + " stabilization is in locked mode ", targetCoordinates != null);
2261 
2262                     if (enableAutoZoomRequest != null && enableAutoZoomRequest) {
2263                         int[] autoZoomPaddingRegion =
2264                                 result.get(ExtensionCaptureResult.EFV_AUTO_ZOOM_PADDING_REGION);
2265                         mCollector.expectTrue("Expected EFV_AUTO_ZOOM_PADDING_REGION when "
2266                                 + "auto zoom is enabled and stabilization is in locked mode ",
2267                                 autoZoomPaddingRegion != null);
2268                     }
2269                 }
2270             }
2271 
2272             Float paddingZoomFactor = result.get(ExtensionCaptureResult.EFV_PADDING_ZOOM_FACTOR);
2273             if (paddingZoomFactor != null) {
2274                 mCollector.expectTrue("Expected EFV_PADDING_ZOOM_FACTOR > 1 ",
2275                         paddingZoomFactor > 1);
2276             }
2277 
2278             if (mStaticInfo == null) {
2279                 return;
2280             }
2281 
2282             Rect activeArraySize = mStaticInfo.getActiveArraySizeChecked();
2283             PointF[] targetCoordinates = result.get(ExtensionCaptureResult.EFV_TARGET_COORDINATES);
2284             if (targetCoordinates != null) {
2285                 for (PointF point: targetCoordinates) {
2286                     mCollector.expectTrue("Target coordinate not within active array region",
2287                             activeArraySize.contains((int) Math.ceil(point.x),
2288                             (int) Math.ceil(point.y)));
2289                 }
2290             }
2291         }
2292     }
2293 
2294     public interface AutoFocusStateListener {
onDone(boolean success)2295         void onDone(boolean success);
onScan()2296         void onScan();
onInactive()2297         void onInactive();
2298     }
2299 
2300     private class TestAutoFocusProxy implements AutoFocusStateMachine.AutoFocusStateListener {
2301         private final AutoFocusStateListener mListener;
2302 
TestAutoFocusProxy(AutoFocusStateListener listener)2303         TestAutoFocusProxy(AutoFocusStateListener listener) {
2304             mListener = listener;
2305         }
2306 
2307         @Override
onAutoFocusSuccess(CaptureResult result, boolean locked)2308         public void onAutoFocusSuccess(CaptureResult result, boolean locked) {
2309             mListener.onDone(true);
2310         }
2311 
2312         @Override
onAutoFocusFail(CaptureResult result, boolean locked)2313         public void onAutoFocusFail(CaptureResult result, boolean locked) {
2314             mListener.onDone(false);
2315         }
2316 
2317         @Override
onAutoFocusScan(CaptureResult result)2318         public void onAutoFocusScan(CaptureResult result) {
2319             mListener.onScan();
2320         }
2321 
2322         @Override
onAutoFocusInactive(CaptureResult result)2323         public void onAutoFocusInactive(CaptureResult result) {
2324             mListener.onInactive();
2325         }
2326     }
2327 
2328     // Verify that camera extension sessions can support AF and AF metering controls. The test
2329     // goal is to check that AF related controls and results are supported and can be used to
2330     // lock the AF state and not to do an exhaustive check of the AF state transitions or manual AF.
2331     @Test
testAFMetering()2332     public void testAFMetering() throws Exception {
2333         final int METERING_REGION_SCALE_RATIO = 8;
2334         for (String id : getCameraIdsUnderTest()) {
2335             StaticMetadata staticMeta =
2336                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2337             if (!staticMeta.isColorOutputSupported()) {
2338                 continue;
2339             }
2340             if (!staticMeta.hasFocuser()) {
2341                 continue;
2342             }
2343 
2344             boolean perFrameControl = staticMeta.isPerFrameControlSupported();
2345             final Rect activeArraySize = staticMeta.getActiveArraySizeChecked();
2346             int regionWidth = activeArraySize.width() / METERING_REGION_SCALE_RATIO - 1;
2347             int regionHeight = activeArraySize.height() / METERING_REGION_SCALE_RATIO - 1;
2348             int centerX = activeArraySize.width() / 2;
2349             int centerY = activeArraySize.height() / 2;
2350 
2351             // Center region
2352             MeteringRectangle[] afMeteringRects = {new MeteringRectangle(
2353                     centerX - regionWidth / 2, centerY - regionHeight / 2,
2354                     regionWidth, regionHeight,
2355                     MeteringRectangle.METERING_WEIGHT_MAX)};
2356 
2357             updatePreviewSurfaceTexture();
2358             CameraExtensionCharacteristics extensionChars =
2359                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2360             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2361             for (Integer extension : supportedExtensions) {
2362                 Set<CaptureRequest.Key> supportedRequestKeys =
2363                         extensionChars.getAvailableCaptureRequestKeys(extension);
2364                 // The night extension is required to support AF controls starting with Android V
2365                 if ((Build.VERSION.DEVICE_INITIAL_SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM)
2366                     && (extension == CameraExtensionCharacteristics.EXTENSION_NIGHT)) {
2367                     assertTrue(supportedRequestKeys.containsAll(
2368                             Arrays.asList(FOCUS_CAPTURE_REQUEST_SET)));
2369                 } else if (!supportedRequestKeys.containsAll(
2370                         Arrays.asList(FOCUS_CAPTURE_REQUEST_SET))) {
2371                     continue;
2372                 }
2373                 Set<CaptureResult.Key> supportedResultKeys =
2374                         extensionChars.getAvailableCaptureResultKeys(extension);
2375                 assertTrue(supportedResultKeys.containsAll(
2376                         Arrays.asList(FOCUS_CAPTURE_RESULT_SET)));
2377 
2378                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
2379                         mSurfaceTexture.getClass());
2380                 Size maxSize =
2381                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
2382                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
2383                         maxSize.getHeight());
2384                 Surface texturedSurface = new Surface(mSurfaceTexture);
2385 
2386                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2387                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2388 
2389                 BlockingExtensionSessionCallback sessionListener =
2390                         new BlockingExtensionSessionCallback(mock(
2391                                 CameraExtensionSession.StateCallback.class));
2392                 ExtensionSessionConfiguration configuration =
2393                         new ExtensionSessionConfiguration(extension, outputConfigs,
2394                                 new HandlerExecutor(mTestRule.getHandler()),
2395                                 sessionListener);
2396 
2397                 try {
2398                     mTestRule.openDevice(id);
2399                     CameraDevice camera = mTestRule.getCamera();
2400                     camera.createExtensionSession(configuration);
2401                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
2402                             SESSION_CONFIGURE_TIMEOUT_MS);
2403                     assertNotNull(extensionSession);
2404 
2405                     // Check passive AF
2406                     AutoFocusStateListener mockAFListener =
2407                             mock(AutoFocusStateListener.class);
2408                     AutoFocusStateMachine afState = new AutoFocusStateMachine(
2409                             new TestAutoFocusProxy(mockAFListener));
2410                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2411                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2412                     SimpleCaptureCallback repeatingCaptureCallback =
2413                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2414                                     extensionChars.getAvailableCaptureResultKeys(extension),
2415                                     mCollector, afState, null /*flashState*/,
2416                                     null /*aeState*/, perFrameControl);
2417 
2418                     CaptureRequest.Builder captureBuilder =
2419                             mTestRule.getCamera().createCaptureRequest(
2420                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2421                     captureBuilder.addTarget(texturedSurface);
2422                     afState.setPassiveAutoFocus(true /*picture*/, captureBuilder);
2423                     captureBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects);
2424                     CaptureRequest request = captureBuilder.build();
2425                     int passiveSequenceId = extensionSession.setRepeatingRequest(request,
2426                             new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback);
2427                     assertTrue(passiveSequenceId > 0);
2428 
2429                     verify(repeatingCallbackMock,
2430                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2431                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
2432                                     any(TotalCaptureResult.class));
2433 
2434                     verify(mockAFListener,
2435                             timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS).atLeastOnce())
2436                             .onDone(anyBoolean());
2437 
2438                     // Check active AF
2439                     mockAFListener = mock(AutoFocusStateListener.class);
2440                     CameraExtensionSession.ExtensionCaptureCallback callbackMock = mock(
2441                             CameraExtensionSession.ExtensionCaptureCallback.class);
2442                     AutoFocusStateMachine activeAFState = new AutoFocusStateMachine(
2443                             new TestAutoFocusProxy(mockAFListener));
2444                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
2445                             new SimpleCaptureCallback(extension, callbackMock,
2446                                     extensionChars.getAvailableCaptureResultKeys(extension),
2447                                     mCollector, activeAFState, null /*flashState*/,
2448                                     null /*aeState*/, perFrameControl);
2449 
2450                     CaptureRequest.Builder triggerBuilder =
2451                             mTestRule.getCamera().createCaptureRequest(
2452                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2453                     triggerBuilder.addTarget(texturedSurface);
2454                     afState.setActiveAutoFocus(captureBuilder, triggerBuilder);
2455                     afState.unlockAutoFocus(captureBuilder, triggerBuilder);
2456                     triggerBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, afMeteringRects);
2457                     request = captureBuilder.build();
2458                     int activeSequenceId = extensionSession.setRepeatingRequest(request,
2459                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2460                     assertTrue((activeSequenceId > 0) &&
2461                             (activeSequenceId != passiveSequenceId));
2462 
2463                     verify(callbackMock,
2464                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2465                             .onCaptureResultAvailable(eq(extensionSession), eq(request),
2466                                     any(TotalCaptureResult.class));
2467 
2468                     CaptureRequest triggerRequest = triggerBuilder.build();
2469                     reset(mockAFListener);
2470                     int triggerSequenceId = extensionSession.capture(triggerRequest,
2471                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2472                     assertTrue((triggerSequenceId > 0) &&
2473                             (activeSequenceId != triggerSequenceId) &&
2474                             (triggerSequenceId != passiveSequenceId));
2475 
2476                     verify(callbackMock,
2477                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2478                             .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest),
2479                                     any(TotalCaptureResult.class));
2480 
2481                     afState.lockAutoFocus(captureBuilder, triggerBuilder);
2482                     triggerRequest = triggerBuilder.build();
2483                     reset(mockAFListener);
2484                     extensionSession.capture(triggerRequest,
2485                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2486 
2487                     verify(callbackMock,
2488                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2489                             .onCaptureResultAvailable(eq(extensionSession), eq(triggerRequest),
2490                                     any(TotalCaptureResult.class));
2491 
2492                     verify(mockAFListener, timeout(WAIT_FOR_FOCUS_DONE_TIMEOUT_MS)
2493                             .atLeast(1)).onDone(anyBoolean());
2494 
2495                     extensionSession.stopRepeating();
2496 
2497                     extensionSession.close();
2498 
2499                     sessionListener.getStateWaiter().waitForState(
2500                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2501                             SESSION_CLOSE_TIMEOUT_MS);
2502                 } finally {
2503                     mTestRule.closeDevice(id);
2504                     texturedSurface.release();
2505                 }
2506             }
2507         }
2508     }
2509 
2510     // Verify that camera extension sessions can support the zoom ratio control.
2511     @Test
testZoomRatio()2512     public void testZoomRatio() throws Exception {
2513         final int ZOOM_RATIO_STEPS = 10;
2514         for (String id : getCameraIdsUnderTest()) {
2515             StaticMetadata staticMeta =
2516                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2517             if (!staticMeta.isColorOutputSupported()) {
2518                 continue;
2519             }
2520 
2521             CameraExtensionCharacteristics extensionChars =
2522                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2523 
2524             updatePreviewSurfaceTexture();
2525             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2526             for (Integer extension : supportedExtensions) {
2527                 final Range<Float> zoomRatioRange;
2528                 final float maxZoom;
2529                 if (Build.VERSION.DEVICE_INITIAL_SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM) {
2530                     zoomRatioRange =
2531                             extensionChars.get(extension,
2532                                     CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE);
2533                     assertNotNull(
2534                             "Zoom ratio range must be present in CameraExtensionCharacteristics",
2535                             zoomRatioRange);
2536                     maxZoom = zoomRatioRange.getUpper();
2537                 } else {
2538                     zoomRatioRange = staticMeta.getZoomRatioRangeChecked();
2539                     maxZoom = staticMeta.getAvailableMaxDigitalZoomChecked();
2540                 }
2541 
2542                 if (zoomRatioRange.getUpper().equals(zoomRatioRange.getLower())) {
2543                     continue;
2544                 }
2545 
2546                 if (Math.abs(maxZoom - 1.0f) < ZOOM_ERROR_MARGIN) {
2547                     return;
2548                 }
2549 
2550                 Float zoomStep  =
2551                         (zoomRatioRange.getUpper() - zoomRatioRange.getLower()) / ZOOM_RATIO_STEPS;
2552                 if (zoomStep < ZOOM_ERROR_MARGIN) {
2553                     continue;
2554                 }
2555 
2556                 ArrayList<Float> candidateZoomRatios = new ArrayList<>(ZOOM_RATIO_STEPS);
2557                 for (int step = 0; step < (ZOOM_RATIO_STEPS - 1); step++) {
2558                     candidateZoomRatios.add(step, zoomRatioRange.getLower() + step * zoomStep);
2559                 }
2560                 candidateZoomRatios.add(ZOOM_RATIO_STEPS - 1, zoomRatioRange.getUpper());
2561 
2562                 Set<CaptureRequest.Key> supportedRequestKeys =
2563                         extensionChars.getAvailableCaptureRequestKeys(extension);
2564                 // The Night extension is required to support zoom controls start with Android V
2565                 if ((Build.VERSION.DEVICE_INITIAL_SDK_INT >= Build.VERSION_CODES.VANILLA_ICE_CREAM)
2566                         && (extension == CameraExtensionCharacteristics.EXTENSION_NIGHT)) {
2567                     assertTrue(supportedRequestKeys.containsAll(
2568                             Arrays.asList(ZOOM_CAPTURE_REQUEST_SET)));
2569                 } else if (!supportedRequestKeys.containsAll(
2570                         Arrays.asList(ZOOM_CAPTURE_REQUEST_SET))) {
2571                     continue;
2572                 }
2573                 Set<CaptureResult.Key> supportedResultKeys =
2574                         extensionChars.getAvailableCaptureResultKeys(extension);
2575                 assertTrue(supportedResultKeys.containsAll(
2576                         Arrays.asList(ZOOM_CAPTURE_RESULT_SET)));
2577 
2578                 List<Size> extensionSizes = extensionChars.getExtensionSupportedSizes(extension,
2579                         mSurfaceTexture.getClass());
2580                 Size maxSize =
2581                         CameraTestUtils.getMaxSize(extensionSizes.toArray(new Size[0]));
2582                 mSurfaceTexture.setDefaultBufferSize(maxSize.getWidth(),
2583                         maxSize.getHeight());
2584                 Surface texturedSurface = new Surface(mSurfaceTexture);
2585 
2586                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2587                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2588 
2589                 BlockingExtensionSessionCallback sessionListener =
2590                         new BlockingExtensionSessionCallback(mock(
2591                                 CameraExtensionSession.StateCallback.class));
2592                 ExtensionSessionConfiguration configuration =
2593                         new ExtensionSessionConfiguration(extension, outputConfigs,
2594                                 new HandlerExecutor(mTestRule.getHandler()),
2595                                 sessionListener);
2596 
2597                 try {
2598                     mTestRule.openDevice(id);
2599                     CameraDevice camera = mTestRule.getCamera();
2600                     camera.createExtensionSession(configuration);
2601                     CameraExtensionSession extensionSession = sessionListener.waitAndGetSession(
2602                             SESSION_CONFIGURE_TIMEOUT_MS);
2603                     assertNotNull(extensionSession);
2604 
2605                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2606                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2607                     SimpleCaptureCallback repeatingCaptureCallback =
2608                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2609                                     extensionChars.getAvailableCaptureResultKeys(extension),
2610                                     mCollector);
2611 
2612                     CaptureRequest.Builder captureBuilder =
2613                             mTestRule.getCamera().createCaptureRequest(
2614                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2615                     captureBuilder.addTarget(texturedSurface);
2616                     for (Float currentZoomRatio : candidateZoomRatios) {
2617                         captureBuilder.set(CaptureRequest.CONTROL_ZOOM_RATIO, currentZoomRatio);
2618                         CaptureRequest request = captureBuilder.build();
2619 
2620                         int seqId = extensionSession.setRepeatingRequest(request,
2621                                 new HandlerExecutor(mTestRule.getHandler()),
2622                                 repeatingCaptureCallback);
2623                         assertTrue(seqId > 0);
2624 
2625                         verify(repeatingCallbackMock,
2626                                 timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2627                                 .onCaptureResultAvailable(eq(extensionSession), eq(request),
2628                                         any(TotalCaptureResult.class));
2629                     }
2630 
2631                     extensionSession.stopRepeating();
2632 
2633                     extensionSession.close();
2634 
2635                     sessionListener.getStateWaiter().waitForState(
2636                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2637                             SESSION_CLOSE_TIMEOUT_MS);
2638                 } finally {
2639                     mTestRule.closeDevice(id);
2640                     texturedSurface.release();
2641                 }
2642             }
2643         }
2644     }
2645 
2646     public interface FlashStateListener {
onFired()2647         public void onFired();
onReady()2648         public void onReady();
onCharging()2649         public void onCharging();
onPartial()2650         public void onPartial();
onUnavailable()2651         public void onUnavailable();
2652     }
2653 
2654     public interface AutoExposureStateListener {
onInactive()2655       public void onInactive();
onSearching()2656       public void onSearching();
onConverged()2657       public void onConverged();
onLocked()2658       public void onLocked();
onFlashRequired()2659       public void onFlashRequired();
onPrecapture()2660       public void onPrecapture();
2661     }
2662 
2663     // Verify that camera extension sessions can support Flash related controls. The test
2664     // goal is to check that Flash controls and results are supported and can be used to
2665     // turn on torch, run the pre-capture sequence and active the main flash.
2666     @Test
testFlash()2667     public void testFlash() throws Exception {
2668         final CaptureRequest.Key[] FLASH_CAPTURE_REQUEST_SET = {CaptureRequest.CONTROL_AE_MODE,
2669                 CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_LOCK,
2670                 CaptureRequest.FLASH_MODE};
2671         final CaptureResult.Key[] FLASH_CAPTURE_RESULT_SET = {CaptureResult.CONTROL_AE_MODE,
2672                 CaptureResult.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureResult.CONTROL_AE_LOCK,
2673                 CaptureResult.CONTROL_AE_STATE, CaptureResult.FLASH_MODE,
2674                 CaptureResult.FLASH_STATE};
2675         for (String id : getCameraIdsUnderTest()) {
2676             StaticMetadata staticMeta =
2677                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2678             if (!staticMeta.isColorOutputSupported()) {
2679                 continue;
2680             }
2681             if (!staticMeta.hasFlash()) {
2682                 continue;
2683             }
2684 
2685             boolean perFrameControl = staticMeta.isPerFrameControlSupported();
2686             updatePreviewSurfaceTexture();
2687             CameraExtensionCharacteristics extensionChars =
2688                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2689             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2690             for (Integer extension : supportedExtensions) {
2691                 Set<CaptureRequest.Key> supportedRequestKeys =
2692                         extensionChars.getAvailableCaptureRequestKeys(extension);
2693                 if (!supportedRequestKeys.containsAll(Arrays.asList(FLASH_CAPTURE_REQUEST_SET))) {
2694                     continue;
2695                 }
2696                 Set<CaptureResult.Key> supportedResultKeys =
2697                         extensionChars.getAvailableCaptureResultKeys(extension);
2698                 assertTrue(supportedResultKeys.containsAll(
2699                         Arrays.asList(FLASH_CAPTURE_RESULT_SET)));
2700 
2701                 int captureFormat = ImageFormat.JPEG;
2702                 List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2703                         captureFormat);
2704                 assertFalse("No Jpeg output supported", captureSizes.isEmpty());
2705                 Size captureMaxSize = captureSizes.get(0);
2706 
2707                 SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false, 1);
2708                 ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
2709                         captureMaxSize, captureFormat, /*maxImages*/ 1, imageListener,
2710                         mTestRule.getHandler());
2711                 Surface imageReaderSurface = extensionImageReader.getSurface();
2712                 OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
2713                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2714                 outputConfigs.add(readerOutput);
2715 
2716                 List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
2717                         mSurfaceTexture.getClass());
2718                 Size previewSize = repeatingSizes.get(0);
2719 
2720                 mSurfaceTexture.setDefaultBufferSize(previewSize.getWidth(),
2721                         previewSize.getHeight());
2722                 Surface texturedSurface = new Surface(mSurfaceTexture);
2723                 outputConfigs.add(new OutputConfiguration(texturedSurface));
2724 
2725                 BlockingExtensionSessionCallback sessionListener =
2726                         new BlockingExtensionSessionCallback(mock(
2727                                 CameraExtensionSession.StateCallback.class));
2728                 ExtensionSessionConfiguration configuration =
2729                         new ExtensionSessionConfiguration(extension, outputConfigs,
2730                                 new HandlerExecutor(mTestRule.getHandler()),
2731                                 sessionListener);
2732                 try {
2733                     mTestRule.openDevice(id);
2734                     CameraDevice camera = mTestRule.getCamera();
2735                     camera.createExtensionSession(configuration);
2736                     CameraExtensionSession extensionSession =
2737                             sessionListener.waitAndGetSession(
2738                                     SESSION_CONFIGURE_TIMEOUT_MS);
2739                     assertNotNull(extensionSession);
2740 
2741                     // Test torch on and off
2742                     CaptureRequest.Builder captureBuilder =
2743                             mTestRule.getCamera().createCaptureRequest(
2744                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
2745                     captureBuilder.addTarget(texturedSurface);
2746                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2747                             CameraMetadata.CONTROL_AE_MODE_ON);
2748                     captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false);
2749                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_TORCH);
2750                     FlashStateListener mockFlashListener = mock(FlashStateListener.class);
2751                     AutoExposureStateListener mockAEStateListener =
2752                             mock(AutoExposureStateListener.class);
2753                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
2754                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2755                     SimpleCaptureCallback repeatingCaptureCallback =
2756                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
2757                                     extensionChars.getAvailableCaptureResultKeys(extension),
2758                                     mCollector, null /*afState*/, mockFlashListener,
2759                                     mockAEStateListener, perFrameControl);
2760                     CaptureRequest repeatingRequest = captureBuilder.build();
2761                     int repeatingSequenceId =
2762                             extensionSession.setRepeatingRequest(repeatingRequest,
2763                                     new HandlerExecutor(mTestRule.getHandler()),
2764                                     repeatingCaptureCallback);
2765                     assertTrue(repeatingSequenceId > 0);
2766 
2767                     verify(repeatingCallbackMock,
2768                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2769                             .onCaptureResultAvailable(eq(extensionSession),
2770                                     eq(repeatingRequest), any(TotalCaptureResult.class));
2771                     verify(mockFlashListener,
2772                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired();
2773 
2774                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2775                             CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH);
2776                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF);
2777                     repeatingRequest = captureBuilder.build();
2778                     reset(mockFlashListener);
2779                     repeatingSequenceId = extensionSession.setRepeatingRequest(repeatingRequest,
2780                                     new HandlerExecutor(mTestRule.getHandler()),
2781                                     repeatingCaptureCallback);
2782                     assertTrue(repeatingSequenceId > 0);
2783 
2784                     verify(repeatingCallbackMock,
2785                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2786                             .onCaptureResultAvailable(eq(extensionSession),
2787                                     eq(repeatingRequest), any(TotalCaptureResult.class));
2788                     verify(mockFlashListener,
2789                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onReady();
2790 
2791                     // Test AE pre-capture sequence
2792                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_OFF);
2793                     captureBuilder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER,
2794                             CameraMetadata.CONTROL_AE_PRECAPTURE_TRIGGER_START);
2795                     CaptureRequest triggerRequest = captureBuilder.build();
2796                     int triggerSeqId = extensionSession.capture(triggerRequest,
2797                             new HandlerExecutor(mTestRule.getHandler()), repeatingCaptureCallback);
2798                     assertTrue(triggerSeqId > 0);
2799 
2800                     verify(repeatingCallbackMock,
2801                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce())
2802                             .onCaptureResultAvailable(eq(extensionSession),
2803                                     eq(triggerRequest), any(TotalCaptureResult.class));
2804                     verify(mockAEStateListener,
2805                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onPrecapture();
2806                     verify(mockAEStateListener,
2807                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onConverged();
2808 
2809                     // Test main flash
2810                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
2811                             android.hardware.camera2.CameraDevice.TEMPLATE_STILL_CAPTURE);
2812                     captureBuilder.addTarget(imageReaderSurface);
2813                     captureBuilder.set(CaptureRequest.CONTROL_AE_MODE,
2814                             CameraMetadata.CONTROL_AE_MODE_ON_ALWAYS_FLASH);
2815                     captureBuilder.set(CaptureRequest.CONTROL_AE_LOCK, true);
2816                     captureBuilder.set(CaptureRequest.FLASH_MODE, CameraMetadata.FLASH_MODE_SINGLE);
2817                     CameraExtensionSession.ExtensionCaptureCallback mockCaptureCallback =
2818                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
2819                     reset(mockFlashListener);
2820                     SimpleCaptureCallback captureCallback =
2821                             new SimpleCaptureCallback(extension, mockCaptureCallback,
2822                                     extensionChars.getAvailableCaptureResultKeys(extension),
2823                                     mCollector, null /*afState*/, mockFlashListener,
2824                                     mockAEStateListener, perFrameControl);
2825 
2826                     CaptureRequest captureRequest = captureBuilder.build();
2827                     int captureSequenceId = extensionSession.capture(captureRequest,
2828                             new HandlerExecutor(mTestRule.getHandler()), captureCallback);
2829                     assertTrue(captureSequenceId > 0);
2830 
2831                     Image img =
2832                             imageListener.getImage(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS);
2833                     validateImage(img, captureMaxSize.getWidth(),
2834                             captureMaxSize.getHeight(), captureFormat, null);
2835                     long imgTs = img.getTimestamp();
2836                     img.close();
2837 
2838                     verify(mockCaptureCallback,
2839                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
2840                             .onCaptureStarted(eq(extensionSession), eq(captureRequest), eq(imgTs));
2841                     verify(mockCaptureCallback,
2842                             timeout(MULTI_FRAME_CAPTURE_IMAGE_TIMEOUT_MS).times(1))
2843                             .onCaptureResultAvailable(eq(extensionSession),
2844                                     eq(captureRequest), any(TotalCaptureResult.class));
2845                     verify(mockFlashListener,
2846                             timeout(REPEATING_REQUEST_TIMEOUT_MS).atLeastOnce()).onFired();
2847 
2848                     extensionSession.stopRepeating();
2849 
2850                     extensionSession.close();
2851 
2852                     sessionListener.getStateWaiter().waitForState(
2853                             BlockingExtensionSessionCallback.SESSION_CLOSED,
2854                             SESSION_CLOSE_TIMEOUT_MS);
2855                 } finally {
2856                     mTestRule.closeDevice(id);
2857                     texturedSurface.release();
2858                     extensionImageReader.close();
2859                 }
2860             }
2861         }
2862     }
2863 
2864     // Verify 'CameraExtensionSession.StillCaptureLatency' behavior
2865     @Test
testSessionStillCaptureLatency()2866     public void testSessionStillCaptureLatency() throws Exception {
2867         final long CAPTURE_LATENCY_MS = 100;
2868         final long PROCESSING_LATENCY_MS = 200;
2869         final long DIFFERENT_PROCESSING_LATENCY_MS = 201;
2870         CameraExtensionSession.StillCaptureLatency stillCaptureLatency =
2871                 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS,
2872                         PROCESSING_LATENCY_MS);
2873         assertEquals(stillCaptureLatency.getCaptureLatency(), CAPTURE_LATENCY_MS);
2874         assertEquals(stillCaptureLatency.getProcessingLatency(), PROCESSING_LATENCY_MS);
2875         assertNotNull(stillCaptureLatency.toString());
2876         CameraExtensionSession.StillCaptureLatency differentStillCaptureLatency =
2877                 new CameraExtensionSession.StillCaptureLatency(CAPTURE_LATENCY_MS,
2878                         DIFFERENT_PROCESSING_LATENCY_MS);
2879         assertFalse(stillCaptureLatency.equals(differentStillCaptureLatency));
2880         assertFalse(stillCaptureLatency.hashCode() ==
2881                 differentStillCaptureLatency.hashCode());
2882     }
2883 
2884     @Test
testIllegalArguments()2885     public void testIllegalArguments() throws Exception {
2886         for (String id : getCameraIdsUnderTest()) {
2887             StaticMetadata staticMeta =
2888                     new StaticMetadata(mTestRule.getCameraManager().getCameraCharacteristics(id));
2889             if (!staticMeta.isColorOutputSupported()) {
2890                 continue;
2891             }
2892             updatePreviewSurfaceTexture();
2893             CameraExtensionCharacteristics extensionChars =
2894                     mTestRule.getCameraManager().getCameraExtensionCharacteristics(id);
2895             List<Integer> supportedExtensions = extensionChars.getSupportedExtensions();
2896             for (Integer extension : supportedExtensions) {
2897                 List<OutputConfiguration> outputConfigs = new ArrayList<>();
2898                 BlockingExtensionSessionCallback sessionListener =
2899                         new BlockingExtensionSessionCallback(mock(
2900                                 CameraExtensionSession.StateCallback.class));
2901                 ExtensionSessionConfiguration configuration =
2902                         new ExtensionSessionConfiguration(extension, outputConfigs,
2903                                 new HandlerExecutor(mTestRule.getHandler()),
2904                                 sessionListener);
2905 
2906                 try {
2907                     mTestRule.openDevice(id);
2908                     CameraDevice camera = mTestRule.getCamera();
2909                     try {
2910                         camera.createExtensionSession(configuration);
2911                         fail("should get IllegalArgumentException due to absent output surfaces");
2912                     } catch (IllegalArgumentException e) {
2913                         // Expected, we can proceed further
2914                     }
2915 
2916                     int captureFormat = ImageFormat.YUV_420_888;
2917                     List<Size> captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2918                             captureFormat);
2919                     if (captureSizes.isEmpty()) {
2920                         captureFormat = ImageFormat.JPEG;
2921                         captureSizes = extensionChars.getExtensionSupportedSizes(extension,
2922                                 captureFormat);
2923                     }
2924                     Size captureMaxSize =
2925                             CameraTestUtils.getMaxSize(captureSizes.toArray(new Size[0]));
2926 
2927                     mSurfaceTexture.setDefaultBufferSize(1, 1);
2928                     Surface texturedSurface = new Surface(mSurfaceTexture);
2929                     outputConfigs.add(new OutputConfiguration(texturedSurface));
2930                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
2931                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
2932 
2933                     try {
2934                         camera.createExtensionSession(configuration);
2935                         fail("should get IllegalArgumentException due to illegal repeating request"
2936                                 + " output surface");
2937                     } catch (IllegalArgumentException e) {
2938                         // Expected, we can proceed further
2939                     } finally {
2940                         outputConfigs.clear();
2941                     }
2942 
2943                     SimpleImageReaderListener imageListener = new SimpleImageReaderListener(false,
2944                             1);
2945                     Size invalidCaptureSize = new Size(1, 1);
2946                     ImageReader extensionImageReader = CameraTestUtils.makeImageReader(
2947                             invalidCaptureSize, captureFormat, /*maxImages*/ 1,
2948                             imageListener, mTestRule.getHandler());
2949                     Surface imageReaderSurface = extensionImageReader.getSurface();
2950                     OutputConfiguration readerOutput = new OutputConfiguration(imageReaderSurface);
2951                     outputConfigs.add(readerOutput);
2952                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
2953                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
2954 
2955                     try{
2956                         camera.createExtensionSession(configuration);
2957                         fail("should get IllegalArgumentException due to illegal multi-frame"
2958                                 + " request output surface");
2959                     } catch (IllegalArgumentException e) {
2960                         // Expected, we can proceed further
2961                     } finally {
2962                         outputConfigs.clear();
2963                         extensionImageReader.close();
2964                     }
2965 
2966                     // Pick a supported preview/repeating size with aspect ratio close to the
2967                     // multi-frame capture size
2968                     List<Size> repeatingSizes = extensionChars.getExtensionSupportedSizes(extension,
2969                             mSurfaceTexture.getClass());
2970                     Size maxRepeatingSize =
2971                             CameraTestUtils.getMaxSize(repeatingSizes.toArray(new Size[0]));
2972                     List<Size> previewSizes = getSupportedPreviewSizes(id,
2973                             mTestRule.getCameraManager(),
2974                             getPreviewSizeBound(mTestRule.getWindowManager(), PREVIEW_SIZE_BOUND));
2975                     List<Size> supportedPreviewSizes =
2976                             previewSizes.stream().filter(repeatingSizes::contains).collect(
2977                                     Collectors.toList());
2978                     if (!supportedPreviewSizes.isEmpty()) {
2979                         float targetAr =
2980                                 ((float) captureMaxSize.getWidth()) / captureMaxSize.getHeight();
2981                         for (Size s : supportedPreviewSizes) {
2982                             float currentAr = ((float) s.getWidth()) / s.getHeight();
2983                             if (Math.abs(targetAr - currentAr) < 0.01) {
2984                                 maxRepeatingSize = s;
2985                                 break;
2986                             }
2987                         }
2988                     }
2989 
2990                     imageListener = new SimpleImageReaderListener(false, 1);
2991                     extensionImageReader = CameraTestUtils.makeImageReader(captureMaxSize,
2992                             captureFormat, /*maxImages*/ 1, imageListener, mTestRule.getHandler());
2993                     imageReaderSurface = extensionImageReader.getSurface();
2994                     readerOutput = new OutputConfiguration(imageReaderSurface);
2995                     outputConfigs.add(readerOutput);
2996 
2997                     mSurfaceTexture.setDefaultBufferSize(maxRepeatingSize.getWidth(),
2998                             maxRepeatingSize.getHeight());
2999                     texturedSurface = new Surface(mSurfaceTexture);
3000                     outputConfigs.add(new OutputConfiguration(texturedSurface));
3001 
3002                     configuration = new ExtensionSessionConfiguration(extension, outputConfigs,
3003                             new HandlerExecutor(mTestRule.getHandler()), sessionListener);
3004                     camera.createExtensionSession(configuration);
3005                     CameraExtensionSession extensionSession =
3006                             sessionListener.waitAndGetSession(
3007                                     SESSION_CONFIGURE_TIMEOUT_MS);
3008                     assertNotNull(extensionSession);
3009 
3010                     CaptureRequest.Builder captureBuilder =
3011                             mTestRule.getCamera().createCaptureRequest(
3012                                     android.hardware.camera2.CameraDevice.TEMPLATE_PREVIEW);
3013                     captureBuilder.addTarget(imageReaderSurface);
3014                     CameraExtensionSession.ExtensionCaptureCallback repeatingCallbackMock =
3015                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
3016                     SimpleCaptureCallback repeatingCaptureCallback =
3017                             new SimpleCaptureCallback(extension, repeatingCallbackMock,
3018                                     extensionChars.getAvailableCaptureResultKeys(extension),
3019                                     mCollector);
3020                     CaptureRequest repeatingRequest = captureBuilder.build();
3021                     try {
3022                         extensionSession.setRepeatingRequest(repeatingRequest,
3023                                 new HandlerExecutor(mTestRule.getHandler()),
3024                                 repeatingCaptureCallback);
3025                         fail("should get IllegalArgumentException due to illegal repeating request"
3026                                 + " output target");
3027                     } catch (IllegalArgumentException e) {
3028                         // Expected, we can proceed further
3029                     }
3030 
3031                     extensionSession.close();
3032 
3033                     sessionListener.getStateWaiter().waitForState(
3034                             BlockingExtensionSessionCallback.SESSION_CLOSED,
3035                             SESSION_CLOSE_TIMEOUT_MS);
3036 
3037                     texturedSurface.release();
3038                     extensionImageReader.close();
3039 
3040                     captureBuilder = mTestRule.getCamera().createCaptureRequest(
3041                             CameraDevice.TEMPLATE_PREVIEW);
3042                     captureBuilder.addTarget(texturedSurface);
3043                     CameraExtensionSession.ExtensionCaptureCallback captureCallback =
3044                             mock(CameraExtensionSession.ExtensionCaptureCallback.class);
3045 
3046                     CaptureRequest captureRequest = captureBuilder.build();
3047                     try {
3048                         extensionSession.setRepeatingRequest(captureRequest,
3049                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
3050                         fail("should get IllegalStateException due to closed session");
3051                     } catch (IllegalStateException e) {
3052                         // Expected, we can proceed further
3053                     }
3054 
3055                     try {
3056                         extensionSession.stopRepeating();
3057                         fail("should get IllegalStateException due to closed session");
3058                     } catch (IllegalStateException e) {
3059                         // Expected, we can proceed further
3060                     }
3061 
3062                     try {
3063                         extensionSession.capture(captureRequest,
3064                                 new HandlerExecutor(mTestRule.getHandler()), captureCallback);
3065                         fail("should get IllegalStateException due to closed session");
3066                     } catch (IllegalStateException e) {
3067                         // Expected, we can proceed further
3068                     }
3069                 } finally {
3070                     mTestRule.closeDevice(id);
3071                 }
3072             }
3073         }
3074     }
3075 }
3076