1 /*
2  * Copyright (C) 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 package android.hardware.camera2.impl;
18 
19 import static com.android.internal.util.function.pooled.PooledLambda.obtainRunnable;
20 
21 import android.annotation.NonNull;
22 import android.annotation.Nullable;
23 import android.app.compat.CompatChanges;
24 import android.compat.annotation.ChangeId;
25 import android.compat.annotation.EnabledSince;
26 import android.content.Context;
27 import android.graphics.ImageFormat;
28 import android.hardware.ICameraService;
29 import android.hardware.camera2.CameraAccessException;
30 import android.hardware.camera2.CameraCaptureSession;
31 import android.hardware.camera2.CameraCharacteristics;
32 import android.hardware.camera2.CameraDevice;
33 import android.hardware.camera2.CameraExtensionCharacteristics;
34 import android.hardware.camera2.CameraManager;
35 import android.hardware.camera2.CameraMetadata;
36 import android.hardware.camera2.CameraOfflineSession;
37 import android.hardware.camera2.CaptureFailure;
38 import android.hardware.camera2.CaptureRequest;
39 import android.hardware.camera2.CaptureResult;
40 import android.hardware.camera2.ICameraDeviceCallbacks;
41 import android.hardware.camera2.ICameraDeviceUser;
42 import android.hardware.camera2.ICameraOfflineSession;
43 import android.hardware.camera2.TotalCaptureResult;
44 import android.hardware.camera2.params.ExtensionSessionConfiguration;
45 import android.hardware.camera2.params.InputConfiguration;
46 import android.hardware.camera2.params.MultiResolutionStreamConfigurationMap;
47 import android.hardware.camera2.params.MultiResolutionStreamInfo;
48 import android.hardware.camera2.params.OutputConfiguration;
49 import android.hardware.camera2.params.SessionConfiguration;
50 import android.hardware.camera2.params.StreamConfigurationMap;
51 import android.hardware.camera2.utils.SubmitInfo;
52 import android.hardware.camera2.utils.SurfaceUtils;
53 import android.os.Binder;
54 import android.os.Build;
55 import android.os.Handler;
56 import android.os.IBinder;
57 import android.os.Looper;
58 import android.os.RemoteException;
59 import android.os.ServiceSpecificException;
60 import android.os.SystemClock;
61 import android.util.Log;
62 import android.util.Range;
63 import android.util.Size;
64 import android.util.SparseArray;
65 import android.view.Surface;
66 
67 import com.android.internal.camera.flags.Flags;
68 
69 import java.util.AbstractMap.SimpleEntry;
70 import java.util.ArrayList;
71 import java.util.Arrays;
72 import java.util.Collection;
73 import java.util.HashMap;
74 import java.util.HashSet;
75 import java.util.Iterator;
76 import java.util.List;
77 import java.util.Map;
78 import java.util.Objects;
79 import java.util.Set;
80 import java.util.concurrent.Executor;
81 import java.util.concurrent.ExecutorService;
82 import java.util.concurrent.Executors;
83 import java.util.concurrent.atomic.AtomicBoolean;
84 
85 /**
86  * HAL2.1+ implementation of CameraDevice. Use CameraManager#open to instantiate
87  */
88 public class CameraDeviceImpl extends CameraDevice
89         implements IBinder.DeathRecipient {
90     private final String TAG;
91     private final boolean DEBUG = false;
92 
93     private static final int REQUEST_ID_NONE = -1;
94 
95     /**
96      * Starting {@link Build.VERSION_CODES#VANILLA_ICE_CREAM},
97      * {@link #isSessionConfigurationSupported} also checks for compatibility of session parameters
98      * when supported by the HAL. This ChangeId guards enabling that functionality for apps
99      * that target {@link Build.VERSION_CODES#VANILLA_ICE_CREAM} and above.
100      */
101     @ChangeId
102     @EnabledSince(targetSdkVersion = Build.VERSION_CODES.VANILLA_ICE_CREAM)
103     static final long CHECK_PARAMS_IN_IS_SESSION_CONFIGURATION_SUPPORTED = 320741775;
104 
105     // TODO: guard every function with if (!mRemoteDevice) check (if it was closed)
106     private ICameraDeviceUserWrapper mRemoteDevice;
107     private boolean mRemoteDeviceInit = false;
108 
109     // CameraDeviceSetup object to delegate some of the newer calls to.
110     @Nullable private final CameraDeviceSetup mCameraDeviceSetup;
111 
112     // Lock to synchronize cross-thread access to device public interface
113     final Object mInterfaceLock = new Object(); // access from this class and Session only!
114     private final CameraDeviceCallbacks mCallbacks = new CameraDeviceCallbacks();
115 
116     private final StateCallback mDeviceCallback;
117     private volatile StateCallbackKK mSessionStateCallback;
118     private final Executor mDeviceExecutor;
119 
120     private final AtomicBoolean mClosing = new AtomicBoolean();
121     private boolean mInError = false;
122     private boolean mIdle = true;
123 
124     /** map request IDs to callback/request data */
125     private SparseArray<CaptureCallbackHolder> mCaptureCallbackMap =
126             new SparseArray<CaptureCallbackHolder>();
127 
128     /** map request IDs which have batchedOutputs to requestCount*/
129     private HashMap<Integer, Integer> mBatchOutputMap = new HashMap<>();
130 
131     private int mRepeatingRequestId = REQUEST_ID_NONE;
132     // Latest repeating request list's types
133     private int[] mRepeatingRequestTypes;
134 
135     // Cache failed requests to process later in case of a repeating error callback
136     private int mFailedRepeatingRequestId = REQUEST_ID_NONE;
137     private int[] mFailedRepeatingRequestTypes;
138 
139     // Map stream IDs to input/output configurations
140     private SimpleEntry<Integer, InputConfiguration> mConfiguredInput =
141             new SimpleEntry<>(REQUEST_ID_NONE, null);
142     private final SparseArray<OutputConfiguration> mConfiguredOutputs =
143             new SparseArray<>();
144 
145     // Cache all stream IDs capable of supporting offline mode.
146     private final HashSet<Integer> mOfflineSupport = new HashSet<>();
147 
148     private final String mCameraId;
149     private final CameraCharacteristics mCharacteristics;
150     private Map<String, CameraCharacteristics> mPhysicalIdsToChars;
151     private final CameraManager mCameraManager;
152     private final int mTotalPartialCount;
153     private final Context mContext;
154 
155     private static final long NANO_PER_SECOND = 1000000000; //ns
156 
157     /**
158      * A list tracking request and its expected last regular/reprocess/zslStill frame
159      * number. Updated when calling ICameraDeviceUser methods.
160      */
161     private final List<RequestLastFrameNumbersHolder> mRequestLastFrameNumbersList =
162             new ArrayList<>();
163 
164     /**
165      * An object tracking received frame numbers.
166      * Updated when receiving callbacks from ICameraDeviceCallbacks.
167      */
168     private FrameNumberTracker mFrameNumberTracker = new FrameNumberTracker();
169 
170     private CameraCaptureSessionCore mCurrentSession;
171     private CameraExtensionSessionImpl mCurrentExtensionSession;
172     private CameraAdvancedExtensionSessionImpl mCurrentAdvancedExtensionSession;
173     private int mNextSessionId = 0;
174 
175     private final int mAppTargetSdkVersion;
176 
177     private ExecutorService mOfflineSwitchService;
178     private CameraOfflineSessionImpl mOfflineSessionImpl;
179 
180     // Runnables for all state transitions, except error, which needs the
181     // error code argument
182 
183     private final Runnable mCallOnOpened = new Runnable() {
184         @Override
185         public void run() {
186             StateCallbackKK sessionCallback = null;
187             synchronized(mInterfaceLock) {
188                 if (mRemoteDevice == null) return; // Camera already closed
189 
190                 sessionCallback = mSessionStateCallback;
191             }
192             if (sessionCallback != null) {
193                 sessionCallback.onOpened(CameraDeviceImpl.this);
194             }
195             mDeviceCallback.onOpened(CameraDeviceImpl.this);
196         }
197     };
198 
199     private final Runnable mCallOnUnconfigured = new Runnable() {
200         @Override
201         public void run() {
202             StateCallbackKK sessionCallback = null;
203             synchronized(mInterfaceLock) {
204                 if (mRemoteDevice == null) return; // Camera already closed
205 
206                 sessionCallback = mSessionStateCallback;
207             }
208             if (sessionCallback != null) {
209                 sessionCallback.onUnconfigured(CameraDeviceImpl.this);
210             }
211         }
212     };
213 
214     private final Runnable mCallOnActive = new Runnable() {
215         @Override
216         public void run() {
217             StateCallbackKK sessionCallback = null;
218             synchronized(mInterfaceLock) {
219                 if (mRemoteDevice == null) return; // Camera already closed
220 
221                 sessionCallback = mSessionStateCallback;
222             }
223             if (sessionCallback != null) {
224                 sessionCallback.onActive(CameraDeviceImpl.this);
225             }
226         }
227     };
228 
229     private final Runnable mCallOnBusy = new Runnable() {
230         @Override
231         public void run() {
232             StateCallbackKK sessionCallback = null;
233             synchronized(mInterfaceLock) {
234                 if (mRemoteDevice == null) return; // Camera already closed
235 
236                 sessionCallback = mSessionStateCallback;
237             }
238             if (sessionCallback != null) {
239                 sessionCallback.onBusy(CameraDeviceImpl.this);
240             }
241         }
242     };
243 
244     private final Runnable mCallOnClosed = new Runnable() {
245         private boolean mClosedOnce = false;
246 
247         @Override
248         public void run() {
249             if (mClosedOnce) {
250                 throw new AssertionError("Don't post #onClosed more than once");
251             }
252             StateCallbackKK sessionCallback = null;
253             synchronized(mInterfaceLock) {
254                 sessionCallback = mSessionStateCallback;
255             }
256             if (sessionCallback != null) {
257                 sessionCallback.onClosed(CameraDeviceImpl.this);
258             }
259             mDeviceCallback.onClosed(CameraDeviceImpl.this);
260             mClosedOnce = true;
261         }
262     };
263 
264     private final Runnable mCallOnIdle = new Runnable() {
265         @Override
266         public void run() {
267             StateCallbackKK sessionCallback = null;
268             synchronized(mInterfaceLock) {
269                 if (mRemoteDevice == null) return; // Camera already closed
270 
271                 sessionCallback = mSessionStateCallback;
272             }
273             if (sessionCallback != null) {
274                 sessionCallback.onIdle(CameraDeviceImpl.this);
275             }
276         }
277     };
278 
279     private final Runnable mCallOnDisconnected = new Runnable() {
280         @Override
281         public void run() {
282             StateCallbackKK sessionCallback = null;
283             synchronized(mInterfaceLock) {
284                 if (mRemoteDevice == null) return; // Camera already closed
285 
286                 sessionCallback = mSessionStateCallback;
287             }
288             if (sessionCallback != null) {
289                 sessionCallback.onDisconnected(CameraDeviceImpl.this);
290             }
291             mDeviceCallback.onDisconnected(CameraDeviceImpl.this);
292         }
293     };
294 
295     private class ClientStateCallback extends StateCallback {
296         private final Executor mClientExecutor;
297         private final StateCallback mClientStateCallback;
298 
ClientStateCallback(@onNull Executor clientExecutor, @NonNull StateCallback clientStateCallback)299         private ClientStateCallback(@NonNull Executor clientExecutor,
300                 @NonNull StateCallback clientStateCallback) {
301             mClientExecutor = clientExecutor;
302             mClientStateCallback = clientStateCallback;
303         }
304 
onClosed(@onNull CameraDevice camera)305         public void onClosed(@NonNull CameraDevice camera) {
306             mClientExecutor.execute(new Runnable() {
307                 @Override
308                 public void run() {
309                     mClientStateCallback.onClosed(camera);
310                 }
311             });
312         }
313         @Override
onOpened(@onNull CameraDevice camera)314         public void onOpened(@NonNull CameraDevice camera) {
315             mClientExecutor.execute(new Runnable() {
316                 @Override
317                 public void run() {
318                     mClientStateCallback.onOpened(camera);
319                 }
320             });
321         }
322 
323         @Override
onDisconnected(@onNull CameraDevice camera)324         public void onDisconnected(@NonNull CameraDevice camera) {
325             mClientExecutor.execute(new Runnable() {
326                 @Override
327                 public void run() {
328                     mClientStateCallback.onDisconnected(camera);
329                 }
330             });
331         }
332 
333         @Override
onError(@onNull CameraDevice camera, int error)334         public void onError(@NonNull CameraDevice camera, int error) {
335             mClientExecutor.execute(new Runnable() {
336                 @Override
337                 public void run() {
338                     mClientStateCallback.onError(camera, error);
339                 }
340             });
341         }
342     }
343 
CameraDeviceImpl(String cameraId, StateCallback callback, Executor executor, CameraCharacteristics characteristics, @NonNull CameraManager manager, int appTargetSdkVersion, Context ctx, @Nullable CameraDevice.CameraDeviceSetup cameraDeviceSetup)344     public CameraDeviceImpl(String cameraId, StateCallback callback, Executor executor,
345                         CameraCharacteristics characteristics,
346                         @NonNull CameraManager manager,
347                         int appTargetSdkVersion,
348                         Context ctx,
349                         @Nullable CameraDevice.CameraDeviceSetup cameraDeviceSetup) {
350         if (cameraId == null || callback == null || executor == null || characteristics == null
351                 || manager == null) {
352             throw new IllegalArgumentException("Null argument given");
353         }
354         mCameraId = cameraId;
355         if (Flags.singleThreadExecutor()) {
356             mDeviceCallback = new ClientStateCallback(executor, callback);
357             mDeviceExecutor = Executors.newSingleThreadExecutor();
358         } else {
359             mDeviceCallback = callback;
360             mDeviceExecutor = executor;
361         }
362         mCharacteristics = characteristics;
363         mCameraManager = manager;
364         mAppTargetSdkVersion = appTargetSdkVersion;
365         mContext = ctx;
366         mCameraDeviceSetup = cameraDeviceSetup;
367 
368         final int MAX_TAG_LEN = 23;
369         String tag = String.format("CameraDevice-JV-%s", mCameraId);
370         if (tag.length() > MAX_TAG_LEN) {
371             tag = tag.substring(0, MAX_TAG_LEN);
372         }
373         TAG = tag;
374 
375         Integer partialCount =
376                 mCharacteristics.get(CameraCharacteristics.REQUEST_PARTIAL_RESULT_COUNT);
377         if (partialCount == null) {
378             // 1 means partial result is not supported.
379             mTotalPartialCount = 1;
380         } else {
381             mTotalPartialCount = partialCount;
382         }
383     }
384 
getPhysicalIdToChars()385     private Map<String, CameraCharacteristics> getPhysicalIdToChars() {
386         if (mPhysicalIdsToChars == null) {
387             try {
388                 mPhysicalIdsToChars = mCameraManager.getPhysicalIdToCharsMap(mCharacteristics);
389             } catch (CameraAccessException e) {
390                 Log.e(TAG, "Unable to query the physical characteristics map!");
391             }
392         }
393 
394         return mPhysicalIdsToChars;
395     }
396 
getCallbacks()397     public CameraDeviceCallbacks getCallbacks() {
398         return mCallbacks;
399     }
400 
401     /**
402      * Set remote device, which triggers initial onOpened/onUnconfigured callbacks
403      *
404      * <p>This function may post onDisconnected and throw CAMERA_DISCONNECTED if remoteDevice dies
405      * during setup.</p>
406      *
407      */
setRemoteDevice(ICameraDeviceUser remoteDevice)408     public void setRemoteDevice(ICameraDeviceUser remoteDevice) throws CameraAccessException {
409         synchronized(mInterfaceLock) {
410             // TODO: Move from decorator to direct binder-mediated exceptions
411             // If setRemoteFailure already called, do nothing
412             if (mInError) return;
413 
414             mRemoteDevice = new ICameraDeviceUserWrapper(remoteDevice);
415 
416             IBinder remoteDeviceBinder = remoteDevice.asBinder();
417             // For legacy camera device, remoteDevice is in the same process, and
418             // asBinder returns NULL.
419             if (remoteDeviceBinder != null) {
420                 try {
421                     remoteDeviceBinder.linkToDeath(this, /*flag*/ 0);
422                 } catch (RemoteException e) {
423                     CameraDeviceImpl.this.mDeviceExecutor.execute(mCallOnDisconnected);
424 
425                     throw new CameraAccessException(CameraAccessException.CAMERA_DISCONNECTED,
426                             "The camera device has encountered a serious error");
427                 }
428             }
429 
430             mDeviceExecutor.execute(mCallOnOpened);
431             mDeviceExecutor.execute(mCallOnUnconfigured);
432 
433             mRemoteDeviceInit = true;
434         }
435     }
436 
437     /**
438      * Call to indicate failed connection to a remote camera device.
439      *
440      * <p>This places the camera device in the error state and informs the callback.
441      * Use in place of setRemoteDevice() when startup fails.</p>
442      */
setRemoteFailure(final ServiceSpecificException failure)443     public void setRemoteFailure(final ServiceSpecificException failure) {
444         int failureCode = StateCallback.ERROR_CAMERA_DEVICE;
445         boolean failureIsError = true;
446 
447         switch (failure.errorCode) {
448             case ICameraService.ERROR_CAMERA_IN_USE:
449                 failureCode = StateCallback.ERROR_CAMERA_IN_USE;
450                 break;
451             case ICameraService.ERROR_MAX_CAMERAS_IN_USE:
452                 failureCode = StateCallback.ERROR_MAX_CAMERAS_IN_USE;
453                 break;
454             case ICameraService.ERROR_DISABLED:
455                 failureCode = StateCallback.ERROR_CAMERA_DISABLED;
456                 break;
457             case ICameraService.ERROR_DISCONNECTED:
458                 failureIsError = false;
459                 break;
460             case ICameraService.ERROR_INVALID_OPERATION:
461                 failureCode = StateCallback.ERROR_CAMERA_DEVICE;
462                 break;
463             default:
464                 Log.e(TAG, "Unexpected failure in opening camera device: " + failure.errorCode +
465                         failure.getMessage());
466                 break;
467         }
468         final int code = failureCode;
469         final boolean isError = failureIsError;
470         synchronized(mInterfaceLock) {
471             mInError = true;
472             mDeviceExecutor.execute(new Runnable() {
473                 @Override
474                 public void run() {
475                     if (isError) {
476                         mDeviceCallback.onError(CameraDeviceImpl.this, code);
477                     } else {
478                         mDeviceCallback.onDisconnected(CameraDeviceImpl.this);
479                     }
480                 }
481             });
482         }
483     }
484 
485     @Override
getId()486     public String getId() {
487         return mCameraId;
488     }
489 
configureOutputs(List<Surface> outputs)490     public void configureOutputs(List<Surface> outputs) throws CameraAccessException {
491         // Leave this here for backwards compatibility with older code using this directly
492         ArrayList<OutputConfiguration> outputConfigs = new ArrayList<>(outputs.size());
493         for (Surface s : outputs) {
494             outputConfigs.add(new OutputConfiguration(s));
495         }
496         configureStreamsChecked(/*inputConfig*/null, outputConfigs,
497                 /*operatingMode*/ICameraDeviceUser.NORMAL_MODE, /*sessionParams*/ null,
498                 SystemClock.uptimeMillis());
499 
500     }
501 
502     /**
503      * Attempt to configure the input and outputs; the device goes to idle and then configures the
504      * new input and outputs if possible.
505      *
506      * <p>The configuration may gracefully fail, if input configuration is not supported,
507      * if there are too many outputs, if the formats are not supported, or if the sizes for that
508      * format is not supported. In this case this function will return {@code false} and the
509      * unconfigured callback will be fired.</p>
510      *
511      * <p>If the configuration succeeds (with 1 or more outputs with or without an input),
512      * then the idle callback is fired. Unconfiguring the device always fires the idle callback.</p>
513      *
514      * @param inputConfig input configuration or {@code null} for no input
515      * @param outputs a list of one or more surfaces, or {@code null} to unconfigure
516      * @param operatingMode If the stream configuration is for a normal session,
517      *     a constrained high speed session, or something else.
518      * @param sessionParams Session parameters.
519      * @param createSessionStartTimeMs The timestamp when session creation starts, measured by
520      *     uptimeMillis().
521      * @return whether or not the configuration was successful
522      *
523      * @throws CameraAccessException if there were any unexpected problems during configuration
524      */
configureStreamsChecked(InputConfiguration inputConfig, List<OutputConfiguration> outputs, int operatingMode, CaptureRequest sessionParams, long createSessionStartTime)525     public boolean configureStreamsChecked(InputConfiguration inputConfig,
526             List<OutputConfiguration> outputs, int operatingMode, CaptureRequest sessionParams,
527             long createSessionStartTime)
528                     throws CameraAccessException {
529         // Treat a null input the same an empty list
530         if (outputs == null) {
531             outputs = new ArrayList<OutputConfiguration>();
532         }
533         if (outputs.size() == 0 && inputConfig != null) {
534             throw new IllegalArgumentException("cannot configure an input stream without " +
535                     "any output streams");
536         }
537 
538         checkInputConfiguration(inputConfig);
539 
540         boolean success = false;
541 
542         synchronized(mInterfaceLock) {
543             checkIfCameraClosedOrInError();
544             // Streams to create
545             HashSet<OutputConfiguration> addSet = new HashSet<OutputConfiguration>(outputs);
546             // Streams to delete
547             List<Integer> deleteList = new ArrayList<Integer>();
548 
549             // Determine which streams need to be created, which to be deleted
550             for (int i = 0; i < mConfiguredOutputs.size(); ++i) {
551                 int streamId = mConfiguredOutputs.keyAt(i);
552                 OutputConfiguration outConfig = mConfiguredOutputs.valueAt(i);
553 
554                 if (!outputs.contains(outConfig) || outConfig.isDeferredConfiguration()) {
555                     // Always delete the deferred output configuration when the session
556                     // is created, as the deferred output configuration doesn't have unique surface
557                     // related identifies.
558                     deleteList.add(streamId);
559                 } else {
560                     addSet.remove(outConfig);  // Don't create a stream previously created
561                 }
562             }
563 
564             mDeviceExecutor.execute(mCallOnBusy);
565             stopRepeating();
566 
567             try {
568                 waitUntilIdle();
569 
570                 mRemoteDevice.beginConfigure();
571 
572                 // reconfigure the input stream if the input configuration is different.
573                 InputConfiguration currentInputConfig = mConfiguredInput.getValue();
574                 if (inputConfig != currentInputConfig &&
575                         (inputConfig == null || !inputConfig.equals(currentInputConfig))) {
576                     if (currentInputConfig != null) {
577                         mRemoteDevice.deleteStream(mConfiguredInput.getKey());
578                         mConfiguredInput = new SimpleEntry<Integer, InputConfiguration>(
579                                 REQUEST_ID_NONE, null);
580                     }
581                     if (inputConfig != null) {
582                         int streamId = mRemoteDevice.createInputStream(inputConfig.getWidth(),
583                                 inputConfig.getHeight(), inputConfig.getFormat(),
584                                 inputConfig.isMultiResolution());
585                         mConfiguredInput = new SimpleEntry<Integer, InputConfiguration>(
586                                 streamId, inputConfig);
587                     }
588                 }
589 
590                 // Delete all streams first (to free up HW resources)
591                 for (Integer streamId : deleteList) {
592                     mRemoteDevice.deleteStream(streamId);
593                     mConfiguredOutputs.delete(streamId);
594                 }
595 
596                 // Add all new streams
597                 for (OutputConfiguration outConfig : outputs) {
598                     if (addSet.contains(outConfig)) {
599                         int streamId = mRemoteDevice.createStream(outConfig);
600                         mConfiguredOutputs.put(streamId, outConfig);
601                     }
602                 }
603 
604                 int offlineStreamIds[];
605                 if (sessionParams != null) {
606                     offlineStreamIds = mRemoteDevice.endConfigure(operatingMode,
607                             sessionParams.getNativeCopy(), createSessionStartTime);
608                 } else {
609                     offlineStreamIds = mRemoteDevice.endConfigure(operatingMode, null,
610                             createSessionStartTime);
611                 }
612 
613                 mOfflineSupport.clear();
614                 if ((offlineStreamIds != null) && (offlineStreamIds.length > 0)) {
615                     for (int offlineStreamId : offlineStreamIds) {
616                         mOfflineSupport.add(offlineStreamId);
617                     }
618                 }
619 
620                 success = true;
621             } catch (IllegalArgumentException e) {
622                 // OK. camera service can reject stream config if it's not supported by HAL
623                 // This is only the result of a programmer misusing the camera2 api.
624                 Log.w(TAG, "Stream configuration failed due to: " + e.getMessage());
625                 return false;
626             } catch (CameraAccessException e) {
627                 if (e.getReason() == CameraAccessException.CAMERA_IN_USE) {
628                     throw new IllegalStateException("The camera is currently busy." +
629                             " You must wait until the previous operation completes.", e);
630                 }
631                 throw e;
632             } finally {
633                 if (success && outputs.size() > 0) {
634                     mDeviceExecutor.execute(mCallOnIdle);
635                 } else {
636                     // Always return to the 'unconfigured' state if we didn't hit a fatal error
637                     mDeviceExecutor.execute(mCallOnUnconfigured);
638                 }
639             }
640         }
641 
642         return success;
643     }
644 
645     @Override
createCaptureSession(List<Surface> outputs, CameraCaptureSession.StateCallback callback, Handler handler)646     public void createCaptureSession(List<Surface> outputs,
647             CameraCaptureSession.StateCallback callback, Handler handler)
648             throws CameraAccessException {
649         List<OutputConfiguration> outConfigurations = new ArrayList<>(outputs.size());
650         for (Surface surface : outputs) {
651             outConfigurations.add(new OutputConfiguration(surface));
652         }
653         createCaptureSessionInternal(null, outConfigurations, callback,
654                 checkAndWrapHandler(handler), /*operatingMode*/ICameraDeviceUser.NORMAL_MODE,
655                 /*sessionParams*/ null);
656     }
657 
658     @Override
createCaptureSessionByOutputConfigurations( List<OutputConfiguration> outputConfigurations, CameraCaptureSession.StateCallback callback, Handler handler)659     public void createCaptureSessionByOutputConfigurations(
660             List<OutputConfiguration> outputConfigurations,
661             CameraCaptureSession.StateCallback callback, Handler handler)
662             throws CameraAccessException {
663         if (DEBUG) {
664             Log.d(TAG, "createCaptureSessionByOutputConfigurations");
665         }
666 
667         // OutputConfiguration objects are immutable, but need to have our own array
668         List<OutputConfiguration> currentOutputs = new ArrayList<>(outputConfigurations);
669 
670         createCaptureSessionInternal(null, currentOutputs, callback, checkAndWrapHandler(handler),
671                 /*operatingMode*/ICameraDeviceUser.NORMAL_MODE, /*sessionParams*/null);
672     }
673 
674     @Override
createReprocessableCaptureSession(InputConfiguration inputConfig, List<Surface> outputs, CameraCaptureSession.StateCallback callback, Handler handler)675     public void createReprocessableCaptureSession(InputConfiguration inputConfig,
676             List<Surface> outputs, CameraCaptureSession.StateCallback callback, Handler handler)
677             throws CameraAccessException {
678         if (DEBUG) {
679             Log.d(TAG, "createReprocessableCaptureSession");
680         }
681 
682         if (inputConfig == null) {
683             throw new IllegalArgumentException("inputConfig cannot be null when creating a " +
684                     "reprocessable capture session");
685         }
686         List<OutputConfiguration> outConfigurations = new ArrayList<>(outputs.size());
687         for (Surface surface : outputs) {
688             outConfigurations.add(new OutputConfiguration(surface));
689         }
690         createCaptureSessionInternal(inputConfig, outConfigurations, callback,
691                 checkAndWrapHandler(handler), /*operatingMode*/ICameraDeviceUser.NORMAL_MODE,
692                 /*sessionParams*/ null);
693     }
694 
695     @Override
createReprocessableCaptureSessionByConfigurations(InputConfiguration inputConfig, List<OutputConfiguration> outputs, android.hardware.camera2.CameraCaptureSession.StateCallback callback, Handler handler)696     public void createReprocessableCaptureSessionByConfigurations(InputConfiguration inputConfig,
697             List<OutputConfiguration> outputs,
698             android.hardware.camera2.CameraCaptureSession.StateCallback callback, Handler handler)
699                     throws CameraAccessException {
700         if (DEBUG) {
701             Log.d(TAG, "createReprocessableCaptureSessionWithConfigurations");
702         }
703 
704         if (inputConfig == null) {
705             throw new IllegalArgumentException("inputConfig cannot be null when creating a " +
706                     "reprocessable capture session");
707         }
708 
709         if (outputs == null) {
710             throw new IllegalArgumentException("Output configurations cannot be null when " +
711                     "creating a reprocessable capture session");
712         }
713 
714         // OutputConfiguration objects aren't immutable, make a copy before using.
715         List<OutputConfiguration> currentOutputs = new ArrayList<OutputConfiguration>();
716         for (OutputConfiguration output : outputs) {
717             currentOutputs.add(new OutputConfiguration(output));
718         }
719         createCaptureSessionInternal(inputConfig, currentOutputs,
720                 callback, checkAndWrapHandler(handler),
721                 /*operatingMode*/ICameraDeviceUser.NORMAL_MODE, /*sessionParams*/ null);
722     }
723 
724     @Override
createConstrainedHighSpeedCaptureSession(List<Surface> outputs, android.hardware.camera2.CameraCaptureSession.StateCallback callback, Handler handler)725     public void createConstrainedHighSpeedCaptureSession(List<Surface> outputs,
726             android.hardware.camera2.CameraCaptureSession.StateCallback callback, Handler handler)
727             throws CameraAccessException {
728         if (outputs == null || outputs.size() == 0 || outputs.size() > 2) {
729             throw new IllegalArgumentException(
730                     "Output surface list must not be null and the size must be no more than 2");
731         }
732         List<OutputConfiguration> outConfigurations = new ArrayList<>(outputs.size());
733         for (Surface surface : outputs) {
734             outConfigurations.add(new OutputConfiguration(surface));
735         }
736         createCaptureSessionInternal(null, outConfigurations, callback,
737                 checkAndWrapHandler(handler),
738                 /*operatingMode*/ICameraDeviceUser.CONSTRAINED_HIGH_SPEED_MODE,
739                 /*sessionParams*/ null);
740     }
741 
742     @Override
createCustomCaptureSession(InputConfiguration inputConfig, List<OutputConfiguration> outputs, int operatingMode, android.hardware.camera2.CameraCaptureSession.StateCallback callback, Handler handler)743     public void createCustomCaptureSession(InputConfiguration inputConfig,
744             List<OutputConfiguration> outputs,
745             int operatingMode,
746             android.hardware.camera2.CameraCaptureSession.StateCallback callback,
747             Handler handler) throws CameraAccessException {
748         List<OutputConfiguration> currentOutputs = new ArrayList<OutputConfiguration>();
749         for (OutputConfiguration output : outputs) {
750             currentOutputs.add(new OutputConfiguration(output));
751         }
752         createCaptureSessionInternal(inputConfig, currentOutputs, callback,
753                 checkAndWrapHandler(handler), operatingMode, /*sessionParams*/ null);
754     }
755 
756     @Override
createCaptureSession(SessionConfiguration config)757     public void createCaptureSession(SessionConfiguration config)
758             throws CameraAccessException {
759         if (config == null) {
760             throw new IllegalArgumentException("Invalid session configuration");
761         }
762 
763         List<OutputConfiguration> outputConfigs = config.getOutputConfigurations();
764         if (outputConfigs == null) {
765             throw new IllegalArgumentException("Invalid output configurations");
766         }
767         if (config.getExecutor() == null) {
768             throw new IllegalArgumentException("Invalid executor");
769         }
770         createCaptureSessionInternal(config.getInputConfiguration(), outputConfigs,
771                 config.getStateCallback(), config.getExecutor(), config.getSessionType(),
772                 config.getSessionParameters());
773     }
774 
createCaptureSessionInternal(InputConfiguration inputConfig, List<OutputConfiguration> outputConfigurations, CameraCaptureSession.StateCallback callback, Executor executor, int operatingMode, CaptureRequest sessionParams)775     private void createCaptureSessionInternal(InputConfiguration inputConfig,
776             List<OutputConfiguration> outputConfigurations,
777             CameraCaptureSession.StateCallback callback, Executor executor,
778             int operatingMode, CaptureRequest sessionParams) throws CameraAccessException {
779         long createSessionStartTime = SystemClock.uptimeMillis();
780         synchronized(mInterfaceLock) {
781             if (DEBUG) {
782                 Log.d(TAG, "createCaptureSessionInternal");
783             }
784 
785             checkIfCameraClosedOrInError();
786 
787             boolean isConstrainedHighSpeed =
788                     (operatingMode == ICameraDeviceUser.CONSTRAINED_HIGH_SPEED_MODE);
789             if (isConstrainedHighSpeed && inputConfig != null) {
790                 throw new IllegalArgumentException("Constrained high speed session doesn't support"
791                         + " input configuration yet.");
792             }
793 
794             if (mCurrentExtensionSession != null) {
795                 mCurrentExtensionSession.commitStats();
796             }
797 
798             if (mCurrentAdvancedExtensionSession != null) {
799                 mCurrentAdvancedExtensionSession.commitStats();
800             }
801 
802             // Notify current session that it's going away, before starting camera operations
803             // After this call completes, the session is not allowed to call into CameraDeviceImpl
804             if (mCurrentSession != null) {
805                 mCurrentSession.replaceSessionClose();
806             }
807 
808             if (mCurrentExtensionSession != null) {
809                 mCurrentExtensionSession.release(false /*skipCloseNotification*/);
810                 mCurrentExtensionSession = null;
811             }
812 
813             if (mCurrentAdvancedExtensionSession != null) {
814                 mCurrentAdvancedExtensionSession.release(false /*skipCloseNotification*/);
815                 mCurrentAdvancedExtensionSession = null;
816             }
817 
818             // TODO: dont block for this
819             boolean configureSuccess = true;
820             CameraAccessException pendingException = null;
821             Surface input = null;
822             try {
823                 // configure streams and then block until IDLE
824                 configureSuccess = configureStreamsChecked(inputConfig, outputConfigurations,
825                         operatingMode, sessionParams, createSessionStartTime);
826                 if (configureSuccess == true && inputConfig != null) {
827                     input = mRemoteDevice.getInputSurface();
828                 }
829             } catch (CameraAccessException e) {
830                 configureSuccess = false;
831                 pendingException = e;
832                 input = null;
833                 if (DEBUG) {
834                     Log.v(TAG, "createCaptureSession - failed with exception ", e);
835                 }
836             }
837 
838             // Fire onConfigured if configureOutputs succeeded, fire onConfigureFailed otherwise.
839             CameraCaptureSessionCore newSession = null;
840             if (isConstrainedHighSpeed) {
841                 ArrayList<Surface> surfaces = new ArrayList<>(outputConfigurations.size());
842                 for (OutputConfiguration outConfig : outputConfigurations) {
843                     surfaces.add(outConfig.getSurface());
844                 }
845                 StreamConfigurationMap config =
846                     getCharacteristics().get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
847                 SurfaceUtils.checkConstrainedHighSpeedSurfaces(surfaces, /*fpsRange*/null, config);
848 
849                 newSession = new CameraConstrainedHighSpeedCaptureSessionImpl(mNextSessionId++,
850                         callback, executor, this, mDeviceExecutor, configureSuccess,
851                         mCharacteristics);
852             } else {
853                 newSession = new CameraCaptureSessionImpl(mNextSessionId++, input,
854                         callback, executor, this, mDeviceExecutor, configureSuccess);
855             }
856 
857             // TODO: wait until current session closes, then create the new session
858             mCurrentSession = newSession;
859 
860             if (pendingException != null) {
861                 throw pendingException;
862             }
863 
864             mSessionStateCallback = mCurrentSession.getDeviceStateCallback();
865         }
866     }
867 
868     @Override
isSessionConfigurationSupported( @onNull SessionConfiguration sessionConfig)869     public boolean isSessionConfigurationSupported(
870             @NonNull SessionConfiguration sessionConfig) throws CameraAccessException,
871             UnsupportedOperationException, IllegalArgumentException {
872         synchronized (mInterfaceLock) {
873             checkIfCameraClosedOrInError();
874             if (CompatChanges.isChangeEnabled(CHECK_PARAMS_IN_IS_SESSION_CONFIGURATION_SUPPORTED)
875                     && Flags.cameraDeviceSetup()
876                     && mCameraDeviceSetup != null) {
877                 return mCameraDeviceSetup.isSessionConfigurationSupported(sessionConfig);
878             }
879             return mRemoteDevice.isSessionConfigurationSupported(sessionConfig);
880         }
881     }
882 
883     /**
884      * For use by backwards-compatibility code only.
885      */
setSessionListener(StateCallbackKK sessionCallback)886     public void setSessionListener(StateCallbackKK sessionCallback) {
887         synchronized(mInterfaceLock) {
888             mSessionStateCallback = sessionCallback;
889         }
890     }
891 
892     /**
893      * Disable CONTROL_ENABLE_ZSL based on targetSdkVersion and capture template.
894      */
disableZslIfNeeded(CameraMetadataNative request, int targetSdkVersion, int templateType)895     public static void disableZslIfNeeded(CameraMetadataNative request,
896             int targetSdkVersion, int templateType) {
897         // If targetSdkVersion is at least O, no need to set ENABLE_ZSL to false
898         // for STILL_CAPTURE template.
899         if (targetSdkVersion >= Build.VERSION_CODES.O
900                 && templateType == TEMPLATE_STILL_CAPTURE) {
901             return;
902         }
903 
904         Boolean enableZsl = request.get(CaptureRequest.CONTROL_ENABLE_ZSL);
905         if (enableZsl == null) {
906             // If enableZsl is not available, don't override.
907             return;
908         }
909 
910         request.set(CaptureRequest.CONTROL_ENABLE_ZSL, false);
911     }
912 
913     @Override
createCaptureRequest(int templateType, Set<String> physicalCameraIdSet)914     public CaptureRequest.Builder createCaptureRequest(int templateType,
915             Set<String> physicalCameraIdSet)
916             throws CameraAccessException {
917         synchronized(mInterfaceLock) {
918             checkIfCameraClosedOrInError();
919 
920             for (String physicalId : physicalCameraIdSet) {
921                 if (physicalId == getId()) {
922                     throw new IllegalStateException("Physical id matches the logical id!");
923                 }
924             }
925 
926             CameraMetadataNative templatedRequest = null;
927 
928             templatedRequest = mRemoteDevice.createDefaultRequest(templateType);
929 
930             disableZslIfNeeded(templatedRequest, mAppTargetSdkVersion, templateType);
931 
932             CaptureRequest.Builder builder = new CaptureRequest.Builder(
933                     templatedRequest, /*reprocess*/false, CameraCaptureSession.SESSION_ID_NONE,
934                     getId(), physicalCameraIdSet);
935 
936             return builder;
937         }
938     }
939 
940     @Override
createCaptureRequest(int templateType)941     public CaptureRequest.Builder createCaptureRequest(int templateType)
942             throws CameraAccessException {
943         synchronized(mInterfaceLock) {
944             checkIfCameraClosedOrInError();
945 
946             CameraMetadataNative templatedRequest = null;
947 
948             templatedRequest = mRemoteDevice.createDefaultRequest(templateType);
949 
950             disableZslIfNeeded(templatedRequest, mAppTargetSdkVersion, templateType);
951 
952             CaptureRequest.Builder builder = new CaptureRequest.Builder(
953                     templatedRequest, /*reprocess*/false, CameraCaptureSession.SESSION_ID_NONE,
954                     getId(), /*physicalCameraIdSet*/ null);
955 
956             return builder;
957         }
958     }
959 
960     @Override
createReprocessCaptureRequest(TotalCaptureResult inputResult)961     public CaptureRequest.Builder createReprocessCaptureRequest(TotalCaptureResult inputResult)
962             throws CameraAccessException {
963         synchronized(mInterfaceLock) {
964             checkIfCameraClosedOrInError();
965 
966             CameraMetadataNative resultMetadata = new
967                     CameraMetadataNative(inputResult.getNativeCopy());
968 
969             CaptureRequest.Builder builder = new CaptureRequest.Builder(resultMetadata,
970                     /*reprocess*/true, inputResult.getSessionId(), getId(),
971                     /*physicalCameraIdSet*/ null);
972             builder.set(CaptureRequest.CONTROL_CAPTURE_INTENT,
973                     CameraMetadata.CONTROL_CAPTURE_INTENT_STILL_CAPTURE);
974 
975             return builder;
976         }
977     }
978 
prepare(Surface surface)979     public void prepare(Surface surface) throws CameraAccessException {
980         if (surface == null) throw new IllegalArgumentException("Surface is null");
981 
982         synchronized(mInterfaceLock) {
983             checkIfCameraClosedOrInError();
984             int streamId = -1;
985             for (int i = 0; i < mConfiguredOutputs.size(); i++) {
986                 final List<Surface> surfaces = mConfiguredOutputs.valueAt(i).getSurfaces();
987                 if (surfaces.contains(surface)) {
988                     streamId = mConfiguredOutputs.keyAt(i);
989                     break;
990                 }
991             }
992             if (streamId == -1) {
993                 throw new IllegalArgumentException("Surface is not part of this session");
994             }
995 
996             mRemoteDevice.prepare(streamId);
997         }
998     }
999 
prepare(int maxCount, Surface surface)1000     public void prepare(int maxCount, Surface surface) throws CameraAccessException {
1001         if (surface == null) throw new IllegalArgumentException("Surface is null");
1002         if (maxCount <= 0) throw new IllegalArgumentException("Invalid maxCount given: " +
1003                 maxCount);
1004 
1005         synchronized(mInterfaceLock) {
1006             checkIfCameraClosedOrInError();
1007             int streamId = -1;
1008             for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1009                 if (surface == mConfiguredOutputs.valueAt(i).getSurface()) {
1010                     streamId = mConfiguredOutputs.keyAt(i);
1011                     break;
1012                 }
1013             }
1014             if (streamId == -1) {
1015                 throw new IllegalArgumentException("Surface is not part of this session");
1016             }
1017 
1018             mRemoteDevice.prepare2(maxCount, streamId);
1019         }
1020     }
1021 
updateOutputConfiguration(OutputConfiguration config)1022     public void updateOutputConfiguration(OutputConfiguration config)
1023             throws CameraAccessException {
1024         synchronized(mInterfaceLock) {
1025             checkIfCameraClosedOrInError();
1026             int streamId = -1;
1027             for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1028                 if (config.getSurface() == mConfiguredOutputs.valueAt(i).getSurface()) {
1029                     streamId = mConfiguredOutputs.keyAt(i);
1030                     break;
1031                 }
1032             }
1033             if (streamId == -1) {
1034                 throw new IllegalArgumentException("Invalid output configuration");
1035             }
1036 
1037             mRemoteDevice.updateOutputConfiguration(streamId, config);
1038             mConfiguredOutputs.put(streamId, config);
1039         }
1040     }
1041 
switchToOffline( @onNull Collection<Surface> offlineOutputs, @NonNull Executor executor, @NonNull CameraOfflineSession.CameraOfflineSessionCallback listener)1042     public CameraOfflineSession switchToOffline(
1043             @NonNull Collection<Surface> offlineOutputs, @NonNull Executor executor,
1044             @NonNull CameraOfflineSession.CameraOfflineSessionCallback listener)
1045             throws CameraAccessException {
1046         if (offlineOutputs.isEmpty()) {
1047             throw new IllegalArgumentException("Invalid offline surfaces!");
1048         }
1049 
1050         HashSet<Integer> offlineStreamIds = new HashSet<Integer>();
1051         SparseArray<OutputConfiguration> offlineConfiguredOutputs =
1052                 new SparseArray<OutputConfiguration>();
1053         CameraOfflineSession ret;
1054 
1055         synchronized(mInterfaceLock) {
1056             checkIfCameraClosedOrInError();
1057             if (mOfflineSessionImpl != null) {
1058                 throw new IllegalStateException("Switch to offline mode already in progress");
1059             }
1060 
1061             for (Surface surface : offlineOutputs) {
1062                 int streamId = -1;
1063                 for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1064                     if (surface == mConfiguredOutputs.valueAt(i).getSurface()) {
1065                         streamId = mConfiguredOutputs.keyAt(i);
1066                         offlineConfiguredOutputs.append(streamId, mConfiguredOutputs.valueAt(i));
1067                         break;
1068                     }
1069                 }
1070                 if (streamId == -1) {
1071                     throw new IllegalArgumentException("Offline surface is not part of this" +
1072                             " session");
1073                 }
1074 
1075                 if (!mOfflineSupport.contains(streamId)) {
1076                     throw new IllegalArgumentException("Surface: "  + surface + " does not " +
1077                             " support offline mode");
1078                 }
1079 
1080                 offlineStreamIds.add(streamId);
1081             }
1082             stopRepeating();
1083 
1084             mOfflineSessionImpl = new CameraOfflineSessionImpl(mCameraId,
1085                     mCharacteristics, executor, listener, offlineConfiguredOutputs,
1086                     mConfiguredInput, mConfiguredOutputs, mFrameNumberTracker, mCaptureCallbackMap,
1087                     mRequestLastFrameNumbersList);
1088             ret = mOfflineSessionImpl;
1089 
1090             mOfflineSwitchService = Executors.newSingleThreadExecutor();
1091             mConfiguredOutputs.clear();
1092             mConfiguredInput = new SimpleEntry<Integer, InputConfiguration>(REQUEST_ID_NONE, null);
1093             mIdle = true;
1094             mCaptureCallbackMap = new SparseArray<CaptureCallbackHolder>();
1095             mBatchOutputMap = new HashMap<>();
1096             mFrameNumberTracker = new FrameNumberTracker();
1097 
1098             mCurrentSession.closeWithoutDraining();
1099             mCurrentSession = null;
1100         }
1101 
1102         mOfflineSwitchService.execute(new Runnable() {
1103             @Override
1104             public void run() {
1105                 // We cannot hold 'mInterfaceLock' during the remote IPC in 'switchToOffline'.
1106                 // The call will block until all non-offline requests are completed and/or flushed.
1107                 // The results/errors need to be handled by 'CameraDeviceCallbacks' which also sync
1108                 // on 'mInterfaceLock'.
1109                 try {
1110                     ICameraOfflineSession remoteOfflineSession = mRemoteDevice.switchToOffline(
1111                             mOfflineSessionImpl.getCallbacks(),
1112                             Arrays.stream(offlineStreamIds.toArray(
1113                                     new Integer[offlineStreamIds.size()])).mapToInt(
1114                                             Integer::intValue).toArray());
1115                     mOfflineSessionImpl.setRemoteSession(remoteOfflineSession);
1116                 } catch (CameraAccessException e) {
1117                     mOfflineSessionImpl.notifyFailedSwitch();
1118                 } finally {
1119                     mOfflineSessionImpl = null;
1120                 }
1121             }
1122         });
1123 
1124         return ret;
1125     }
1126 
supportsOfflineProcessing(Surface surface)1127     public boolean supportsOfflineProcessing(Surface surface) {
1128         if (surface == null) throw new IllegalArgumentException("Surface is null");
1129 
1130         synchronized(mInterfaceLock) {
1131             int streamId = -1;
1132             for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1133                 if (surface == mConfiguredOutputs.valueAt(i).getSurface()) {
1134                     streamId = mConfiguredOutputs.keyAt(i);
1135                     break;
1136                 }
1137             }
1138             if (streamId == -1) {
1139                 throw new IllegalArgumentException("Surface is not part of this session");
1140             }
1141 
1142             return mOfflineSupport.contains(streamId);
1143         }
1144     }
1145 
tearDown(Surface surface)1146     public void tearDown(Surface surface) throws CameraAccessException {
1147         if (surface == null) throw new IllegalArgumentException("Surface is null");
1148 
1149         synchronized(mInterfaceLock) {
1150             checkIfCameraClosedOrInError();
1151             int streamId = -1;
1152             for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1153                 if (surface == mConfiguredOutputs.valueAt(i).getSurface()) {
1154                     streamId = mConfiguredOutputs.keyAt(i);
1155                     break;
1156                 }
1157             }
1158             if (streamId == -1) {
1159                 throw new IllegalArgumentException("Surface is not part of this session");
1160             }
1161 
1162             mRemoteDevice.tearDown(streamId);
1163         }
1164     }
1165 
finalizeOutputConfigs(List<OutputConfiguration> outputConfigs)1166     public void finalizeOutputConfigs(List<OutputConfiguration> outputConfigs)
1167             throws CameraAccessException {
1168         if (outputConfigs == null || outputConfigs.size() == 0) {
1169             throw new IllegalArgumentException("deferred config is null or empty");
1170         }
1171 
1172         synchronized(mInterfaceLock) {
1173             checkIfCameraClosedOrInError();
1174 
1175             for (OutputConfiguration config : outputConfigs) {
1176                 int streamId = -1;
1177                 for (int i = 0; i < mConfiguredOutputs.size(); i++) {
1178                     // Have to use equal here, as createCaptureSessionByOutputConfigurations() and
1179                     // createReprocessableCaptureSessionByConfigurations() do a copy of the configs.
1180                     if (config.equals(mConfiguredOutputs.valueAt(i))) {
1181                         streamId = mConfiguredOutputs.keyAt(i);
1182                         break;
1183                     }
1184                 }
1185                 if (streamId == -1) {
1186                     throw new IllegalArgumentException("Deferred config is not part of this "
1187                             + "session");
1188                 }
1189 
1190                 if (config.getSurfaces().size() == 0) {
1191                     throw new IllegalArgumentException("The final config for stream " + streamId
1192                             + " must have at least 1 surface");
1193                 }
1194                 mRemoteDevice.finalizeOutputConfigurations(streamId, config);
1195                 mConfiguredOutputs.put(streamId, config);
1196             }
1197         }
1198     }
1199 
capture(CaptureRequest request, CaptureCallback callback, Executor executor)1200     public int capture(CaptureRequest request, CaptureCallback callback, Executor executor)
1201             throws CameraAccessException {
1202         if (DEBUG) {
1203             Log.d(TAG, "calling capture");
1204         }
1205         List<CaptureRequest> requestList = new ArrayList<CaptureRequest>();
1206         requestList.add(request);
1207         return submitCaptureRequest(requestList, callback, executor, /*streaming*/false);
1208     }
1209 
captureBurst(List<CaptureRequest> requests, CaptureCallback callback, Executor executor)1210     public int captureBurst(List<CaptureRequest> requests, CaptureCallback callback,
1211             Executor executor) throws CameraAccessException {
1212         if (requests == null || requests.isEmpty()) {
1213             throw new IllegalArgumentException("At least one request must be given");
1214         }
1215         return submitCaptureRequest(requests, callback, executor, /*streaming*/false);
1216     }
1217 
1218     /**
1219      * This method checks lastFrameNumber returned from ICameraDeviceUser methods for
1220      * starting and stopping repeating request and flushing.
1221      *
1222      * <p>If lastFrameNumber is NO_FRAMES_CAPTURED, it means that the request was never
1223      * sent to HAL. Then onCaptureSequenceAborted is immediately triggered.
1224      * If lastFrameNumber is non-negative, then the requestId and lastFrameNumber as the last
1225      * regular frame number will be added to the list mRequestLastFrameNumbersList.</p>
1226      *
1227      * @param requestId the request ID of the current repeating request.
1228      * @param lastFrameNumber last frame number returned from binder.
1229      * @param repeatingRequestTypes the repeating requests' types.
1230      */
checkEarlyTriggerSequenceCompleteLocked( final int requestId, final long lastFrameNumber, final int[] repeatingRequestTypes)1231     private void checkEarlyTriggerSequenceCompleteLocked(
1232             final int requestId, final long lastFrameNumber,
1233             final int[] repeatingRequestTypes) {
1234         // lastFrameNumber being equal to NO_FRAMES_CAPTURED means that the request
1235         // was never sent to HAL. Should trigger onCaptureSequenceAborted immediately.
1236         if (lastFrameNumber == CameraCaptureSession.CaptureCallback.NO_FRAMES_CAPTURED) {
1237             final CaptureCallbackHolder holder;
1238             int index = mCaptureCallbackMap.indexOfKey(requestId);
1239             holder = (index >= 0) ? mCaptureCallbackMap.valueAt(index) : null;
1240             if (holder != null) {
1241                 mCaptureCallbackMap.removeAt(index);
1242                 if (DEBUG) {
1243                     Log.v(TAG, String.format(
1244                             "remove holder for requestId %d, "
1245                             + "because lastFrame is %d.",
1246                             requestId, lastFrameNumber));
1247 
1248                     Log.v(TAG, "immediately trigger onCaptureSequenceAborted because"
1249                             + " request did not reach HAL");
1250                 }
1251 
1252                 Runnable resultDispatch = new Runnable() {
1253                     @Override
1254                     public void run() {
1255                         if (!CameraDeviceImpl.this.isClosed()) {
1256                             if (DEBUG) {
1257                                 Log.d(TAG, String.format(
1258                                         "early trigger sequence complete for request %d",
1259                                         requestId));
1260                             }
1261                             holder.getCallback().onCaptureSequenceAborted(
1262                                     CameraDeviceImpl.this,
1263                                     requestId);
1264                         }
1265                     }
1266                 };
1267                 final long ident = Binder.clearCallingIdentity();
1268                 try {
1269                     holder.getExecutor().execute(resultDispatch);
1270                 } finally {
1271                     Binder.restoreCallingIdentity(ident);
1272                 }
1273             } else {
1274                 Log.w(TAG, String.format(
1275                         "did not register callback to request %d",
1276                         requestId));
1277             }
1278         } else {
1279             // This function is only called for regular/ZslStill request so lastFrameNumber is the
1280             // last regular/ZslStill frame number.
1281             mRequestLastFrameNumbersList.add(new RequestLastFrameNumbersHolder(requestId,
1282                     lastFrameNumber, repeatingRequestTypes));
1283 
1284             // It is possible that the last frame has already arrived, so we need to check
1285             // for sequence completion right away
1286             checkAndFireSequenceComplete();
1287         }
1288     }
1289 
getRequestTypes(final CaptureRequest[] requestArray)1290     private int[] getRequestTypes(final CaptureRequest[] requestArray) {
1291         int[] requestTypes = new int[requestArray.length];
1292         for (int i = 0; i < requestArray.length; i++) {
1293             requestTypes[i] = requestArray[i].getRequestType();
1294         }
1295         return requestTypes;
1296     }
1297 
hasBatchedOutputs(List<CaptureRequest> requestList)1298     private boolean hasBatchedOutputs(List<CaptureRequest> requestList) {
1299         boolean hasBatchedOutputs = true;
1300         for (int i = 0; i < requestList.size(); i++) {
1301             CaptureRequest request = requestList.get(i);
1302             if (!request.isPartOfCRequestList()) {
1303                 hasBatchedOutputs = false;
1304                 break;
1305             }
1306             if (i == 0) {
1307                 Collection<Surface> targets = request.getTargets();
1308                 if (targets.size() != 2) {
1309                     hasBatchedOutputs = false;
1310                     break;
1311                 }
1312             }
1313         }
1314         return hasBatchedOutputs;
1315     }
1316 
updateTracker(int requestId, long frameNumber, int requestType, CaptureResult result, boolean isPartialResult)1317     private void updateTracker(int requestId, long frameNumber,
1318             int requestType, CaptureResult result, boolean isPartialResult) {
1319         int requestCount = 1;
1320         // If the request has batchedOutputs update each frame within the batch.
1321         if (mBatchOutputMap.containsKey(requestId)) {
1322             requestCount = mBatchOutputMap.get(requestId);
1323             for (int i = 0; i < requestCount; i++) {
1324                 mFrameNumberTracker.updateTracker(frameNumber - (requestCount - 1 - i),
1325                         result, isPartialResult, requestType);
1326             }
1327         } else {
1328             mFrameNumberTracker.updateTracker(frameNumber, result,
1329                     isPartialResult, requestType);
1330         }
1331     }
1332 
submitCaptureRequest(List<CaptureRequest> requestList, CaptureCallback callback, Executor executor, boolean repeating)1333     private int submitCaptureRequest(List<CaptureRequest> requestList, CaptureCallback callback,
1334             Executor executor, boolean repeating) throws CameraAccessException {
1335 
1336         // Need a valid executor, or current thread needs to have a looper, if
1337         // callback is valid
1338         executor = checkExecutor(executor, callback);
1339 
1340         synchronized(mInterfaceLock) {
1341             checkIfCameraClosedOrInError();
1342 
1343             // Make sure that there all requests have at least 1 surface; all surfaces are non-null;
1344             for (CaptureRequest request : requestList) {
1345                 if (request.getTargets().isEmpty()) {
1346                     throw new IllegalArgumentException(
1347                             "Each request must have at least one Surface target");
1348                 }
1349 
1350                 for (Surface surface : request.getTargets()) {
1351                     if (surface == null) {
1352                         throw new IllegalArgumentException("Null Surface targets are not allowed");
1353                     }
1354                 }
1355             }
1356 
1357             if (repeating) {
1358                 stopRepeating();
1359             }
1360 
1361             SubmitInfo requestInfo;
1362 
1363             CaptureRequest[] requestArray = requestList.toArray(new CaptureRequest[requestList.size()]);
1364             // Convert Surface to streamIdx and surfaceIdx
1365             for (CaptureRequest request : requestArray) {
1366                 request.convertSurfaceToStreamId(mConfiguredOutputs);
1367             }
1368 
1369             requestInfo = mRemoteDevice.submitRequestList(requestArray, repeating);
1370             if (DEBUG) {
1371                 Log.v(TAG, "last frame number " + requestInfo.getLastFrameNumber());
1372             }
1373 
1374             for (CaptureRequest request : requestArray) {
1375                 request.recoverStreamIdToSurface();
1376             }
1377 
1378             // If the request has batched outputs, then store the
1379             // requestCount and requestId in the map.
1380             boolean hasBatchedOutputs = hasBatchedOutputs(requestList);
1381             if (hasBatchedOutputs) {
1382                 int requestCount = requestList.size();
1383                 mBatchOutputMap.put(requestInfo.getRequestId(), requestCount);
1384             }
1385 
1386             if (callback != null) {
1387                 mCaptureCallbackMap.put(requestInfo.getRequestId(),
1388                         new CaptureCallbackHolder(
1389                             callback, requestList, executor, repeating, mNextSessionId - 1));
1390             } else {
1391                 if (DEBUG) {
1392                     Log.d(TAG, "Listen for request " + requestInfo.getRequestId() + " is null");
1393                 }
1394             }
1395 
1396             if (repeating) {
1397                 if (mRepeatingRequestId != REQUEST_ID_NONE) {
1398                     checkEarlyTriggerSequenceCompleteLocked(mRepeatingRequestId,
1399                             requestInfo.getLastFrameNumber(),
1400                             mRepeatingRequestTypes);
1401                 }
1402                 mRepeatingRequestId = requestInfo.getRequestId();
1403                 mRepeatingRequestTypes = getRequestTypes(requestArray);
1404             } else {
1405                 mRequestLastFrameNumbersList.add(
1406                     new RequestLastFrameNumbersHolder(requestList, requestInfo));
1407             }
1408 
1409             if (mIdle) {
1410                 mDeviceExecutor.execute(mCallOnActive);
1411             }
1412             mIdle = false;
1413 
1414             return requestInfo.getRequestId();
1415         }
1416     }
1417 
setRepeatingRequest(CaptureRequest request, CaptureCallback callback, Executor executor)1418     public int setRepeatingRequest(CaptureRequest request, CaptureCallback callback,
1419             Executor executor) throws CameraAccessException {
1420         List<CaptureRequest> requestList = new ArrayList<CaptureRequest>();
1421         requestList.add(request);
1422         return submitCaptureRequest(requestList, callback, executor, /*streaming*/true);
1423     }
1424 
setRepeatingBurst(List<CaptureRequest> requests, CaptureCallback callback, Executor executor)1425     public int setRepeatingBurst(List<CaptureRequest> requests, CaptureCallback callback,
1426             Executor executor) throws CameraAccessException {
1427         if (requests == null || requests.isEmpty()) {
1428             throw new IllegalArgumentException("At least one request must be given");
1429         }
1430         return submitCaptureRequest(requests, callback, executor, /*streaming*/true);
1431     }
1432 
stopRepeating()1433     public void stopRepeating() throws CameraAccessException {
1434 
1435         synchronized(mInterfaceLock) {
1436             checkIfCameraClosedOrInError();
1437             if (mRepeatingRequestId != REQUEST_ID_NONE) {
1438 
1439                 int requestId = mRepeatingRequestId;
1440                 mRepeatingRequestId = REQUEST_ID_NONE;
1441                 mFailedRepeatingRequestId = REQUEST_ID_NONE;
1442                 int[] requestTypes = mRepeatingRequestTypes;
1443                 mRepeatingRequestTypes = null;
1444                 mFailedRepeatingRequestTypes = null;
1445 
1446                 long lastFrameNumber;
1447                 try {
1448                     lastFrameNumber = mRemoteDevice.cancelRequest(requestId);
1449                 } catch (IllegalArgumentException e) {
1450                     if (DEBUG) {
1451                         Log.v(TAG, "Repeating request was already stopped for request " +
1452                                 requestId);
1453                     }
1454                     // Cache request id and request types in case of a race with
1455                     // "onRepeatingRequestError" which may no yet be scheduled on another thread
1456                     // or blocked by us.
1457                     mFailedRepeatingRequestId = requestId;
1458                     mFailedRepeatingRequestTypes = requestTypes;
1459 
1460                     // Repeating request was already stopped. Nothing more to do.
1461                     return;
1462                 }
1463 
1464                 checkEarlyTriggerSequenceCompleteLocked(requestId, lastFrameNumber, requestTypes);
1465             }
1466         }
1467     }
1468 
waitUntilIdle()1469     private void waitUntilIdle() throws CameraAccessException {
1470 
1471         synchronized(mInterfaceLock) {
1472             checkIfCameraClosedOrInError();
1473 
1474             if (mRepeatingRequestId != REQUEST_ID_NONE) {
1475                 throw new IllegalStateException("Active repeating request ongoing");
1476             }
1477 
1478             mRemoteDevice.waitUntilIdle();
1479         }
1480     }
1481 
flush()1482     public void flush() throws CameraAccessException {
1483         synchronized(mInterfaceLock) {
1484             checkIfCameraClosedOrInError();
1485 
1486             mDeviceExecutor.execute(mCallOnBusy);
1487 
1488             // If already idle, just do a busy->idle transition immediately, don't actually
1489             // flush.
1490             if (mIdle) {
1491                 mDeviceExecutor.execute(mCallOnIdle);
1492                 return;
1493             }
1494 
1495             long lastFrameNumber = mRemoteDevice.flush();
1496             if (mRepeatingRequestId != REQUEST_ID_NONE) {
1497                 checkEarlyTriggerSequenceCompleteLocked(mRepeatingRequestId, lastFrameNumber,
1498                         mRepeatingRequestTypes);
1499                 mRepeatingRequestId = REQUEST_ID_NONE;
1500                 mRepeatingRequestTypes = null;
1501             }
1502         }
1503     }
1504 
1505     @Override
close()1506     public void close() {
1507         synchronized (mInterfaceLock) {
1508             if (mClosing.getAndSet(true)) {
1509                 return;
1510             }
1511 
1512             if (mOfflineSwitchService != null) {
1513                 mOfflineSwitchService.shutdownNow();
1514                 mOfflineSwitchService = null;
1515             }
1516 
1517             // Let extension sessions commit stats before disconnecting remoteDevice
1518             if (mCurrentExtensionSession != null) {
1519                 mCurrentExtensionSession.commitStats();
1520             }
1521 
1522             if (mCurrentAdvancedExtensionSession != null) {
1523                 mCurrentAdvancedExtensionSession.commitStats();
1524             }
1525 
1526             if (mRemoteDevice != null) {
1527                 mRemoteDevice.disconnect();
1528                 mRemoteDevice.unlinkToDeath(this, /*flags*/0);
1529             }
1530 
1531             if (mCurrentExtensionSession != null) {
1532                 mCurrentExtensionSession.release(true /*skipCloseNotification*/);
1533                 mCurrentExtensionSession = null;
1534             }
1535 
1536             if (mCurrentAdvancedExtensionSession != null) {
1537                 mCurrentAdvancedExtensionSession.release(true /*skipCloseNotification*/);
1538                 mCurrentAdvancedExtensionSession = null;
1539             }
1540 
1541             // Only want to fire the onClosed callback once;
1542             // either a normal close where the remote device is valid
1543             // or a close after a startup error (no remote device but in error state)
1544             if (mRemoteDevice != null || mInError) {
1545                 mDeviceExecutor.execute(mCallOnClosed);
1546             }
1547 
1548             mRemoteDevice = null;
1549         }
1550     }
1551 
1552     @Override
finalize()1553     protected void finalize() throws Throwable {
1554         try {
1555             close();
1556         }
1557         finally {
1558             super.finalize();
1559         }
1560     }
1561 
checkInputConfigurationWithStreamConfigurationsAs( InputConfiguration inputConfig, StreamConfigurationMap configMap)1562     private boolean checkInputConfigurationWithStreamConfigurationsAs(
1563             InputConfiguration inputConfig, StreamConfigurationMap configMap) {
1564         int[] inputFormats = configMap.getInputFormats();
1565         boolean validFormat = false;
1566         int inputFormat = inputConfig.getFormat();
1567         for (int format : inputFormats) {
1568             if (format == inputFormat) {
1569                 validFormat = true;
1570             }
1571         }
1572 
1573         // Allow RAW formats, even when not advertised.
1574         if (isRawFormat(inputFormat)) {
1575             return true;
1576         }
1577 
1578         if (validFormat == false) {
1579             return false;
1580         }
1581 
1582         boolean validSize = false;
1583         Size[] inputSizes = configMap.getInputSizes(inputFormat);
1584         for (Size s : inputSizes) {
1585             if (inputConfig.getWidth() == s.getWidth() &&
1586                     inputConfig.getHeight() == s.getHeight()) {
1587                 validSize = true;
1588             }
1589         }
1590 
1591         if (validSize == false) {
1592             return false;
1593         }
1594         return true;
1595     }
1596 
checkInputConfigurationWithStreamConfigurations( InputConfiguration inputConfig, boolean maxResolution)1597     private boolean checkInputConfigurationWithStreamConfigurations(
1598             InputConfiguration inputConfig, boolean maxResolution) {
1599         // Check if either this logical camera or any of its physical cameras support the
1600         // input config. If they do, the input config is valid.
1601         CameraCharacteristics.Key<StreamConfigurationMap> ck =
1602                 CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP;
1603 
1604         if (maxResolution) {
1605             ck = CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP_MAXIMUM_RESOLUTION;
1606         }
1607 
1608         StreamConfigurationMap configMap = mCharacteristics.get(ck);
1609 
1610         if (configMap != null &&
1611                 checkInputConfigurationWithStreamConfigurationsAs(inputConfig, configMap)) {
1612             return true;
1613         }
1614 
1615         for (Map.Entry<String, CameraCharacteristics> entry : getPhysicalIdToChars().entrySet()) {
1616             configMap = entry.getValue().get(ck);
1617 
1618             if (configMap != null &&
1619                     checkInputConfigurationWithStreamConfigurationsAs(inputConfig, configMap)) {
1620                 // Input config supported.
1621                 return true;
1622             }
1623         }
1624         return false;
1625     }
1626 
checkInputConfiguration(InputConfiguration inputConfig)1627     private void checkInputConfiguration(InputConfiguration inputConfig) {
1628         if (inputConfig == null) {
1629             return;
1630         }
1631         int inputFormat = inputConfig.getFormat();
1632         if (inputConfig.isMultiResolution()) {
1633             MultiResolutionStreamConfigurationMap configMap = mCharacteristics.get(
1634                     CameraCharacteristics.SCALER_MULTI_RESOLUTION_STREAM_CONFIGURATION_MAP);
1635 
1636             int[] inputFormats = configMap.getInputFormats();
1637             boolean validFormat = false;
1638             for (int format : inputFormats) {
1639                 if (format == inputFormat) {
1640                     validFormat = true;
1641                 }
1642             }
1643 
1644             // Allow RAW formats, even when not advertised.
1645             if (Flags.multiResRawReprocessing() && isRawFormat(inputFormat)) {
1646                 return;
1647             }
1648 
1649             if (validFormat == false) {
1650                 throw new IllegalArgumentException("multi-resolution input format " +
1651                         inputFormat + " is not valid");
1652             }
1653 
1654             boolean validSize = false;
1655             Collection<MultiResolutionStreamInfo> inputStreamInfo =
1656                     configMap.getInputInfo(inputFormat);
1657             for (MultiResolutionStreamInfo info : inputStreamInfo) {
1658                 if (inputConfig.getWidth() == info.getWidth() &&
1659                         inputConfig.getHeight() == info.getHeight()) {
1660                     validSize = true;
1661                 }
1662             }
1663 
1664             if (validSize == false) {
1665                 throw new IllegalArgumentException("Multi-resolution input size " +
1666                         inputConfig.getWidth() + "x" + inputConfig.getHeight() + " is not valid");
1667             }
1668         } else {
1669             if (!checkInputConfigurationWithStreamConfigurations(inputConfig, /*maxRes*/false) &&
1670                     !checkInputConfigurationWithStreamConfigurations(inputConfig, /*maxRes*/true)) {
1671                 throw new IllegalArgumentException("Input config with format " +
1672                         inputFormat + " and size " + inputConfig.getWidth() + "x" +
1673                         inputConfig.getHeight() + " not supported by camera id " + mCameraId);
1674             }
1675         }
1676     }
1677 
1678     /**
1679      * A callback for notifications about the state of a camera device, adding in the callbacks that
1680      * were part of the earlier KK API design, but now only used internally.
1681      */
1682     public static abstract class StateCallbackKK extends StateCallback {
1683         /**
1684          * The method called when a camera device has no outputs configured.
1685          *
1686          */
onUnconfigured(CameraDevice camera)1687         public void onUnconfigured(CameraDevice camera) {
1688             // Default empty implementation
1689         }
1690 
1691         /**
1692          * The method called when a camera device begins processing
1693          * {@link CaptureRequest capture requests}.
1694          *
1695          */
onActive(CameraDevice camera)1696         public void onActive(CameraDevice camera) {
1697             // Default empty implementation
1698         }
1699 
1700         /**
1701          * The method called when a camera device is busy.
1702          *
1703          */
onBusy(CameraDevice camera)1704         public void onBusy(CameraDevice camera) {
1705             // Default empty implementation
1706         }
1707 
1708         /**
1709          * The method called when a camera device has finished processing all
1710          * submitted capture requests and has reached an idle state.
1711          *
1712          */
onIdle(CameraDevice camera)1713         public void onIdle(CameraDevice camera) {
1714             // Default empty implementation
1715         }
1716 
1717         /**
1718          * This method is called when camera device's non-repeating request queue is empty,
1719          * and is ready to start capturing next image.
1720          */
onRequestQueueEmpty()1721         public void onRequestQueueEmpty() {
1722             // Default empty implementation
1723         }
1724 
1725         /**
1726          * The method called when the camera device has finished preparing
1727          * an output Surface
1728          */
onSurfacePrepared(Surface surface)1729         public void onSurfacePrepared(Surface surface) {
1730             // Default empty implementation
1731         }
1732     }
1733 
checkAndFireSequenceComplete()1734     private void checkAndFireSequenceComplete() {
1735         long completedFrameNumber = mFrameNumberTracker.getCompletedFrameNumber();
1736         long completedReprocessFrameNumber = mFrameNumberTracker.getCompletedReprocessFrameNumber();
1737         long completedZslStillFrameNumber = mFrameNumberTracker.getCompletedZslStillFrameNumber();
1738 
1739         Iterator<RequestLastFrameNumbersHolder> iter = mRequestLastFrameNumbersList.iterator();
1740         while (iter.hasNext()) {
1741             final RequestLastFrameNumbersHolder requestLastFrameNumbers = iter.next();
1742             final int requestId = requestLastFrameNumbers.getRequestId();
1743             final CaptureCallbackHolder holder;
1744             if (mRemoteDevice == null) {
1745                 Log.w(TAG, "Camera closed while checking sequences");
1746                 return;
1747             }
1748             if (!requestLastFrameNumbers.isSequenceCompleted()) {
1749                 long lastRegularFrameNumber =
1750                         requestLastFrameNumbers.getLastRegularFrameNumber();
1751                 long lastReprocessFrameNumber =
1752                         requestLastFrameNumbers.getLastReprocessFrameNumber();
1753                 long lastZslStillFrameNumber =
1754                         requestLastFrameNumbers.getLastZslStillFrameNumber();
1755                 if (lastRegularFrameNumber <= completedFrameNumber
1756                         && lastReprocessFrameNumber <= completedReprocessFrameNumber
1757                         && lastZslStillFrameNumber <= completedZslStillFrameNumber) {
1758                     if (DEBUG) {
1759                         Log.v(TAG, String.format(
1760                                 "Mark requestId %d as completed, because lastRegularFrame %d "
1761                                 + "is <= %d, lastReprocessFrame %d is <= %d, "
1762                                 + "lastZslStillFrame %d is <= %d", requestId,
1763                                 lastRegularFrameNumber, completedFrameNumber,
1764                                 lastReprocessFrameNumber, completedReprocessFrameNumber,
1765                                 lastZslStillFrameNumber, completedZslStillFrameNumber));
1766                     }
1767                     requestLastFrameNumbers.markSequenceCompleted();
1768                 }
1769 
1770                 // Call onCaptureSequenceCompleted
1771                 int index = mCaptureCallbackMap.indexOfKey(requestId);
1772                 holder = (index >= 0) ?
1773                         mCaptureCallbackMap.valueAt(index) : null;
1774                 if (holder != null && requestLastFrameNumbers.isSequenceCompleted()) {
1775                     Runnable resultDispatch = new Runnable() {
1776                         @Override
1777                         public void run() {
1778                             if (!CameraDeviceImpl.this.isClosed()){
1779                                 if (DEBUG) {
1780                                     Log.d(TAG, String.format(
1781                                             "fire sequence complete for request %d",
1782                                             requestId));
1783                                 }
1784 
1785                                 holder.getCallback().onCaptureSequenceCompleted(
1786                                     CameraDeviceImpl.this,
1787                                     requestId,
1788                                     requestLastFrameNumbers.getLastFrameNumber());
1789                             }
1790                         }
1791                     };
1792                     final long ident = Binder.clearCallingIdentity();
1793                     try {
1794                         holder.getExecutor().execute(resultDispatch);
1795                     } finally {
1796                         Binder.restoreCallingIdentity(ident);
1797                     }
1798                 }
1799             }
1800 
1801             if (requestLastFrameNumbers.isSequenceCompleted() &&
1802                     requestLastFrameNumbers.isInflightCompleted()) {
1803                 int index = mCaptureCallbackMap.indexOfKey(requestId);
1804                 if (index >= 0) {
1805                     mCaptureCallbackMap.removeAt(index);
1806                 }
1807                 if (DEBUG) {
1808                     Log.v(TAG, String.format(
1809                             "Remove holder for requestId %d", requestId));
1810                 }
1811                 iter.remove();
1812             }
1813         }
1814     }
1815 
removeCompletedCallbackHolderLocked(long lastCompletedRegularFrameNumber, long lastCompletedReprocessFrameNumber, long lastCompletedZslStillFrameNumber)1816     private void removeCompletedCallbackHolderLocked(long lastCompletedRegularFrameNumber,
1817             long lastCompletedReprocessFrameNumber, long lastCompletedZslStillFrameNumber) {
1818         if (DEBUG) {
1819             Log.v(TAG, String.format("remove completed callback holders for "
1820                     + "lastCompletedRegularFrameNumber %d, "
1821                     + "lastCompletedReprocessFrameNumber %d, "
1822                     + "lastCompletedZslStillFrameNumber %d",
1823                     lastCompletedRegularFrameNumber,
1824                     lastCompletedReprocessFrameNumber,
1825                     lastCompletedZslStillFrameNumber));
1826         }
1827 
1828         Iterator<RequestLastFrameNumbersHolder> iter = mRequestLastFrameNumbersList.iterator();
1829         while (iter.hasNext()) {
1830             final RequestLastFrameNumbersHolder requestLastFrameNumbers = iter.next();
1831             final int requestId = requestLastFrameNumbers.getRequestId();
1832             final CaptureCallbackHolder holder;
1833             if (mRemoteDevice == null) {
1834                 Log.w(TAG, "Camera closed while removing completed callback holders");
1835                 return;
1836             }
1837 
1838             long lastRegularFrameNumber =
1839                     requestLastFrameNumbers.getLastRegularFrameNumber();
1840             long lastReprocessFrameNumber =
1841                     requestLastFrameNumbers.getLastReprocessFrameNumber();
1842             long lastZslStillFrameNumber =
1843                     requestLastFrameNumbers.getLastZslStillFrameNumber();
1844 
1845             if (lastRegularFrameNumber <= lastCompletedRegularFrameNumber
1846                         && lastReprocessFrameNumber <= lastCompletedReprocessFrameNumber
1847                         && lastZslStillFrameNumber <= lastCompletedZslStillFrameNumber) {
1848 
1849                 if (requestLastFrameNumbers.isSequenceCompleted()) {
1850                     int index = mCaptureCallbackMap.indexOfKey(requestId);
1851                     if (index >= 0) {
1852                         mCaptureCallbackMap.removeAt(index);
1853                     }
1854                     if (DEBUG) {
1855                         Log.v(TAG, String.format(
1856                                 "Remove holder for requestId %d, because lastRegularFrame %d "
1857                                 + "is <= %d, lastReprocessFrame %d is <= %d, "
1858                                 + "lastZslStillFrame %d is <= %d", requestId,
1859                                 lastRegularFrameNumber, lastCompletedRegularFrameNumber,
1860                                 lastReprocessFrameNumber, lastCompletedReprocessFrameNumber,
1861                                 lastZslStillFrameNumber, lastCompletedZslStillFrameNumber));
1862                     }
1863                     iter.remove();
1864                 } else {
1865                     if (DEBUG) {
1866                         Log.v(TAG, "Sequence not yet completed for request id " + requestId);
1867                     }
1868                     requestLastFrameNumbers.markInflightCompleted();
1869                 }
1870             }
1871         }
1872     }
1873 
onDeviceError(final int errorCode, CaptureResultExtras resultExtras)1874     public void onDeviceError(final int errorCode, CaptureResultExtras resultExtras) {
1875         if (DEBUG) {
1876             Log.d(TAG, String.format(
1877                     "Device error received, code %d, frame number %d, request ID %d, subseq ID %d",
1878                     errorCode, resultExtras.getFrameNumber(), resultExtras.getRequestId(),
1879                     resultExtras.getSubsequenceId()));
1880         }
1881 
1882         synchronized(mInterfaceLock) {
1883             if (mRemoteDevice == null && mRemoteDeviceInit) {
1884                 return; // Camera already closed, user is not interested in errors anymore.
1885             }
1886 
1887             // Redirect device callback to the offline session in case we are in the middle
1888             // of an offline switch
1889             if (mOfflineSessionImpl != null) {
1890                 mOfflineSessionImpl.getCallbacks().onDeviceError(errorCode, resultExtras);
1891                 return;
1892             }
1893 
1894             switch (errorCode) {
1895                 case CameraDeviceCallbacks.ERROR_CAMERA_DISCONNECTED: {
1896                     final long ident = Binder.clearCallingIdentity();
1897                     try {
1898                         mDeviceExecutor.execute(mCallOnDisconnected);
1899                     } finally {
1900                         Binder.restoreCallingIdentity(ident);
1901                     }
1902                     break;
1903                 }
1904                 case CameraDeviceCallbacks.ERROR_CAMERA_REQUEST:
1905                 case CameraDeviceCallbacks.ERROR_CAMERA_RESULT:
1906                 case CameraDeviceCallbacks.ERROR_CAMERA_BUFFER:
1907                     onCaptureErrorLocked(errorCode, resultExtras);
1908                     break;
1909                 case CameraDeviceCallbacks.ERROR_CAMERA_DEVICE:
1910                     scheduleNotifyError(StateCallback.ERROR_CAMERA_DEVICE);
1911                     break;
1912                 case CameraDeviceCallbacks.ERROR_CAMERA_DISABLED:
1913                     scheduleNotifyError(StateCallback.ERROR_CAMERA_DISABLED);
1914                     break;
1915                 default:
1916                     Log.e(TAG, "Unknown error from camera device: " + errorCode);
1917                     scheduleNotifyError(StateCallback.ERROR_CAMERA_SERVICE);
1918             }
1919         }
1920     }
1921 
scheduleNotifyError(int code)1922     private void scheduleNotifyError(int code) {
1923         mInError = true;
1924         final long ident = Binder.clearCallingIdentity();
1925         try {
1926             mDeviceExecutor.execute(obtainRunnable(
1927                         CameraDeviceImpl::notifyError, this, code).recycleOnUse());
1928         } finally {
1929             Binder.restoreCallingIdentity(ident);
1930         }
1931     }
1932 
notifyError(int code)1933     private void notifyError(int code) {
1934         if (!CameraDeviceImpl.this.isClosed()) {
1935             mDeviceCallback.onError(CameraDeviceImpl.this, code);
1936         }
1937     }
1938 
1939     /**
1940      * Called by onDeviceError for handling single-capture failures.
1941      */
onCaptureErrorLocked(int errorCode, CaptureResultExtras resultExtras)1942     private void onCaptureErrorLocked(int errorCode, CaptureResultExtras resultExtras) {
1943 
1944         final int requestId = resultExtras.getRequestId();
1945         final int subsequenceId = resultExtras.getSubsequenceId();
1946         final long frameNumber = resultExtras.getFrameNumber();
1947         final String errorPhysicalCameraId = resultExtras.getErrorPhysicalCameraId();
1948         final CaptureCallbackHolder holder = mCaptureCallbackMap.get(requestId);
1949 
1950         if (holder == null) {
1951             Log.e(TAG, String.format("Receive capture error on unknown request ID %d",
1952                     requestId));
1953             return;
1954         }
1955 
1956         final CaptureRequest request = holder.getRequest(subsequenceId);
1957 
1958         Runnable failureDispatch = null;
1959         if (errorCode == CameraDeviceCallbacks.ERROR_CAMERA_BUFFER) {
1960             // Because 1 stream id could map to multiple surfaces, we need to specify both
1961             // streamId and surfaceId.
1962             OutputConfiguration config = mConfiguredOutputs.get(
1963                     resultExtras.getErrorStreamId());
1964             if (config == null) {
1965                 Log.v(TAG, String.format(
1966                         "Stream %d has been removed. Skipping buffer lost callback",
1967                         resultExtras.getErrorStreamId()));
1968                 return;
1969             }
1970             for (Surface surface : config.getSurfaces()) {
1971                 if (!request.containsTarget(surface)) {
1972                     continue;
1973                 }
1974                 if (DEBUG) {
1975                     Log.v(TAG, String.format(
1976                             "Lost output buffer reported for frame %d, target %s",
1977                             frameNumber, surface));
1978                 }
1979                 failureDispatch = new Runnable() {
1980                     @Override
1981                     public void run() {
1982                         if (!isClosed()){
1983                             holder.getCallback().onCaptureBufferLost(CameraDeviceImpl.this, request,
1984                                     surface, frameNumber);
1985                         }
1986                     }
1987                 };
1988                 // Dispatch the failure callback
1989                 final long ident = Binder.clearCallingIdentity();
1990                 try {
1991                     holder.getExecutor().execute(failureDispatch);
1992                 } finally {
1993                     Binder.restoreCallingIdentity(ident);
1994                 }
1995             }
1996         } else {
1997             boolean mayHaveBuffers = (errorCode == CameraDeviceCallbacks.ERROR_CAMERA_RESULT);
1998 
1999             // This is only approximate - exact handling needs the camera service and HAL to
2000             // disambiguate between request failures to due abort and due to real errors.  For
2001             // now, assume that if the session believes we're mid-abort, then the error is due
2002             // to abort.
2003             int reason = (mCurrentSession != null && mCurrentSession.isAborting()) ?
2004                     CaptureFailure.REASON_FLUSHED :
2005                     CaptureFailure.REASON_ERROR;
2006 
2007             final CaptureFailure failure = new CaptureFailure(
2008                 request,
2009                 reason,
2010                 mayHaveBuffers,
2011                 requestId,
2012                 frameNumber,
2013                 errorPhysicalCameraId);
2014 
2015             failureDispatch = new Runnable() {
2016                 @Override
2017                 public void run() {
2018                     if (!isClosed()){
2019                         holder.getCallback().onCaptureFailed(CameraDeviceImpl.this, request,
2020                                 failure);
2021                     }
2022                 }
2023             };
2024 
2025             // Fire onCaptureSequenceCompleted if appropriate
2026             if (DEBUG) {
2027                 Log.v(TAG, String.format("got error frame %d", frameNumber));
2028             }
2029 
2030             // Do not update frame number tracker for physical camera result error.
2031             if (errorPhysicalCameraId == null) {
2032                 // Update FrameNumberTracker for every frame during HFR mode.
2033                 if (mBatchOutputMap.containsKey(requestId)) {
2034                     for (int i = 0; i < mBatchOutputMap.get(requestId); i++) {
2035                         mFrameNumberTracker.updateTracker(frameNumber - (subsequenceId - i),
2036                                 /*error*/true, request.getRequestType());
2037                     }
2038                 } else {
2039                     mFrameNumberTracker.updateTracker(frameNumber,
2040                             /*error*/true, request.getRequestType());
2041                 }
2042 
2043                 checkAndFireSequenceComplete();
2044             }
2045 
2046             // Dispatch the failure callback
2047             final long ident = Binder.clearCallingIdentity();
2048             try {
2049                 holder.getExecutor().execute(failureDispatch);
2050             } finally {
2051                 Binder.restoreCallingIdentity(ident);
2052             }
2053         }
2054 
2055     }
2056 
onDeviceIdle()2057     public void onDeviceIdle() {
2058         if (DEBUG) {
2059             Log.d(TAG, "Camera now idle");
2060         }
2061         synchronized(mInterfaceLock) {
2062             if (mRemoteDevice == null) return; // Camera already closed
2063 
2064             // Redirect device callback to the offline session in case we are in the middle
2065             // of an offline switch
2066             if (mOfflineSessionImpl != null) {
2067                 mOfflineSessionImpl.getCallbacks().onDeviceIdle();
2068                 return;
2069             }
2070 
2071             // Remove all capture callbacks now that device has gone to IDLE state.
2072             removeCompletedCallbackHolderLocked(
2073                     Long.MAX_VALUE, /*lastCompletedRegularFrameNumber*/
2074                     Long.MAX_VALUE, /*lastCompletedReprocessFrameNumber*/
2075                     Long.MAX_VALUE /*lastCompletedZslStillFrameNumber*/);
2076 
2077             if (!CameraDeviceImpl.this.mIdle) {
2078                 final long ident = Binder.clearCallingIdentity();
2079                 try {
2080                     mDeviceExecutor.execute(mCallOnIdle);
2081                 } finally {
2082                     Binder.restoreCallingIdentity(ident);
2083                 }
2084             }
2085             mIdle = true;
2086         }
2087     }
2088 
2089     public class CameraDeviceCallbacks extends ICameraDeviceCallbacks.Stub {
2090 
2091         @Override
asBinder()2092         public IBinder asBinder() {
2093             return this;
2094         }
2095 
2096         @Override
onDeviceError(final int errorCode, CaptureResultExtras resultExtras)2097         public void onDeviceError(final int errorCode, CaptureResultExtras resultExtras) {
2098             CameraDeviceImpl.this.onDeviceError(errorCode, resultExtras);
2099         }
2100 
2101         @Override
onRepeatingRequestError(long lastFrameNumber, int repeatingRequestId)2102         public void onRepeatingRequestError(long lastFrameNumber, int repeatingRequestId) {
2103             if (DEBUG) {
2104                 Log.d(TAG, "Repeating request error received. Last frame number is " +
2105                         lastFrameNumber);
2106             }
2107 
2108             synchronized(mInterfaceLock) {
2109                 // Camera is already closed or no repeating request is present.
2110                 if (mRemoteDevice == null || mRepeatingRequestId == REQUEST_ID_NONE) {
2111                     if ((mFailedRepeatingRequestId == repeatingRequestId) &&
2112                             (mFailedRepeatingRequestTypes != null) && (mRemoteDevice != null)) {
2113                         Log.v(TAG, "Resuming stop of failed repeating request with id: " +
2114                                 mFailedRepeatingRequestId);
2115 
2116                         checkEarlyTriggerSequenceCompleteLocked(mFailedRepeatingRequestId,
2117                                 lastFrameNumber, mFailedRepeatingRequestTypes);
2118                         mFailedRepeatingRequestId = REQUEST_ID_NONE;
2119                         mFailedRepeatingRequestTypes = null;
2120                     }
2121                     return;
2122                 }
2123 
2124                 // Redirect device callback to the offline session in case we are in the middle
2125                 // of an offline switch
2126                 if (mOfflineSessionImpl != null) {
2127                     mOfflineSessionImpl.getCallbacks().onRepeatingRequestError(
2128                            lastFrameNumber, repeatingRequestId);
2129                     return;
2130                 }
2131 
2132                 checkEarlyTriggerSequenceCompleteLocked(mRepeatingRequestId, lastFrameNumber,
2133                         mRepeatingRequestTypes);
2134                 // Check if there is already a new repeating request
2135                 if (mRepeatingRequestId == repeatingRequestId) {
2136                     mRepeatingRequestId = REQUEST_ID_NONE;
2137                     mRepeatingRequestTypes = null;
2138                 }
2139             }
2140         }
2141 
2142         @Override
onDeviceIdle()2143         public void onDeviceIdle() {
2144             CameraDeviceImpl.this.onDeviceIdle();
2145         }
2146 
2147         @Override
onCaptureStarted(final CaptureResultExtras resultExtras, final long timestamp)2148         public void onCaptureStarted(final CaptureResultExtras resultExtras, final long timestamp) {
2149             int requestId = resultExtras.getRequestId();
2150             final long frameNumber = resultExtras.getFrameNumber();
2151             final long lastCompletedRegularFrameNumber =
2152                     resultExtras.getLastCompletedRegularFrameNumber();
2153             final long lastCompletedReprocessFrameNumber =
2154                     resultExtras.getLastCompletedReprocessFrameNumber();
2155             final long lastCompletedZslFrameNumber =
2156                     resultExtras.getLastCompletedZslFrameNumber();
2157             final boolean hasReadoutTimestamp = resultExtras.hasReadoutTimestamp();
2158             final long readoutTimestamp = resultExtras.getReadoutTimestamp();
2159 
2160             if (DEBUG) {
2161                 Log.d(TAG, "Capture started for id " + requestId + " frame number " + frameNumber
2162                         + ": completedRegularFrameNumber " + lastCompletedRegularFrameNumber
2163                         + ", completedReprocessFrameNUmber " + lastCompletedReprocessFrameNumber
2164                         + ", completedZslFrameNumber " + lastCompletedZslFrameNumber
2165                         + ", hasReadoutTimestamp " + hasReadoutTimestamp
2166                         + (hasReadoutTimestamp ? ", readoutTimestamp " + readoutTimestamp : "")) ;
2167             }
2168             final CaptureCallbackHolder holder;
2169 
2170             synchronized(mInterfaceLock) {
2171                 if (mRemoteDevice == null) return; // Camera already closed
2172 
2173 
2174                 // Redirect device callback to the offline session in case we are in the middle
2175                 // of an offline switch
2176                 if (mOfflineSessionImpl != null) {
2177                     mOfflineSessionImpl.getCallbacks().onCaptureStarted(resultExtras,
2178                             timestamp);
2179                     return;
2180                 }
2181 
2182                 // Check if it's okay to remove completed callbacks from mCaptureCallbackMap.
2183                 // A callback is completed if the corresponding inflight request has been removed
2184                 // from the inflight queue in cameraservice.
2185                 removeCompletedCallbackHolderLocked(lastCompletedRegularFrameNumber,
2186                         lastCompletedReprocessFrameNumber, lastCompletedZslFrameNumber);
2187 
2188                 // Get the callback for this frame ID, if there is one
2189                 holder = CameraDeviceImpl.this.mCaptureCallbackMap.get(requestId);
2190 
2191                 if (holder == null) {
2192                     return;
2193                 }
2194 
2195                 if (isClosed()) return;
2196 
2197                 // Dispatch capture start notice
2198                 final long ident = Binder.clearCallingIdentity();
2199                 try {
2200                     holder.getExecutor().execute(
2201                         new Runnable() {
2202                             @Override
2203                             public void run() {
2204                                 if (!CameraDeviceImpl.this.isClosed()) {
2205                                     final int subsequenceId = resultExtras.getSubsequenceId();
2206                                     final CaptureRequest request = holder.getRequest(subsequenceId);
2207 
2208                                     if (holder.hasBatchedOutputs()) {
2209                                         // Send derived onCaptureStarted for requests within the
2210                                         // batch
2211                                         final Range<Integer> fpsRange =
2212                                             request.get(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE);
2213                                         for (int i = 0; i < holder.getRequestCount(); i++) {
2214                                             holder.getCallback().onCaptureStarted(
2215                                                 CameraDeviceImpl.this,
2216                                                 holder.getRequest(i),
2217                                                 timestamp - (subsequenceId - i) *
2218                                                 NANO_PER_SECOND / fpsRange.getUpper(),
2219                                                 frameNumber - (subsequenceId - i));
2220                                             if (hasReadoutTimestamp) {
2221                                                 holder.getCallback().onReadoutStarted(
2222                                                     CameraDeviceImpl.this,
2223                                                     holder.getRequest(i),
2224                                                     readoutTimestamp - (subsequenceId - i) *
2225                                                     NANO_PER_SECOND / fpsRange.getUpper(),
2226                                                     frameNumber - (subsequenceId - i));
2227                                             }
2228                                         }
2229                                     } else {
2230                                         holder.getCallback().onCaptureStarted(
2231                                             CameraDeviceImpl.this, request,
2232                                             timestamp, frameNumber);
2233                                         if (hasReadoutTimestamp) {
2234                                             holder.getCallback().onReadoutStarted(
2235                                                 CameraDeviceImpl.this, request,
2236                                                 readoutTimestamp, frameNumber);
2237                                         }
2238                                     }
2239                                 }
2240                             }
2241                         });
2242                 } finally {
2243                     Binder.restoreCallingIdentity(ident);
2244                 }
2245             }
2246         }
2247 
2248         @Override
onResultReceived(CameraMetadataNative result, CaptureResultExtras resultExtras, PhysicalCaptureResultInfo physicalResults[])2249         public void onResultReceived(CameraMetadataNative result,
2250                 CaptureResultExtras resultExtras, PhysicalCaptureResultInfo physicalResults[])
2251                 throws RemoteException {
2252             int requestId = resultExtras.getRequestId();
2253             long frameNumber = resultExtras.getFrameNumber();
2254 
2255             if (DEBUG) {
2256                 Log.v(TAG, "Received result frame " + frameNumber + " for id "
2257                         + requestId);
2258             }
2259 
2260             synchronized(mInterfaceLock) {
2261                 if (mRemoteDevice == null) return; // Camera already closed
2262 
2263 
2264                 // Redirect device callback to the offline session in case we are in the middle
2265                 // of an offline switch
2266                 if (mOfflineSessionImpl != null) {
2267                     mOfflineSessionImpl.getCallbacks().onResultReceived(result, resultExtras,
2268                             physicalResults);
2269                     return;
2270                 }
2271 
2272                 // TODO: Handle CameraCharacteristics access from CaptureResult correctly.
2273                 result.set(CameraCharacteristics.LENS_INFO_SHADING_MAP_SIZE,
2274                         getCharacteristics().get(CameraCharacteristics.LENS_INFO_SHADING_MAP_SIZE));
2275 
2276                 final CaptureCallbackHolder holder =
2277                         CameraDeviceImpl.this.mCaptureCallbackMap.get(requestId);
2278 
2279                 boolean isPartialResult =
2280                         (resultExtras.getPartialResultCount() < mTotalPartialCount);
2281 
2282                 // Check if we have a callback for this
2283                 if (holder == null) {
2284                     if (DEBUG) {
2285                         Log.d(TAG,
2286                                 "holder is null, early return at frame "
2287                                         + frameNumber);
2288                     }
2289 
2290                     return;
2291                 }
2292 
2293                 final CaptureRequest request = holder.getRequest(resultExtras.getSubsequenceId());
2294                 int requestType = request.getRequestType();
2295                 if (isClosed()) {
2296                     if (DEBUG) {
2297                         Log.d(TAG,
2298                                 "camera is closed, early return at frame "
2299                                         + frameNumber);
2300                     }
2301 
2302                     updateTracker(requestId, frameNumber, requestType, /*result*/null,
2303                             isPartialResult);
2304 
2305                     return;
2306                 }
2307 
2308 
2309                 Runnable resultDispatch = null;
2310 
2311                 CaptureResult finalResult;
2312                 // Make a copy of the native metadata before it gets moved to a CaptureResult
2313                 // object.
2314                 final CameraMetadataNative resultCopy;
2315                 if (holder.hasBatchedOutputs()) {
2316                     resultCopy = new CameraMetadataNative(result);
2317                 } else {
2318                     resultCopy = null;
2319                 }
2320 
2321                 // Either send a partial result or the final capture completed result
2322                 if (isPartialResult) {
2323                     final CaptureResult resultAsCapture =
2324                             new CaptureResult(getId(), result, request, resultExtras);
2325                     // Partial result
2326                     resultDispatch = new Runnable() {
2327                         @Override
2328                         public void run() {
2329                             if (!CameraDeviceImpl.this.isClosed()) {
2330                                 if (holder.hasBatchedOutputs()) {
2331                                     // Send derived onCaptureProgressed for requests within
2332                                     // the batch.
2333                                     for (int i = 0; i < holder.getRequestCount(); i++) {
2334                                         CameraMetadataNative resultLocal =
2335                                                 new CameraMetadataNative(resultCopy);
2336                                         CaptureResult resultInBatch = new CaptureResult(getId(),
2337                                                 resultLocal, holder.getRequest(i), resultExtras);
2338 
2339                                         holder.getCallback().onCaptureProgressed(
2340                                             CameraDeviceImpl.this,
2341                                             holder.getRequest(i),
2342                                             resultInBatch);
2343                                     }
2344                                 } else {
2345                                     holder.getCallback().onCaptureProgressed(
2346                                         CameraDeviceImpl.this,
2347                                         request,
2348                                         resultAsCapture);
2349                                 }
2350                             }
2351                         }
2352                     };
2353                     finalResult = resultAsCapture;
2354                 } else {
2355                     List<CaptureResult> partialResults =
2356                             mFrameNumberTracker.popPartialResults(frameNumber);
2357                     if (mBatchOutputMap.containsKey(requestId)) {
2358                         int requestCount = mBatchOutputMap.get(requestId);
2359                         for (int i = 1; i < requestCount; i++) {
2360                             mFrameNumberTracker.popPartialResults(frameNumber - (requestCount - i));
2361                         }
2362                     }
2363 
2364                     final long sensorTimestamp =
2365                             result.get(CaptureResult.SENSOR_TIMESTAMP);
2366                     final Range<Integer> fpsRange =
2367                             request.get(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE);
2368                     final int subsequenceId = resultExtras.getSubsequenceId();
2369                     final TotalCaptureResult resultAsCapture = new TotalCaptureResult(getId(),
2370                             result, request, resultExtras, partialResults, holder.getSessionId(),
2371                             physicalResults);
2372                     // Final capture result
2373                     resultDispatch = new Runnable() {
2374                         @Override
2375                         public void run() {
2376                             if (!CameraDeviceImpl.this.isClosed()){
2377                                 if (holder.hasBatchedOutputs()) {
2378                                     // Send derived onCaptureCompleted for requests within
2379                                     // the batch.
2380                                     for (int i = 0; i < holder.getRequestCount(); i++) {
2381                                         resultCopy.set(CaptureResult.SENSOR_TIMESTAMP,
2382                                                 sensorTimestamp - (subsequenceId - i) *
2383                                                 NANO_PER_SECOND/fpsRange.getUpper());
2384                                         CameraMetadataNative resultLocal =
2385                                                 new CameraMetadataNative(resultCopy);
2386                                         // No logical multi-camera support for batched output mode.
2387                                         TotalCaptureResult resultInBatch = new TotalCaptureResult(
2388                                                 getId(), resultLocal, holder.getRequest(i),
2389                                                 resultExtras, partialResults, holder.getSessionId(),
2390                                                 new PhysicalCaptureResultInfo[0]);
2391 
2392                                         holder.getCallback().onCaptureCompleted(
2393                                             CameraDeviceImpl.this,
2394                                             holder.getRequest(i),
2395                                             resultInBatch);
2396                                     }
2397                                 } else {
2398                                     holder.getCallback().onCaptureCompleted(
2399                                         CameraDeviceImpl.this,
2400                                         request,
2401                                         resultAsCapture);
2402                                 }
2403                             }
2404                         }
2405                     };
2406                     finalResult = resultAsCapture;
2407                 }
2408 
2409                 final long ident = Binder.clearCallingIdentity();
2410                 try {
2411                     holder.getExecutor().execute(resultDispatch);
2412                 } finally {
2413                     Binder.restoreCallingIdentity(ident);
2414                 }
2415 
2416                 updateTracker(requestId, frameNumber, requestType, finalResult, isPartialResult);
2417 
2418                 // Fire onCaptureSequenceCompleted
2419                 if (!isPartialResult) {
2420                     checkAndFireSequenceComplete();
2421                 }
2422             }
2423         }
2424 
2425         @Override
onPrepared(int streamId)2426         public void onPrepared(int streamId) {
2427             final OutputConfiguration output;
2428             final StateCallbackKK sessionCallback;
2429 
2430             if (DEBUG) {
2431                 Log.v(TAG, "Stream " + streamId + " is prepared");
2432             }
2433 
2434             synchronized(mInterfaceLock) {
2435                 // Redirect device callback to the offline session in case we are in the middle
2436                 // of an offline switch
2437                 if (mOfflineSessionImpl != null) {
2438                     mOfflineSessionImpl.getCallbacks().onPrepared(streamId);
2439                     return;
2440                 }
2441 
2442                 output = mConfiguredOutputs.get(streamId);
2443                 sessionCallback = mSessionStateCallback;
2444             }
2445 
2446             if (sessionCallback == null) return;
2447 
2448             if (output == null) {
2449                 Log.w(TAG, "onPrepared invoked for unknown output Surface");
2450                 return;
2451             }
2452             final List<Surface> surfaces = output.getSurfaces();
2453             for (Surface surface : surfaces) {
2454                 sessionCallback.onSurfacePrepared(surface);
2455             }
2456         }
2457 
2458         @Override
onRequestQueueEmpty()2459         public void onRequestQueueEmpty() {
2460             final StateCallbackKK sessionCallback;
2461 
2462             if (DEBUG) {
2463                 Log.v(TAG, "Request queue becomes empty");
2464             }
2465 
2466             synchronized(mInterfaceLock) {
2467                 // Redirect device callback to the offline session in case we are in the middle
2468                 // of an offline switch
2469                 if (mOfflineSessionImpl != null) {
2470                     mOfflineSessionImpl.getCallbacks().onRequestQueueEmpty();
2471                     return;
2472                 }
2473 
2474                 sessionCallback = mSessionStateCallback;
2475             }
2476 
2477             if (sessionCallback == null) return;
2478 
2479             sessionCallback.onRequestQueueEmpty();
2480         }
2481 
2482     } // public class CameraDeviceCallbacks
2483 
2484     /**
2485      * A camera specific adapter {@link Executor} that posts all executed tasks onto the given
2486      * {@link Handler}.
2487      *
2488      * @hide
2489      */
2490     private static class CameraHandlerExecutor implements Executor {
2491         private final Handler mHandler;
2492 
CameraHandlerExecutor(@onNull Handler handler)2493         public CameraHandlerExecutor(@NonNull Handler handler) {
2494             mHandler = Objects.requireNonNull(handler);
2495         }
2496 
2497         @Override
execute(Runnable command)2498         public void execute(Runnable command) {
2499             // Return value of 'post()' will be ignored in order to keep the
2500             // same camera behavior. For further details see b/74605221 .
2501             mHandler.post(command);
2502         }
2503     }
2504 
2505     /**
2506      * Default executor management.
2507      *
2508      * <p>
2509      * If executor is null, get the current thread's
2510      * Looper to create a Executor with. If no looper exists, throw
2511      * {@code IllegalArgumentException}.
2512      * </p>
2513      */
checkExecutor(Executor executor)2514     static Executor checkExecutor(Executor executor) {
2515         return (executor == null) ? checkAndWrapHandler(null) : executor;
2516     }
2517 
2518     /**
2519      * Default executor management.
2520      *
2521      * <p>If the callback isn't null, check the executor, otherwise pass it through.</p>
2522      */
checkExecutor(Executor executor, T callback)2523     public static <T> Executor checkExecutor(Executor executor, T callback) {
2524         return (callback != null) ? checkExecutor(executor) : executor;
2525     }
2526 
2527     /**
2528      * Wrap Handler in Executor.
2529      *
2530      * <p>
2531      * If handler is null, get the current thread's
2532      * Looper to create a Executor with. If no looper exists, throw
2533      * {@code IllegalArgumentException}.
2534      * </p>
2535      */
checkAndWrapHandler(Handler handler)2536     public static Executor checkAndWrapHandler(Handler handler) {
2537         return new CameraHandlerExecutor(checkHandler(handler));
2538     }
2539 
2540     /**
2541      * Default handler management.
2542      *
2543      * <p>
2544      * If handler is null, get the current thread's
2545      * Looper to create a Handler with. If no looper exists, throw {@code IllegalArgumentException}.
2546      * </p>
2547      */
checkHandler(Handler handler)2548     static Handler checkHandler(Handler handler) {
2549         if (handler == null) {
2550             Looper looper = Looper.myLooper();
2551             if (looper == null) {
2552                 throw new IllegalArgumentException(
2553                     "No handler given, and current thread has no looper!");
2554             }
2555             handler = new Handler(looper);
2556         }
2557         return handler;
2558     }
2559 
2560     /**
2561      * Default handler management, conditional on there being a callback.
2562      *
2563      * <p>If the callback isn't null, check the handler, otherwise pass it through.</p>
2564      */
checkHandler(Handler handler, T callback)2565     static <T> Handler checkHandler(Handler handler, T callback) {
2566         if (callback != null) {
2567             return checkHandler(handler);
2568         }
2569         return handler;
2570     }
2571 
checkIfCameraClosedOrInError()2572     private void checkIfCameraClosedOrInError() throws CameraAccessException {
2573         if (mRemoteDevice == null) {
2574             throw new IllegalStateException("CameraDevice was already closed");
2575         }
2576         if (mInError) {
2577             throw new CameraAccessException(CameraAccessException.CAMERA_ERROR,
2578                     "The camera device has encountered a serious error");
2579         }
2580     }
2581 
2582     /** Whether the camera device has started to close (may not yet have finished) */
isClosed()2583     private boolean isClosed() {
2584         return mClosing.get();
2585     }
2586 
getCharacteristics()2587     private CameraCharacteristics getCharacteristics() {
2588         return mCharacteristics;
2589     }
2590 
isRawFormat(int format)2591     private boolean isRawFormat(int format) {
2592         return (format == ImageFormat.RAW_PRIVATE || format == ImageFormat.RAW10
2593                 || format == ImageFormat.RAW12 || format == ImageFormat.RAW_SENSOR);
2594     }
2595 
2596     /**
2597      * Listener for binder death.
2598      *
2599      * <p> Handle binder death for ICameraDeviceUser. Trigger onError.</p>
2600      */
2601     @Override
binderDied()2602     public void binderDied() {
2603         Log.w(TAG, "CameraDevice " + mCameraId + " died unexpectedly");
2604 
2605         if (mRemoteDevice == null) {
2606             return; // Camera already closed
2607         }
2608 
2609         mInError = true;
2610         Runnable r = new Runnable() {
2611             @Override
2612             public void run() {
2613                 if (!isClosed()) {
2614                     mDeviceCallback.onError(CameraDeviceImpl.this,
2615                             StateCallback.ERROR_CAMERA_SERVICE);
2616                 }
2617             }
2618         };
2619         final long ident = Binder.clearCallingIdentity();
2620         try {
2621             CameraDeviceImpl.this.mDeviceExecutor.execute(r);
2622         } finally {
2623             Binder.restoreCallingIdentity(ident);
2624         }
2625     }
2626 
2627     @Override
setCameraAudioRestriction( @AMERA_AUDIO_RESTRICTION int mode)2628     public void setCameraAudioRestriction(
2629             @CAMERA_AUDIO_RESTRICTION int mode) throws CameraAccessException {
2630         synchronized(mInterfaceLock) {
2631             checkIfCameraClosedOrInError();
2632             mRemoteDevice.setCameraAudioRestriction(mode);
2633         }
2634     }
2635 
2636     @Override
getCameraAudioRestriction()2637     public @CAMERA_AUDIO_RESTRICTION int getCameraAudioRestriction() throws CameraAccessException {
2638         synchronized(mInterfaceLock) {
2639             checkIfCameraClosedOrInError();
2640             return mRemoteDevice.getGlobalAudioRestriction();
2641         }
2642     }
2643 
2644     @Override
createExtensionSession(ExtensionSessionConfiguration extensionConfiguration)2645     public void createExtensionSession(ExtensionSessionConfiguration extensionConfiguration)
2646             throws CameraAccessException {
2647         HashMap<String, CameraCharacteristics> characteristicsMap = new HashMap<>(
2648                 getPhysicalIdToChars());
2649         characteristicsMap.put(mCameraId, mCharacteristics);
2650         boolean initializationFailed = true;
2651         IBinder token = new Binder(TAG + " : " + mNextSessionId++);
2652         try {
2653             boolean ret = CameraExtensionCharacteristics.registerClient(mContext, token,
2654                     extensionConfiguration.getExtension(), mCameraId,
2655                     CameraExtensionUtils.getCharacteristicsMapNative(characteristicsMap));
2656             if (!ret) {
2657                 token = null;
2658                 throw new UnsupportedOperationException("Unsupported extension!");
2659             }
2660 
2661             if (CameraExtensionCharacteristics.areAdvancedExtensionsSupported(
2662                         extensionConfiguration.getExtension())) {
2663                 mCurrentAdvancedExtensionSession =
2664                         CameraAdvancedExtensionSessionImpl.createCameraAdvancedExtensionSession(
2665                                 this, characteristicsMap, mContext, extensionConfiguration,
2666                                 mNextSessionId, token);
2667             } else {
2668                 mCurrentExtensionSession = CameraExtensionSessionImpl.createCameraExtensionSession(
2669                         this, characteristicsMap, mContext, extensionConfiguration,
2670                         mNextSessionId, token);
2671             }
2672             initializationFailed = false;
2673         } catch (RemoteException e) {
2674             throw new CameraAccessException(CameraAccessException.CAMERA_ERROR);
2675         } finally {
2676             if (initializationFailed && (token != null)) {
2677                 CameraExtensionCharacteristics.unregisterClient(mContext, token,
2678                         extensionConfiguration.getExtension());
2679             }
2680         }
2681     }
2682 }
2683