1 /*
2  * Copyright (C) 2018 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef _ACAMERA_DEVICE_H
17 #define _ACAMERA_DEVICE_H
18 
19 #include <memory>
20 #include <map>
21 #include <set>
22 #include <atomic>
23 #include <utility>
24 #include <vector>
25 #include <utils/StrongPointer.h>
26 #include <utils/Mutex.h>
27 #include <utils/List.h>
28 #include <utils/Vector.h>
29 #include <android/frameworks/cameraservice/device/2.1/ICameraDeviceUser.h>
30 #include <android/frameworks/cameraservice/device/2.0/ICameraDeviceCallback.h>
31 #include <android/frameworks/cameraservice/device/2.0/types.h>
32 #include <fmq/MessageQueue.h>
33 #include <media/stagefright/foundation/ALooper.h>
34 #include <media/stagefright/foundation/AHandler.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 
37 #include <camera/NdkCameraManager.h>
38 #include <camera/NdkCameraCaptureSession.h>
39 
40 #include "ACameraMetadata.h"
41 #include "utils.h"
42 
43 namespace android {
44 namespace acam {
45 
46 using ICameraDeviceCallback = frameworks::cameraservice::device::V2_0::ICameraDeviceCallback;
47 using ICameraDeviceUser_2_0 = frameworks::cameraservice::device::V2_0::ICameraDeviceUser;
48 using ICameraDeviceUser = frameworks::cameraservice::device::V2_1::ICameraDeviceUser;
49 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras;
50 using PhysicalCaptureResultInfo = frameworks::cameraservice::device::V2_0::PhysicalCaptureResultInfo;
51 using PhysicalCameraSettings = frameworks::cameraservice::device::V2_0::PhysicalCameraSettings;
52 using SubmitInfo = frameworks::cameraservice::device::V2_0::SubmitInfo;
53 using CaptureResultExtras = frameworks::cameraservice::device::V2_0::CaptureResultExtras;
54 using ErrorCode = frameworks::cameraservice::device::V2_0::ErrorCode;
55 using FmqSizeOrMetadata = frameworks::cameraservice::device::V2_0::FmqSizeOrMetadata;
56 using StreamConfigurationMode = frameworks::cameraservice::device::V2_0::StreamConfigurationMode;
57 using Status = frameworks::cameraservice::common::V2_0::Status;
58 using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
59 using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
60 using CameraStatusAndId = frameworks::cameraservice::service::V2_0::CameraStatusAndId;
61 
62 using hardware::hidl_vec;
63 using hardware::hidl_string;
64 using utils::native_handle_ptr_wrapper;
65 using utils::CaptureRequest;
66 using utils::OutputConfigurationWrapper;
67 
68 // Wrap ACameraCaptureFailure so it can be ref-counted
69 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure { };
70 
71 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted
72 struct PhysicalCaptureResultInfoLocal {
73     std::string physicalCameraId;
74     CameraMetadata physicalMetadata;
75 };
76 
77 struct ACameraPhysicalCaptureResultInfo: public RefBase {
ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo78     ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfoLocal>& info,
79             int64_t frameNumber) :
80         mPhysicalResultInfo(info), mFrameNumber(frameNumber) {}
81 
82     std::vector<PhysicalCaptureResultInfoLocal> mPhysicalResultInfo;
83     int64_t mFrameNumber;
84 };
85 
86 class CameraDevice final : public RefBase {
87   public:
88     CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
89                   sp<ACameraMetadata> chars,
90                   ACameraDevice* wrapper);
91     ~CameraDevice();
92 
getId()93     inline const char* getId() const { return mCameraId.c_str(); }
94 
95     camera_status_t createCaptureRequest(
96             ACameraDevice_request_template templateId,
97             const ACameraIdList* physicalCameraIdList,
98             ACaptureRequest** request) const;
99 
100     camera_status_t createCaptureSession(
101             const ACaptureSessionOutputContainer*       outputs,
102             const ACaptureRequest* sessionParameters,
103             const ACameraCaptureSession_stateCallbacks* callbacks,
104             /*out*/ACameraCaptureSession** session);
105 
106     camera_status_t isSessionConfigurationSupported(
107             const ACaptureSessionOutputContainer* sessionOutputContainer) const;
108 
109     // Callbacks from camera service
110     class ServiceCallback : public ICameraDeviceCallback {
111       public:
ServiceCallback(CameraDevice * device)112         explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
113         android::hardware::Return<void> onDeviceError(ErrorCode errorCode,
114                            const CaptureResultExtras& resultExtras) override;
115         android::hardware::Return<void> onDeviceIdle() override;
116         android::hardware::Return<void> onCaptureStarted(const CaptureResultExtras& resultExtras,
117                               uint64_t timestamp) override;
118         android::hardware::Return<void> onResultReceived(const FmqSizeOrMetadata& result,
119                               const CaptureResultExtras& resultExtras,
120                               const hidl_vec<PhysicalCaptureResultInfo>& physicalResultInfos) override;
121         android::hardware::Return<void> onRepeatingRequestError(uint64_t lastFrameNumber,
122                 int32_t stoppedSequenceId) override;
123       private:
124         camera_status_t readOneResultMetadata(const FmqSizeOrMetadata& fmqSizeOrMetadata,
125                 ResultMetadataQueue* metadataQueue, CameraMetadata* metadata);
126         const wp<CameraDevice> mDevice;
127     };
getServiceCallback()128     inline sp<ICameraDeviceCallback> getServiceCallback() {
129         return mServiceCallback;
130     };
131 
132     // Camera device is only functional after remote being set
133     void setRemoteDevice(sp<ICameraDeviceUser> remote);
134 
135     bool setDeviceMetadataQueues();
getWrapper()136     inline ACameraDevice* getWrapper() const { return mWrapper; };
137 
138     // Stop the looper thread and unregister the handler
139     void stopLooperAndDisconnect();
140 
141   private:
142     friend ACameraCaptureSession;
143     friend ACameraDevice;
144 
145     camera_status_t checkCameraClosedOrErrorLocked() const;
146 
147     // device goes into fatal error state after this
148     void setCameraDeviceErrorLocked(camera_status_t error);
149 
150     void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service
151 
152     camera_status_t stopRepeatingLocked();
153 
154     camera_status_t flushLocked(ACameraCaptureSession*);
155 
156     camera_status_t waitUntilIdleLocked();
157 
158     template<class T>
159     camera_status_t captureLocked(sp<ACameraCaptureSession> session,
160             /*optional*/T* cbs,
161             int numRequests, ACaptureRequest** requests,
162             /*optional*/int* captureSequenceId);
163 
164     template<class T>
165     camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session,
166             /*optional*/T* cbs,
167             int numRequests, ACaptureRequest** requests,
168             /*optional*/int* captureSequenceId);
169 
170     template<class T>
171     camera_status_t submitRequestsLocked(
172             sp<ACameraCaptureSession> session,
173             /*optional*/T* cbs,
174             int numRequests, ACaptureRequest** requests,
175             /*out*/int* captureSequenceId,
176             bool isRepeating);
177 
178     void addRequestSettingsMetadata(ACaptureRequest *aCaptureRequest, sp<CaptureRequest> &req);
179 
180     camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output);
181 
182     // Since this writes to ICameraDeviceUser's fmq, clients must take care that:
183     //   a) This function is called serially.
184     //   b) This function is called in accordance with ICameraDeviceUser.submitRequestList,
185     //      otherwise, the wrong capture request might have the wrong settings
186     //      metadata associated with it.
187     camera_status_t allocateCaptureRequestLocked(
188             const ACaptureRequest* request, sp<CaptureRequest>& outReq);
189     void allocateOneCaptureRequestMetadata(
190             PhysicalCameraSettings& cameraSettings,
191             const std::string& id, const sp<ACameraMetadata>& metadata);
192 
193     static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req, const char* deviceId);
194     static void freeACaptureRequest(ACaptureRequest*);
195 
196     // only For session to hold device lock
197     // Always grab device lock before grabbing session lock
lockDeviceForSessionOps()198     void lockDeviceForSessionOps() const { mDeviceLock.lock(); };
unlockDevice()199     void unlockDevice() const { mDeviceLock.unlock(); };
200 
201     // For capture session to notify its end of life
202     void notifySessionEndOfLifeLocked(ACameraCaptureSession* session);
203 
204     camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs,
205            const ACaptureRequest* sessionParameters, nsecs_t startTimeNs);
206 
207     // Input message will be posted and cleared after this returns
208     void postSessionMsgAndCleanup(sp<AMessage>& msg);
209 
210     mutable Mutex mDeviceLock;
211     const hidl_string mCameraId;                          // Camera ID
212     const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app
213     const sp<ACameraMetadata> mChars;    // Camera characteristics
214     const sp<ServiceCallback> mServiceCallback;
215     ACameraDevice* mWrapper;
216 
217     // stream id -> pair of (ACameraWindowType* from application, OutputConfiguration used for
218     // camera service)
219     std::map<int, std::pair<native_handle_ptr_wrapper, OutputConfigurationWrapper>> mConfiguredOutputs;
220 
221     // TODO: maybe a bool will suffice for synchronous implementation?
222     std::atomic_bool mClosing;
isClosed()223     inline bool isClosed() { return mClosing; }
224 
225     bool mInError = false;
226     camera_status_t mError = ACAMERA_OK;
227     void onCaptureErrorLocked(
228             ErrorCode errorCode,
229             const CaptureResultExtras& resultExtras);
230 
231     bool mIdle = true;
232     // This will avoid a busy session being deleted before it's back to idle state
233     sp<ACameraCaptureSession> mBusySession;
234 
235     sp<ICameraDeviceUser> mRemote;
236 
237     // Looper thread to handle callback to app
238     sp<ALooper> mCbLooper;
239     // definition of handler and message
240     enum {
241         // Device state callbacks
242         kWhatOnDisconnected,   // onDisconnected
243         kWhatOnError,          // onError
244         // Session state callbacks
245         kWhatSessionStateCb,   // onReady, onActive
246         // Capture callbacks
247         kWhatCaptureStart,     // onCaptureStarted
248         kWhatCaptureResult,    // onCaptureProgressed, onCaptureCompleted
249         kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted
250         kWhatCaptureFail,      // onCaptureFailed
251         kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed
252         kWhatCaptureSeqEnd,    // onCaptureSequenceCompleted
253         kWhatCaptureSeqAbort,  // onCaptureSequenceAborted
254         kWhatCaptureBufferLost,// onCaptureBufferLost
255         // Internal cleanup
256         kWhatCleanUpSessions   // Cleanup cached sp<ACameraCaptureSession>
257     };
258     static const char* kContextKey;
259     static const char* kDeviceKey;
260     static const char* kErrorCodeKey;
261     static const char* kCallbackFpKey;
262     static const char* kSessionSpKey;
263     static const char* kCaptureRequestKey;
264     static const char* kTimeStampKey;
265     static const char* kCaptureResultKey;
266     static const char* kPhysicalCaptureResultKey;
267     static const char* kCaptureFailureKey;
268     static const char* kSequenceIdKey;
269     static const char* kFrameNumberKey;
270     static const char* kAnwKey;
271     static const char* kFailingPhysicalCameraId;
272 
273     class CallbackHandler : public AHandler {
274       public:
275         explicit CallbackHandler(const char *id);
276         void onMessageReceived(const sp<AMessage> &msg) override;
277 
278       private:
279         std::string mId;
280         // This handler will cache all capture session sp until kWhatCleanUpSessions
281         // is processed. This is used to guarantee the last session reference is always
282         // being removed in callback thread without holding camera device lock
283         Vector<sp<ACameraCaptureSession>> mCachedSessions;
284     };
285     sp<CallbackHandler> mHandler;
286 
287     /***********************************
288      * Capture session related members *
289      ***********************************/
290     // The current active session
291     wp<ACameraCaptureSession> mCurrentSession;
292     bool mFlushing = false;
293 
294     int mNextSessionId = 0;
295     // TODO: might need another looper/handler to handle callbacks from service
296 
297     static const int REQUEST_ID_NONE = -1;
298     int mRepeatingSequenceId = REQUEST_ID_NONE;
299 
300     // sequence id -> last frame number map
301     std::map<int32_t, int64_t> mSequenceLastFrameNumberMap;
302 
303     struct CallbackHolder {
304         CallbackHolder(sp<ACameraCaptureSession>          session,
305                        const Vector<sp<CaptureRequest>>&  requests,
306                        bool                               isRepeating,
307                        ACameraCaptureSession_captureCallbacks* cbs);
308         CallbackHolder(sp<ACameraCaptureSession>          session,
309                        const Vector<sp<CaptureRequest>>&  requests,
310                        bool                               isRepeating,
311                        ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs);
312 
313         template <class T>
initCaptureCallbacksCallbackHolder314         void initCaptureCallbacks(T* cbs) {
315             mContext = nullptr;
316             mOnCaptureStarted = nullptr;
317             mOnCaptureProgressed = nullptr;
318             mOnCaptureCompleted = nullptr;
319             mOnLogicalCameraCaptureCompleted = nullptr;
320             mOnLogicalCameraCaptureFailed = nullptr;
321             mOnCaptureFailed = nullptr;
322             mOnCaptureSequenceCompleted = nullptr;
323             mOnCaptureSequenceAborted = nullptr;
324             mOnCaptureBufferLost = nullptr;
325             if (cbs != nullptr) {
326                 mContext = cbs->context;
327                 mOnCaptureStarted = cbs->onCaptureStarted;
328                 mOnCaptureProgressed = cbs->onCaptureProgressed;
329                 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted;
330                 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted;
331                 mOnCaptureBufferLost = cbs->onCaptureBufferLost;
332             }
333         }
334 
335         sp<ACameraCaptureSession>   mSession;
336         Vector<sp<CaptureRequest>>  mRequests;
337         const bool                  mIsRepeating;
338         const bool                  mIsLogicalCameraCallback;
339 
340         void*                       mContext;
341         ACameraCaptureSession_captureCallback_start mOnCaptureStarted;
342         ACameraCaptureSession_captureCallback_result mOnCaptureProgressed;
343         ACameraCaptureSession_captureCallback_result mOnCaptureCompleted;
344         ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted;
345         ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed;
346         ACameraCaptureSession_captureCallback_failed mOnCaptureFailed;
347         ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted;
348         ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted;
349         ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost;
350     };
351     // sequence id -> callbacks map
352     std::map<int, CallbackHolder> mSequenceCallbackMap;
353 
354     static const int64_t NO_FRAMES_CAPTURED = -1;
355     class FrameNumberTracker {
356       public:
357         // TODO: Called in onResultReceived and onCaptureErrorLocked
358         void updateTracker(int64_t frameNumber, bool isError);
getCompletedFrameNumber()359         inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; }
360       private:
361         void update();
362         void updateCompletedFrameNumber(int64_t frameNumber);
363 
364         int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED;
365         List<int64_t> mSkippedFrameNumbers;
366         std::set<int64_t> mFutureErrorSet;
367     };
368     FrameNumberTracker mFrameNumberTracker;
369 
370     void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber);
371     void checkAndFireSequenceCompleteLocked();
372 
373     // Misc variables
374     int32_t mShadingMapSize[2];   // const after constructor
375     int32_t mPartialResultCount;  // const after constructor
376     std::shared_ptr<ResultMetadataQueue> mCaptureRequestMetadataQueue = nullptr;
377     std::shared_ptr<ResultMetadataQueue> mCaptureResultMetadataQueue = nullptr;
378 };
379 
380 } // namespace acam;
381 } // namespace android;
382 
383 /**
384  * ACameraDevice opaque struct definition
385  * Leave outside of android namespace because it's NDK struct
386  */
387 struct ACameraDevice {
ACameraDeviceACameraDevice388     ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
389                   sp<ACameraMetadata> chars) :
390             mDevice(new android::acam::CameraDevice(id, cb, std::move(chars), this)) {}
391 
392     ~ACameraDevice();
393     /*******************
394      * NDK public APIs *
395      *******************/
getIdACameraDevice396     inline const char* getId() const { return mDevice->getId(); }
397 
createCaptureRequestACameraDevice398     camera_status_t createCaptureRequest(
399             ACameraDevice_request_template templateId,
400             const ACameraIdList* physicalCameraIdList,
401             ACaptureRequest** request) const {
402         return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request);
403     }
404 
createCaptureSessionACameraDevice405     camera_status_t createCaptureSession(
406             const ACaptureSessionOutputContainer*       outputs,
407             const ACaptureRequest* sessionParameters,
408             const ACameraCaptureSession_stateCallbacks* callbacks,
409             /*out*/ACameraCaptureSession** session) {
410         return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session);
411     }
412 
isSessionConfigurationSupportedACameraDevice413     camera_status_t isSessionConfigurationSupported(
414             const ACaptureSessionOutputContainer* sessionOutputContainer) const {
415         return mDevice->isSessionConfigurationSupported(sessionOutputContainer);
416     }
417 
418     /***********************
419      * Device interal APIs *
420      ***********************/
getServiceCallbackACameraDevice421     inline android::sp<android::acam::ICameraDeviceCallback> getServiceCallback() {
422         return mDevice->getServiceCallback();
423     };
424 
425     // Camera device is only functional after remote being set
setRemoteDeviceACameraDevice426     inline void setRemoteDevice(android::sp<android::acam::ICameraDeviceUser> remote) {
427         mDevice->setRemoteDevice(remote);
428     }
setDeviceMetadataQueuesACameraDevice429     inline bool setDeviceMetadataQueues() {
430         return mDevice->setDeviceMetadataQueues();
431     }
432   private:
433     android::sp<android::acam::CameraDevice> mDevice;
434 };
435 
436 #endif // _ACAMERA_DEVICE_H
437