1 /*
2  * Copyright (C) 2015 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef _ACAMERA_DEVICE_H
17 #define _ACAMERA_DEVICE_H
18 
19 #include <memory>
20 #include <map>
21 #include <set>
22 #include <atomic>
23 #include <utility>
24 #include <vector>
25 #include <utils/StrongPointer.h>
26 #include <utils/Mutex.h>
27 #include <utils/String8.h>
28 #include <utils/List.h>
29 #include <utils/Vector.h>
30 
31 #include <android/hardware/camera2/BnCameraDeviceCallbacks.h>
32 #include <android/hardware/camera2/ICameraDeviceUser.h>
33 #include <media/stagefright/foundation/ALooper.h>
34 #include <media/stagefright/foundation/AHandler.h>
35 #include <media/stagefright/foundation/AMessage.h>
36 #include <camera/CaptureResult.h>
37 #include <camera/camera2/OutputConfiguration.h>
38 #include <camera/camera2/SessionConfiguration.h>
39 #include <camera/camera2/CaptureRequest.h>
40 
41 #include <camera/NdkCameraManager.h>
42 #include <camera/NdkCameraCaptureSession.h>
43 #include "ACameraMetadata.h"
44 
45 namespace android {
46 namespace acam {
47 
48 // Wrap ACameraCaptureFailure so it can be ref-counted
49 struct CameraCaptureFailure : public RefBase, public ACameraCaptureFailure {};
50 
51 // Wrap PhysicalCaptureResultInfo so that it can be ref-counted
52 struct ACameraPhysicalCaptureResultInfo: public RefBase {
ACameraPhysicalCaptureResultInfoACameraPhysicalCaptureResultInfo53     ACameraPhysicalCaptureResultInfo(const std::vector<PhysicalCaptureResultInfo>& info,
54             int64_t frameNumber) :
55         mPhysicalResultInfo(info), mFrameNumber(frameNumber) {}
56 
57     std::vector<PhysicalCaptureResultInfo> mPhysicalResultInfo;
58     int64_t mFrameNumber;
59 };
60 
61 class CameraDevice final : public RefBase {
62   public:
63     CameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
64                   sp<ACameraMetadata> chars,
65                   ACameraDevice* wrapper);
66     ~CameraDevice();
67 
getId()68     inline const char* getId() const { return mCameraId.string(); }
69 
70     camera_status_t createCaptureRequest(
71             ACameraDevice_request_template templateId,
72             const ACameraIdList* physicalIdList,
73             ACaptureRequest** request) const;
74 
75     camera_status_t createCaptureSession(
76             const ACaptureSessionOutputContainer*       outputs,
77             const ACaptureRequest* sessionParameters,
78             const ACameraCaptureSession_stateCallbacks* callbacks,
79             /*out*/ACameraCaptureSession** session);
80 
81     camera_status_t isSessionConfigurationSupported(
82             const ACaptureSessionOutputContainer* sessionOutputContainer) const;
83 
84     // Callbacks from camera service
85     class ServiceCallback : public hardware::camera2::BnCameraDeviceCallbacks {
86       public:
ServiceCallback(CameraDevice * device)87         explicit ServiceCallback(CameraDevice* device) : mDevice(device) {}
88         binder::Status onDeviceError(int32_t errorCode,
89                            const CaptureResultExtras& resultExtras) override;
90         binder::Status onDeviceIdle() override;
91         binder::Status onCaptureStarted(const CaptureResultExtras& resultExtras,
92                               int64_t timestamp) override;
93         binder::Status onResultReceived(const CameraMetadata& metadata,
94                               const CaptureResultExtras& resultExtras,
95                               const std::vector<PhysicalCaptureResultInfo>& physicalResultInfos) override;
96         binder::Status onPrepared(int streamId) override;
97         binder::Status onRequestQueueEmpty() override;
98         binder::Status onRepeatingRequestError(int64_t lastFrameNumber,
99                 int32_t stoppedSequenceId) override;
100       private:
101         const wp<CameraDevice> mDevice;
102     };
getServiceCallback()103     inline sp<hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() {
104         return mServiceCallback;
105     };
106 
107     // Camera device is only functional after remote being set
108     void setRemoteDevice(sp<hardware::camera2::ICameraDeviceUser> remote);
109 
getWrapper()110     inline ACameraDevice* getWrapper() const { return mWrapper; };
111 
112     // Stop the looper thread and unregister the handler
113     void stopLooper();
114 
115   private:
116     friend ACameraCaptureSession;
117     camera_status_t checkCameraClosedOrErrorLocked() const;
118 
119     // device goes into fatal error state after this
120     void setCameraDeviceErrorLocked(camera_status_t error);
121 
122     void disconnectLocked(sp<ACameraCaptureSession>& session); // disconnect from camera service
123 
124     camera_status_t stopRepeatingLocked();
125 
126     camera_status_t flushLocked(ACameraCaptureSession*);
127 
128     camera_status_t waitUntilIdleLocked();
129 
130 
131     template<class T>
132     camera_status_t captureLocked(sp<ACameraCaptureSession> session,
133             /*optional*/T* cbs,
134             int numRequests, ACaptureRequest** requests,
135             /*optional*/int* captureSequenceId);
136 
137     template<class T>
138     camera_status_t setRepeatingRequestsLocked(sp<ACameraCaptureSession> session,
139             /*optional*/T* cbs,
140             int numRequests, ACaptureRequest** requests,
141             /*optional*/int* captureSequenceId);
142 
143     template<class T>
144     camera_status_t submitRequestsLocked(
145             sp<ACameraCaptureSession> session,
146             /*optional*/T* cbs,
147             int numRequests, ACaptureRequest** requests,
148             /*out*/int* captureSequenceId,
149             bool isRepeating);
150 
151     camera_status_t updateOutputConfigurationLocked(ACaptureSessionOutput *output);
152 
153     camera_status_t allocateCaptureRequest(
154             const ACaptureRequest* request, sp<CaptureRequest>& outReq);
155 
156     static ACaptureRequest* allocateACaptureRequest(sp<CaptureRequest>& req,
157             const std::string& deviceId);
158     static void freeACaptureRequest(ACaptureRequest*);
159 
160     // only For session to hold device lock
161     // Always grab device lock before grabbing session lock
lockDeviceForSessionOps()162     void lockDeviceForSessionOps() const { mDeviceLock.lock(); };
unlockDevice()163     void unlockDevice() const { mDeviceLock.unlock(); };
164 
165     // For capture session to notify its end of life
166     void notifySessionEndOfLifeLocked(ACameraCaptureSession* session);
167 
168     camera_status_t configureStreamsLocked(const ACaptureSessionOutputContainer* outputs,
169            const ACaptureRequest* sessionParameters);
170 
171     // Input message will be posted and cleared after this returns
172     void postSessionMsgAndCleanup(sp<AMessage>& msg);
173 
174     static camera_status_t getIGBPfromAnw(
175             ANativeWindow* anw, sp<IGraphicBufferProducer>& out);
176 
177     static camera_status_t getSurfaceFromANativeWindow(
178             ANativeWindow* anw, sp<Surface>& out);
179 
180     mutable Mutex mDeviceLock;
181     const String8 mCameraId;                          // Camera ID
182     const ACameraDevice_StateCallbacks mAppCallbacks; // Callback to app
183     const sp<ACameraMetadata> mChars;                 // Camera characteristics
184     const sp<ServiceCallback> mServiceCallback;
185     ACameraDevice* mWrapper;
186 
187     // stream id -> pair of (ANW* from application, OutputConfiguration used for camera service)
188     std::map<int, std::pair<ANativeWindow*, OutputConfiguration>> mConfiguredOutputs;
189 
190     // TODO: maybe a bool will suffice for synchronous implementation?
191     std::atomic_bool mClosing;
isClosed()192     inline bool isClosed() { return mClosing; }
193 
194     bool mInError = false;
195     camera_status_t mError = ACAMERA_OK;
196     void onCaptureErrorLocked(
197             int32_t errorCode,
198             const CaptureResultExtras& resultExtras);
199 
200     bool mIdle = true;
201     // This will avoid a busy session being deleted before it's back to idle state
202     sp<ACameraCaptureSession> mBusySession;
203 
204     sp<hardware::camera2::ICameraDeviceUser> mRemote;
205 
206     // Looper thread to handle callback to app
207     sp<ALooper> mCbLooper;
208     // definition of handler and message
209     enum {
210         // Device state callbacks
211         kWhatOnDisconnected,   // onDisconnected
212         kWhatOnError,          // onError
213         // Session state callbacks
214         kWhatSessionStateCb,   // onReady, onActive
215         // Capture callbacks
216         kWhatCaptureStart,     // onCaptureStarted
217         kWhatCaptureResult,    // onCaptureProgressed, onCaptureCompleted
218         kWhatLogicalCaptureResult, // onLogicalCameraCaptureCompleted
219         kWhatCaptureFail,      // onCaptureFailed
220         kWhatLogicalCaptureFail, // onLogicalCameraCaptureFailed
221         kWhatCaptureSeqEnd,    // onCaptureSequenceCompleted
222         kWhatCaptureSeqAbort,  // onCaptureSequenceAborted
223         kWhatCaptureBufferLost,// onCaptureBufferLost
224         // Internal cleanup
225         kWhatCleanUpSessions   // Cleanup cached sp<ACameraCaptureSession>
226     };
227     static const char* kContextKey;
228     static const char* kDeviceKey;
229     static const char* kErrorCodeKey;
230     static const char* kCallbackFpKey;
231     static const char* kSessionSpKey;
232     static const char* kCaptureRequestKey;
233     static const char* kTimeStampKey;
234     static const char* kCaptureResultKey;
235     static const char* kPhysicalCaptureResultKey;
236     static const char* kCaptureFailureKey;
237     static const char* kSequenceIdKey;
238     static const char* kFrameNumberKey;
239     static const char* kAnwKey;
240     static const char* kFailingPhysicalCameraId;
241 
242     class CallbackHandler : public AHandler {
243       public:
244         explicit CallbackHandler(const char* id);
245         void onMessageReceived(const sp<AMessage> &msg) override;
246 
247       private:
248         std::string mId;
249         // This handler will cache all capture session sp until kWhatCleanUpSessions
250         // is processed. This is used to guarantee the last session reference is always
251         // being removed in callback thread without holding camera device lock
252         Vector<sp<ACameraCaptureSession>> mCachedSessions;
253     };
254     sp<CallbackHandler> mHandler;
255 
256     /***********************************
257      * Capture session related members *
258      ***********************************/
259     // The current active session
260     wp<ACameraCaptureSession> mCurrentSession;
261     bool mFlushing = false;
262 
263     int mNextSessionId = 0;
264     // TODO: might need another looper/handler to handle callbacks from service
265 
266     static const int REQUEST_ID_NONE = -1;
267     int mRepeatingSequenceId = REQUEST_ID_NONE;
268 
269     // sequence id -> last frame number map
270     std::map<int, int64_t> mSequenceLastFrameNumberMap;
271 
272     struct CallbackHolder {
273         CallbackHolder(sp<ACameraCaptureSession>          session,
274                        const Vector<sp<CaptureRequest> >& requests,
275                        bool                               isRepeating,
276                        ACameraCaptureSession_captureCallbacks* cbs);
277         CallbackHolder(sp<ACameraCaptureSession>          session,
278                        const Vector<sp<CaptureRequest> >& requests,
279                        bool                               isRepeating,
280                        ACameraCaptureSession_logicalCamera_captureCallbacks* lcbs);
281 
282         template <class T>
initCaptureCallbacksCallbackHolder283         void initCaptureCallbacks(T* cbs) {
284             mContext = nullptr;
285             mOnCaptureStarted = nullptr;
286             mOnCaptureProgressed = nullptr;
287             mOnCaptureCompleted = nullptr;
288             mOnLogicalCameraCaptureCompleted = nullptr;
289             mOnLogicalCameraCaptureFailed = nullptr;
290             mOnCaptureFailed = nullptr;
291             mOnCaptureSequenceCompleted = nullptr;
292             mOnCaptureSequenceAborted = nullptr;
293             mOnCaptureBufferLost = nullptr;
294             if (cbs != nullptr) {
295                 mContext = cbs->context;
296                 mOnCaptureStarted = cbs->onCaptureStarted;
297                 mOnCaptureProgressed = cbs->onCaptureProgressed;
298                 mOnCaptureSequenceCompleted = cbs->onCaptureSequenceCompleted;
299                 mOnCaptureSequenceAborted = cbs->onCaptureSequenceAborted;
300                 mOnCaptureBufferLost = cbs->onCaptureBufferLost;
301             }
302         }
303         sp<ACameraCaptureSession>   mSession;
304         Vector<sp<CaptureRequest> > mRequests;
305         const bool                  mIsRepeating;
306         const bool                  mIsLogicalCameraCallback;
307 
308         void*                       mContext;
309         ACameraCaptureSession_captureCallback_start mOnCaptureStarted;
310         ACameraCaptureSession_captureCallback_result mOnCaptureProgressed;
311         ACameraCaptureSession_captureCallback_result mOnCaptureCompleted;
312         ACameraCaptureSession_logicalCamera_captureCallback_result mOnLogicalCameraCaptureCompleted;
313         ACameraCaptureSession_logicalCamera_captureCallback_failed mOnLogicalCameraCaptureFailed;
314         ACameraCaptureSession_captureCallback_failed mOnCaptureFailed;
315         ACameraCaptureSession_captureCallback_sequenceEnd mOnCaptureSequenceCompleted;
316         ACameraCaptureSession_captureCallback_sequenceAbort mOnCaptureSequenceAborted;
317         ACameraCaptureSession_captureCallback_bufferLost mOnCaptureBufferLost;
318     };
319     // sequence id -> callbacks map
320     std::map<int, CallbackHolder> mSequenceCallbackMap;
321 
322     static const int64_t NO_FRAMES_CAPTURED = -1;
323     class FrameNumberTracker {
324       public:
325         // TODO: Called in onResultReceived and onCaptureErrorLocked
326         void updateTracker(int64_t frameNumber, bool isError);
getCompletedFrameNumber()327         inline int64_t getCompletedFrameNumber() { return mCompletedFrameNumber; }
328       private:
329         void update();
330         void updateCompletedFrameNumber(int64_t frameNumber);
331 
332         int64_t mCompletedFrameNumber = NO_FRAMES_CAPTURED;
333         List<int64_t> mSkippedFrameNumbers;
334         std::set<int64_t> mFutureErrorSet;
335     };
336     FrameNumberTracker mFrameNumberTracker;
337 
338     void checkRepeatingSequenceCompleteLocked(const int sequenceId, const int64_t lastFrameNumber);
339     void checkAndFireSequenceCompleteLocked();
340 
341     // Misc variables
342     int32_t mShadingMapSize[2];   // const after constructor
343     int32_t mPartialResultCount;  // const after constructor
344     std::vector<std::string> mPhysicalIds; // const after constructor
345 
346 };
347 
348 } // namespace acam;
349 } // namespace android;
350 
351 /**
352  * ACameraDevice opaque struct definition
353  * Leave outside of android namespace because it's NDK struct
354  */
355 struct ACameraDevice {
ACameraDeviceACameraDevice356     ACameraDevice(const char* id, ACameraDevice_StateCallbacks* cb,
357                   sp<ACameraMetadata> chars) :
358             mDevice(new android::acam::CameraDevice(id, cb, chars, this)) {}
359 
360     ~ACameraDevice();
361 
362     /*******************
363      * NDK public APIs *
364      *******************/
getIdACameraDevice365     inline const char* getId() const { return mDevice->getId(); }
366 
createCaptureRequestACameraDevice367     camera_status_t createCaptureRequest(
368             ACameraDevice_request_template templateId,
369             const ACameraIdList* physicalCameraIdList,
370             ACaptureRequest** request) const {
371         return mDevice->createCaptureRequest(templateId, physicalCameraIdList, request);
372     }
373 
createCaptureSessionACameraDevice374     camera_status_t createCaptureSession(
375             const ACaptureSessionOutputContainer*       outputs,
376             const ACaptureRequest* sessionParameters,
377             const ACameraCaptureSession_stateCallbacks* callbacks,
378             /*out*/ACameraCaptureSession** session) {
379         return mDevice->createCaptureSession(outputs, sessionParameters, callbacks, session);
380     }
381 
isSessionConfigurationSupportedACameraDevice382     camera_status_t isSessionConfigurationSupported(
383             const ACaptureSessionOutputContainer* sessionOutputContainer) const {
384         return mDevice->isSessionConfigurationSupported(sessionOutputContainer);
385     }
386 
387     /***********************
388      * Device interal APIs *
389      ***********************/
getServiceCallbackACameraDevice390     inline android::sp<android::hardware::camera2::ICameraDeviceCallbacks> getServiceCallback() {
391         return mDevice->getServiceCallback();
392     };
393 
394     // Camera device is only functional after remote being set
setRemoteDeviceACameraDevice395     inline void setRemoteDevice(android::sp<android::hardware::camera2::ICameraDeviceUser> remote) {
396         mDevice->setRemoteDevice(remote);
397     }
398 
399   private:
400     android::sp<android::acam::CameraDevice> mDevice;
401 };
402 
403 #endif // _ACAMERA_DEVICE_H
404