1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ANDROID_SERVERS_CAMERA3OFFLINESESSION_H
18 #define ANDROID_SERVERS_CAMERA3OFFLINESESSION_H
19 
20 #include <memory>
21 #include <mutex>
22 
23 #include <utils/String8.h>
24 #include <utils/String16.h>
25 
26 #include <android/hardware/camera/device/3.6/ICameraOfflineSession.h>
27 
28 #include <fmq/MessageQueue.h>
29 
30 #include "common/CameraOfflineSessionBase.h"
31 
32 #include "device3/Camera3BufferManager.h"
33 #include "device3/DistortionMapper.h"
34 #include "device3/InFlightRequest.h"
35 #include "device3/Camera3OutputUtils.h"
36 #include "device3/RotateAndCropMapper.h"
37 #include "device3/ZoomRatioMapper.h"
38 #include "utils/TagMonitor.h"
39 #include "utils/LatencyHistogram.h"
40 #include <camera_metadata_hidden.h>
41 
42 namespace android {
43 
44 namespace camera3 {
45 
46 class Camera3Stream;
47 class Camera3OutputStreamInterface;
48 class Camera3StreamInterface;
49 
50 } // namespace camera3
51 
52 
53 // An immutable struct containing general states that will be copied from Camera3Device to
54 // Camera3OfflineSession
55 struct Camera3OfflineStates {
Camera3OfflineStatesCamera3OfflineStates56     Camera3OfflineStates(
57             const TagMonitor& tagMonitor, const metadata_vendor_id_t vendorTagId,
58             const bool useHalBufManager, const bool needFixupMonochromeTags,
59             const bool usePartialResult, const uint32_t numPartialResults,
60             const int64_t lastCompletedRegularFN, const int64_t lastCompletedReprocessFN,
61             const int64_t lastCompletedZslFN, const uint32_t nextResultFN,
62             const uint32_t nextReprocResultFN, const uint32_t nextZslResultFN,
63             const uint32_t nextShutterFN, const uint32_t nextReprocShutterFN,
64             const uint32_t nextZslShutterFN, const CameraMetadata& deviceInfo,
65             const std::unordered_map<std::string, CameraMetadata>& physicalDeviceInfoMap,
66             const std::unordered_map<std::string, camera3::DistortionMapper>& distortionMappers,
67             const std::unordered_map<std::string, camera3::ZoomRatioMapper>& zoomRatioMappers,
68             const std::unordered_map<std::string, camera3::RotateAndCropMapper>&
69                 rotateAndCropMappers) :
70             mTagMonitor(tagMonitor), mVendorTagId(vendorTagId),
71             mUseHalBufManager(useHalBufManager), mNeedFixupMonochromeTags(needFixupMonochromeTags),
72             mUsePartialResult(usePartialResult), mNumPartialResults(numPartialResults),
73             mLastCompletedRegularFrameNumber(lastCompletedRegularFN),
74             mLastCompletedReprocessFrameNumber(lastCompletedReprocessFN),
75             mLastCompletedZslFrameNumber(lastCompletedZslFN),
76             mNextResultFrameNumber(nextResultFN),
77             mNextReprocessResultFrameNumber(nextReprocResultFN),
78             mNextZslStillResultFrameNumber(nextZslResultFN),
79             mNextShutterFrameNumber(nextShutterFN),
80             mNextReprocessShutterFrameNumber(nextReprocShutterFN),
81             mNextZslStillShutterFrameNumber(nextZslShutterFN),
82             mDeviceInfo(deviceInfo),
83             mPhysicalDeviceInfoMap(physicalDeviceInfoMap),
84             mDistortionMappers(distortionMappers),
85             mZoomRatioMappers(zoomRatioMappers),
86             mRotateAndCropMappers(rotateAndCropMappers) {}
87 
88     const TagMonitor& mTagMonitor;
89     const metadata_vendor_id_t mVendorTagId;
90 
91     const bool mUseHalBufManager;
92     const bool mNeedFixupMonochromeTags;
93 
94     const bool mUsePartialResult;
95     const uint32_t mNumPartialResults;
96 
97     // The last completed (buffers, result metadata, and error notify) regular
98     // request frame number
99     const int64_t mLastCompletedRegularFrameNumber;
100     // The last completed (buffers, result metadata, and error notify) reprocess
101     // request frame number
102     const int64_t mLastCompletedReprocessFrameNumber;
103     // The last completed (buffers, result metadata, and error notify) zsl
104     // request frame number
105     const int64_t mLastCompletedZslFrameNumber;
106     // the minimal frame number of the next non-reprocess result
107     const uint32_t mNextResultFrameNumber;
108     // the minimal frame number of the next reprocess result
109     const uint32_t mNextReprocessResultFrameNumber;
110     // the minimal frame number of the next ZSL still capture result
111     const uint32_t mNextZslStillResultFrameNumber;
112     // the minimal frame number of the next non-reprocess shutter
113     const uint32_t mNextShutterFrameNumber;
114     // the minimal frame number of the next reprocess shutter
115     const uint32_t mNextReprocessShutterFrameNumber;
116     // the minimal frame number of the next ZSL still capture shutter
117     const uint32_t mNextZslStillShutterFrameNumber;
118 
119     const CameraMetadata& mDeviceInfo;
120 
121     const std::unordered_map<std::string, CameraMetadata>& mPhysicalDeviceInfoMap;
122 
123     const std::unordered_map<std::string, camera3::DistortionMapper>& mDistortionMappers;
124 
125     const std::unordered_map<std::string, camera3::ZoomRatioMapper>& mZoomRatioMappers;
126 
127     const std::unordered_map<std::string, camera3::RotateAndCropMapper>& mRotateAndCropMappers;
128 };
129 
130 /**
131  * Camera3OfflineSession for offline session defined in HIDL ICameraOfflineSession@3.6 or higher
132  */
133 class Camera3OfflineSession :
134             public CameraOfflineSessionBase,
135             virtual public hardware::camera::device::V3_5::ICameraDeviceCallback,
136             public camera3::SetErrorInterface,
137             public camera3::InflightRequestUpdateInterface,
138             public camera3::RequestBufferInterface,
139             public camera3::FlushBufferInterface {
140   public:
141 
142     // initialize by Camera3Device.
143     explicit Camera3OfflineSession(const String8& id,
144             const sp<camera3::Camera3Stream>& inputStream,
145             const camera3::StreamSet& offlineStreamSet,
146             camera3::BufferRecords&& bufferRecords,
147             const camera3::InFlightRequestMap& offlineReqs,
148             const Camera3OfflineStates& offlineStates,
149             sp<hardware::camera::device::V3_6::ICameraOfflineSession> offlineSession);
150 
151     virtual ~Camera3OfflineSession();
152 
153     virtual status_t initialize(wp<NotificationListener> listener) override;
154 
155     /**
156      * CameraOfflineSessionBase interface
157      */
158     status_t disconnect() override;
159     status_t dump(int fd) override;
160 
161     /**
162      * FrameProducer interface
163      */
164     const String8& getId() const override;
165     const CameraMetadata& info() const override;
166     status_t waitForNextFrame(nsecs_t timeout) override;
167     status_t getNextResult(CaptureResult *frame) override;
168 
169     // TODO: methods for notification (error/idle/finished etc) passing
170 
171     /**
172      * End of CameraOfflineSessionBase interface
173      */
174 
175     /**
176      * HIDL ICameraDeviceCallback interface
177      */
178 
179     /**
180      * Implementation of android::hardware::camera::device::V3_5::ICameraDeviceCallback
181      */
182 
183     hardware::Return<void> processCaptureResult_3_4(
184             const hardware::hidl_vec<
185                     hardware::camera::device::V3_4::CaptureResult>& results) override;
186     hardware::Return<void> processCaptureResult(
187             const hardware::hidl_vec<
188                     hardware::camera::device::V3_2::CaptureResult>& results) override;
189     hardware::Return<void> notify(
190             const hardware::hidl_vec<
191                     hardware::camera::device::V3_2::NotifyMsg>& msgs) override;
192 
193     hardware::Return<void> requestStreamBuffers(
194             const hardware::hidl_vec<
195                     hardware::camera::device::V3_5::BufferRequest>& bufReqs,
196             requestStreamBuffers_cb _hidl_cb) override;
197 
198     hardware::Return<void> returnStreamBuffers(
199             const hardware::hidl_vec<
200                     hardware::camera::device::V3_2::StreamBuffer>& buffers) override;
201 
202     /**
203      * End of CameraOfflineSessionBase interface
204      */
205 
206   private:
207     // Camera device ID
208     const String8 mId;
209     sp<camera3::Camera3Stream> mInputStream;
210     camera3::StreamSet mOutputStreams;
211     camera3::BufferRecords mBufferRecords;
212 
213     std::mutex mOfflineReqsLock;
214     camera3::InFlightRequestMap mOfflineReqs;
215 
216     sp<hardware::camera::device::V3_6::ICameraOfflineSession> mSession;
217 
218     TagMonitor mTagMonitor;
219     const metadata_vendor_id_t mVendorTagId;
220 
221     const bool mUseHalBufManager;
222     const bool mNeedFixupMonochromeTags;
223 
224     const bool mUsePartialResult;
225     const uint32_t mNumPartialResults;
226 
227     std::mutex mOutputLock;
228     std::list<CaptureResult> mResultQueue;
229     std::condition_variable mResultSignal;
230     // the last completed frame number of regular requests
231     int64_t mLastCompletedRegularFrameNumber;
232     // the last completed frame number of reprocess requests
233     int64_t mLastCompletedReprocessFrameNumber;
234     // the last completed frame number of ZSL still capture requests
235     int64_t mLastCompletedZslFrameNumber;
236     // the minimal frame number of the next non-reprocess result
237     uint32_t mNextResultFrameNumber;
238     // the minimal frame number of the next reprocess result
239     uint32_t mNextReprocessResultFrameNumber;
240     // the minimal frame number of the next ZSL still capture result
241     uint32_t mNextZslStillResultFrameNumber;
242     // the minimal frame number of the next non-reprocess shutter
243     uint32_t mNextShutterFrameNumber;
244     // the minimal frame number of the next reprocess shutter
245     uint32_t mNextReprocessShutterFrameNumber;
246     // the minimal frame number of the next ZSL still capture shutter
247     uint32_t mNextZslStillShutterFrameNumber;
248     // End of mOutputLock scope
249 
250     const CameraMetadata mDeviceInfo;
251     std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap;
252 
253     std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers;
254 
255     std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers;
256 
257     std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers;
258 
259     mutable std::mutex mLock;
260 
261     enum Status {
262         STATUS_UNINITIALIZED = 0,
263         STATUS_ACTIVE,
264         STATUS_ERROR,
265         STATUS_CLOSED
266     } mStatus;
267 
268     wp<NotificationListener> mListener;
269     // End of mLock protect scope
270 
271     std::mutex mProcessCaptureResultLock;
272     // FMQ to write result on. Must be guarded by mProcessCaptureResultLock.
273     std::unique_ptr<ResultMetadataQueue> mResultMetadataQueue;
274 
275     // Tracking cause of fatal errors when in STATUS_ERROR
276     String8 mErrorCause;
277 
278     // Lock to ensure requestStreamBuffers() callbacks are serialized
279     std::mutex mRequestBufferInterfaceLock;
280     // allow request buffer until all requests are processed or disconnectImpl is called
281     bool mAllowRequestBuffer = true;
282 
283     // For client methods such as disconnect/dump
284     std::mutex mInterfaceLock;
285 
286     // SetErrorInterface
287     void setErrorState(const char *fmt, ...) override;
288     void setErrorStateLocked(const char *fmt, ...) override;
289 
290     // InflightRequestUpdateInterface
291     void onInflightEntryRemovedLocked(nsecs_t duration) override;
292     void checkInflightMapLengthLocked() override;
293     void onInflightMapFlushedLocked() override;
294 
295     // RequestBufferInterface
296     bool startRequestBuffer() override;
297     void endRequestBuffer() override;
298     nsecs_t getWaitDuration() override;
299 
300     // FlushBufferInterface
301     void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override;
302     void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override;
303     std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override;
304 
305     void setErrorStateLockedV(const char *fmt, va_list args);
306 
307     status_t disconnectImpl();
308 }; // class Camera3OfflineSession
309 
310 }; // namespace android
311 
312 #endif
313