1 /*
2  * Copyright (C) 2012 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #define LOG_TAG "Camera2-FrameProcessor"
18 #define ATRACE_TAG ATRACE_TAG_CAMERA
19 //#define LOG_NDEBUG 0
20 
21 #include <utils/Log.h>
22 #include <utils/Trace.h>
23 
24 #include "common/CameraDeviceBase.h"
25 #include "api1/Camera2Client.h"
26 #include "api1/client2/FrameProcessor.h"
27 
28 namespace android {
29 namespace camera2 {
30 
FrameProcessor(wp<CameraDeviceBase> device,sp<Camera2Client> client)31 FrameProcessor::FrameProcessor(wp<CameraDeviceBase> device,
32                                sp<Camera2Client> client) :
33     FrameProcessorBase(device),
34     mClient(client),
35     mLastFrameNumberOfFaces(0),
36     mLast3AFrameNumber(-1) {
37 
38     sp<CameraDeviceBase> d = device.promote();
39     mSynthesize3ANotify = !(d->willNotify3A());
40 
41     {
42         SharedParameters::Lock l(client->getParameters());
43 
44         if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) {
45             mUsePartialResult = (mNumPartialResults > 1);
46         } else {
47             mUsePartialResult = l.mParameters.quirks.partialResults;
48         }
49 
50         // Initialize starting 3A state
51         m3aState.afTriggerId = l.mParameters.afTriggerCounter;
52         m3aState.aeTriggerId = l.mParameters.precaptureTriggerCounter;
53         // Check if lens is fixed-focus
54         if (l.mParameters.focusMode == Parameters::FOCUS_MODE_FIXED) {
55             m3aState.afMode = ANDROID_CONTROL_AF_MODE_OFF;
56         }
57     }
58 }
59 
~FrameProcessor()60 FrameProcessor::~FrameProcessor() {
61 }
62 
processSingleFrame(CaptureResult & frame,const sp<CameraDeviceBase> & device)63 bool FrameProcessor::processSingleFrame(CaptureResult &frame,
64                                         const sp<CameraDeviceBase> &device) {
65 
66     sp<Camera2Client> client = mClient.promote();
67     if (!client.get()) {
68         return false;
69     }
70 
71     bool isPartialResult = false;
72     if (mUsePartialResult) {
73         if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) {
74             isPartialResult = frame.mResultExtras.partialResultCount < mNumPartialResults;
75         } else {
76             camera_metadata_entry_t entry;
77             entry = frame.mMetadata.find(ANDROID_QUIRKS_PARTIAL_RESULT);
78             if (entry.count > 0 &&
79                     entry.data.u8[0] == ANDROID_QUIRKS_PARTIAL_RESULT_PARTIAL) {
80                 isPartialResult = true;
81             }
82         }
83     }
84 
85     if (!isPartialResult && processFaceDetect(frame.mMetadata, client) != OK) {
86         return false;
87     }
88 
89     if (mSynthesize3ANotify) {
90         process3aState(frame, client);
91     }
92 
93     return FrameProcessorBase::processSingleFrame(frame, device);
94 }
95 
processFaceDetect(const CameraMetadata & frame,const sp<Camera2Client> & client)96 status_t FrameProcessor::processFaceDetect(const CameraMetadata &frame,
97         const sp<Camera2Client> &client) {
98     status_t res = BAD_VALUE;
99     ATRACE_CALL();
100     camera_metadata_ro_entry_t entry;
101     bool enableFaceDetect;
102 
103     {
104         SharedParameters::Lock l(client->getParameters());
105         enableFaceDetect = l.mParameters.enableFaceDetect;
106     }
107     entry = frame.find(ANDROID_STATISTICS_FACE_DETECT_MODE);
108 
109     // TODO: This should be an error once implementations are compliant
110     if (entry.count == 0) {
111         return OK;
112     }
113 
114     uint8_t faceDetectMode = entry.data.u8[0];
115 
116     camera_frame_metadata metadata;
117     Vector<camera_face_t> faces;
118     metadata.number_of_faces = 0;
119 
120     if (enableFaceDetect &&
121         faceDetectMode != ANDROID_STATISTICS_FACE_DETECT_MODE_OFF) {
122 
123         SharedParameters::Lock l(client->getParameters());
124         entry = frame.find(ANDROID_STATISTICS_FACE_RECTANGLES);
125         if (entry.count == 0) {
126             // No faces this frame
127             /* warning: locks SharedCameraCallbacks */
128             callbackFaceDetection(client, metadata);
129             return OK;
130         }
131         metadata.number_of_faces = entry.count / 4;
132         if (metadata.number_of_faces >
133                 l.mParameters.fastInfo.maxFaces) {
134             ALOGE("%s: Camera %d: More faces than expected! (Got %d, max %d)",
135                     __FUNCTION__, client->getCameraId(),
136                     metadata.number_of_faces, l.mParameters.fastInfo.maxFaces);
137             return res;
138         }
139         const int32_t *faceRects = entry.data.i32;
140 
141         entry = frame.find(ANDROID_STATISTICS_FACE_SCORES);
142         if (entry.count == 0) {
143             ALOGE("%s: Camera %d: Unable to read face scores",
144                     __FUNCTION__, client->getCameraId());
145             return res;
146         }
147         const uint8_t *faceScores = entry.data.u8;
148 
149         const int32_t *faceLandmarks = NULL;
150         const int32_t *faceIds = NULL;
151 
152         if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
153             entry = frame.find(ANDROID_STATISTICS_FACE_LANDMARKS);
154             if (entry.count == 0) {
155                 ALOGE("%s: Camera %d: Unable to read face landmarks",
156                         __FUNCTION__, client->getCameraId());
157                 return res;
158             }
159             faceLandmarks = entry.data.i32;
160 
161             entry = frame.find(ANDROID_STATISTICS_FACE_IDS);
162 
163             if (entry.count == 0) {
164                 ALOGE("%s: Camera %d: Unable to read face IDs",
165                         __FUNCTION__, client->getCameraId());
166                 return res;
167             }
168             faceIds = entry.data.i32;
169         }
170 
171         entry = frame.find(ANDROID_SCALER_CROP_REGION);
172         if (entry.count < 4) {
173             ALOGE("%s: Camera %d: Unable to read crop region (count = %d)",
174                     __FUNCTION__, client->getCameraId(), entry.count);
175             return res;
176         }
177 
178         Parameters::CropRegion scalerCrop = {
179             static_cast<float>(entry.data.i32[0]),
180             static_cast<float>(entry.data.i32[1]),
181             static_cast<float>(entry.data.i32[2]),
182             static_cast<float>(entry.data.i32[3])};
183 
184         faces.setCapacity(metadata.number_of_faces);
185 
186         size_t maxFaces = metadata.number_of_faces;
187         for (size_t i = 0; i < maxFaces; i++) {
188             if (faceScores[i] == 0) {
189                 metadata.number_of_faces--;
190                 continue;
191             }
192             if (faceScores[i] > 100) {
193                 ALOGW("%s: Face index %zu with out of range score %d",
194                         __FUNCTION__, i, faceScores[i]);
195             }
196 
197             camera_face_t face;
198 
199             face.rect[0] = l.mParameters.arrayXToNormalizedWithCrop(
200                                 faceRects[i*4 + 0], scalerCrop);
201             face.rect[1] = l.mParameters.arrayYToNormalizedWithCrop(
202                                 faceRects[i*4 + 1], scalerCrop);
203             face.rect[2] = l.mParameters.arrayXToNormalizedWithCrop(
204                                 faceRects[i*4 + 2], scalerCrop);
205             face.rect[3] = l.mParameters.arrayYToNormalizedWithCrop(
206                                 faceRects[i*4 + 3], scalerCrop);
207 
208             face.score = faceScores[i];
209             if (faceDetectMode == ANDROID_STATISTICS_FACE_DETECT_MODE_FULL) {
210                 face.id = faceIds[i];
211                 face.left_eye[0] = l.mParameters.arrayXToNormalizedWithCrop(
212                         faceLandmarks[i*6 + 0], scalerCrop);
213                 face.left_eye[1] = l.mParameters.arrayYToNormalizedWithCrop(
214                         faceLandmarks[i*6 + 1], scalerCrop);
215                 face.right_eye[0] = l.mParameters.arrayXToNormalizedWithCrop(
216                         faceLandmarks[i*6 + 2], scalerCrop);
217                 face.right_eye[1] = l.mParameters.arrayYToNormalizedWithCrop(
218                         faceLandmarks[i*6 + 3], scalerCrop);
219                 face.mouth[0] = l.mParameters.arrayXToNormalizedWithCrop(
220                         faceLandmarks[i*6 + 4], scalerCrop);
221                 face.mouth[1] = l.mParameters.arrayYToNormalizedWithCrop(
222                         faceLandmarks[i*6 + 5], scalerCrop);
223             } else {
224                 face.id = 0;
225                 face.left_eye[0] = face.left_eye[1] = -2000;
226                 face.right_eye[0] = face.right_eye[1] = -2000;
227                 face.mouth[0] = face.mouth[1] = -2000;
228             }
229             faces.push_back(face);
230         }
231 
232         metadata.faces = faces.editArray();
233     }
234 
235     /* warning: locks SharedCameraCallbacks */
236     callbackFaceDetection(client, metadata);
237 
238     return OK;
239 }
240 
process3aState(const CaptureResult & frame,const sp<Camera2Client> & client)241 status_t FrameProcessor::process3aState(const CaptureResult &frame,
242         const sp<Camera2Client> &client) {
243 
244     ATRACE_CALL();
245     const CameraMetadata &metadata = frame.mMetadata;
246     camera_metadata_ro_entry_t entry;
247     int cameraId = client->getCameraId();
248 
249     entry = metadata.find(ANDROID_REQUEST_FRAME_COUNT);
250     int32_t frameNumber = entry.data.i32[0];
251 
252     // Don't send 3A notifications for the same frame number twice
253     if (frameNumber <= mLast3AFrameNumber) {
254         ALOGV("%s: Already sent 3A for frame number %d, skipping",
255                 __FUNCTION__, frameNumber);
256         return OK;
257     }
258 
259     mLast3AFrameNumber = frameNumber;
260 
261     // Get 3A states from result metadata
262     bool gotAllStates = true;
263 
264     AlgState new3aState;
265 
266     // TODO: Also use AE mode, AE trigger ID
267 
268     gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AF_MODE,
269             &new3aState.afMode, frameNumber, cameraId);
270 
271     gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AWB_MODE,
272             &new3aState.awbMode, frameNumber, cameraId);
273 
274     gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AE_STATE,
275             &new3aState.aeState, frameNumber, cameraId);
276 
277     gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AF_STATE,
278             &new3aState.afState, frameNumber, cameraId);
279 
280     gotAllStates &= get3aResult<uint8_t>(metadata, ANDROID_CONTROL_AWB_STATE,
281             &new3aState.awbState, frameNumber, cameraId);
282 
283     if (client->getCameraDeviceVersion() >= CAMERA_DEVICE_API_VERSION_3_2) {
284         new3aState.afTriggerId = frame.mResultExtras.afTriggerId;
285         new3aState.aeTriggerId = frame.mResultExtras.precaptureTriggerId;
286     } else {
287         gotAllStates &= get3aResult<int32_t>(metadata, ANDROID_CONTROL_AF_TRIGGER_ID,
288                  &new3aState.afTriggerId, frameNumber, cameraId);
289 
290         gotAllStates &= get3aResult<int32_t>(metadata, ANDROID_CONTROL_AE_PRECAPTURE_ID,
291                  &new3aState.aeTriggerId, frameNumber, cameraId);
292     }
293 
294     if (!gotAllStates) return BAD_VALUE;
295 
296     if (new3aState.aeState != m3aState.aeState) {
297         ALOGV("%s: Camera %d: AE state %d->%d",
298                 __FUNCTION__, cameraId,
299                 m3aState.aeState, new3aState.aeState);
300         client->notifyAutoExposure(new3aState.aeState, new3aState.aeTriggerId);
301     }
302 
303     if (new3aState.afState != m3aState.afState ||
304         new3aState.afMode != m3aState.afMode ||
305         new3aState.afTriggerId != m3aState.afTriggerId) {
306         ALOGV("%s: Camera %d: AF state %d->%d. AF mode %d->%d. Trigger %d->%d",
307                 __FUNCTION__, cameraId,
308                 m3aState.afState, new3aState.afState,
309                 m3aState.afMode, new3aState.afMode,
310                 m3aState.afTriggerId, new3aState.afTriggerId);
311         client->notifyAutoFocus(new3aState.afState, new3aState.afTriggerId);
312     }
313     if (new3aState.awbState != m3aState.awbState ||
314         new3aState.awbMode != m3aState.awbMode) {
315         ALOGV("%s: Camera %d: AWB state %d->%d. AWB mode %d->%d",
316                 __FUNCTION__, cameraId,
317                 m3aState.awbState, new3aState.awbState,
318                 m3aState.awbMode, new3aState.awbMode);
319         client->notifyAutoWhitebalance(new3aState.awbState,
320                 new3aState.aeTriggerId);
321     }
322 
323     m3aState = new3aState;
324 
325     return OK;
326 }
327 
328 template<typename Src, typename T>
get3aResult(const CameraMetadata & result,int32_t tag,T * value,int32_t frameNumber,int cameraId)329 bool FrameProcessor::get3aResult(const CameraMetadata& result, int32_t tag,
330         T* value, int32_t frameNumber, int cameraId) {
331     camera_metadata_ro_entry_t entry;
332     if (value == NULL) {
333         ALOGE("%s: Camera %d: Value to write to is NULL",
334                 __FUNCTION__, cameraId);
335         return false;
336     }
337 
338     entry = result.find(tag);
339     if (entry.count == 0) {
340         ALOGE("%s: Camera %d: No %s provided by HAL for frame %d!",
341                 __FUNCTION__, cameraId,
342                 get_camera_metadata_tag_name(tag), frameNumber);
343         return false;
344     } else {
345         switch(sizeof(Src)){
346             case sizeof(uint8_t):
347                 *value = static_cast<T>(entry.data.u8[0]);
348                 break;
349             case sizeof(int32_t):
350                 *value = static_cast<T>(entry.data.i32[0]);
351                 break;
352             default:
353                 ALOGE("%s: Camera %d: Unsupported source",
354                         __FUNCTION__, cameraId);
355                 return false;
356         }
357     }
358     return true;
359 }
360 
361 
callbackFaceDetection(sp<Camera2Client> client,const camera_frame_metadata & metadata)362 void FrameProcessor::callbackFaceDetection(sp<Camera2Client> client,
363                                      const camera_frame_metadata &metadata) {
364 
365     camera_frame_metadata *metadata_ptr =
366         const_cast<camera_frame_metadata*>(&metadata);
367 
368     /**
369      * Filter out repeated 0-face callbacks,
370      * but not when the last frame was >0
371      */
372     if (metadata.number_of_faces != 0 ||
373         mLastFrameNumberOfFaces != metadata.number_of_faces) {
374 
375         Camera2Client::SharedCameraCallbacks::Lock
376             l(client->mSharedCameraCallbacks);
377         if (l.mRemoteCallback != NULL) {
378             l.mRemoteCallback->dataCallback(CAMERA_MSG_PREVIEW_METADATA,
379                                             NULL,
380                                             metadata_ptr);
381         }
382     }
383 
384     mLastFrameNumberOfFaces = metadata.number_of_faces;
385 }
386 
387 }; // namespace camera2
388 }; // namespace android
389