1 /*
2 * Copyright (C) Texas Instruments - http://www.ti.com/
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 /**
18 * @file OMXFD.cpp
19 *
20 * This file contains functionality for handling face detection.
21 *
22 */
23
24 #undef LOG_TAG
25
26 #define LOG_TAG "CameraHAL"
27
28 #include "CameraHal.h"
29 #include "OMXCameraAdapter.h"
30
31 #define FACE_DETECTION_THRESHOLD 80
32
33 // constants used for face smooth filtering
34 static const int HorizontalFilterThreshold = 40;
35 static const int VerticalFilterThreshold = 40;
36 static const int HorizontalFaceSizeThreshold = 30;
37 static const int VerticalFaceSizeThreshold = 30;
38
39
40 namespace android {
41
setParametersFD(const CameraParameters & params,BaseCameraAdapter::AdapterState state)42 status_t OMXCameraAdapter::setParametersFD(const CameraParameters ¶ms,
43 BaseCameraAdapter::AdapterState state)
44 {
45 status_t ret = NO_ERROR;
46
47 LOG_FUNCTION_NAME;
48
49 LOG_FUNCTION_NAME_EXIT;
50
51 return ret;
52 }
53
startFaceDetection()54 status_t OMXCameraAdapter::startFaceDetection()
55 {
56 status_t ret = NO_ERROR;
57
58 Mutex::Autolock lock(mFaceDetectionLock);
59
60 ret = setFaceDetection(true, mDeviceOrientation);
61 if (ret != NO_ERROR) {
62 goto out;
63 }
64
65 if ( mFaceDetectionRunning ) {
66 mFDSwitchAlgoPriority = true;
67 }
68
69 // Note: White balance will not be face prioritized, since
70 // the algorithm needs full frame statistics, and not face
71 // regions alone.
72
73 faceDetectionNumFacesLastOutput = 0;
74 out:
75 return ret;
76 }
77
stopFaceDetection()78 status_t OMXCameraAdapter::stopFaceDetection()
79 {
80 status_t ret = NO_ERROR;
81 const char *str = NULL;
82 BaseCameraAdapter::AdapterState state;
83 BaseCameraAdapter::getState(state);
84
85 Mutex::Autolock lock(mFaceDetectionLock);
86
87 ret = setFaceDetection(false, mDeviceOrientation);
88 if (ret != NO_ERROR) {
89 goto out;
90 }
91
92 // Reset 3A settings
93 ret = setParameters3A(mParams, state);
94 if (ret != NO_ERROR) {
95 goto out;
96 }
97
98 if (mPending3Asettings) {
99 apply3Asettings(mParameters3A);
100 }
101
102 faceDetectionNumFacesLastOutput = 0;
103 out:
104 return ret;
105 }
106
pauseFaceDetection(bool pause)107 void OMXCameraAdapter::pauseFaceDetection(bool pause)
108 {
109 Mutex::Autolock lock(mFaceDetectionLock);
110 // pausing will only take affect if fd is already running
111 if (mFaceDetectionRunning) {
112 mFaceDetectionPaused = pause;
113 faceDetectionNumFacesLastOutput = 0;
114 }
115 }
116
setFaceDetection(bool enable,OMX_U32 orientation)117 status_t OMXCameraAdapter::setFaceDetection(bool enable, OMX_U32 orientation)
118 {
119 status_t ret = NO_ERROR;
120 OMX_ERRORTYPE eError = OMX_ErrorNone;
121 OMX_CONFIG_OBJDETECTIONTYPE objDetection;
122
123 LOG_FUNCTION_NAME;
124
125 if ( OMX_StateInvalid == mComponentState )
126 {
127 CAMHAL_LOGEA("OMX component is in invalid state");
128 ret = -EINVAL;
129 }
130
131 if ( NO_ERROR == ret )
132 {
133 if ( orientation > 270 ) {
134 orientation = 0;
135 }
136
137 OMX_INIT_STRUCT_PTR (&objDetection, OMX_CONFIG_OBJDETECTIONTYPE);
138 objDetection.nPortIndex = mCameraAdapterParameters.mPrevPortIndex;
139 objDetection.nDeviceOrientation = orientation;
140 if ( enable )
141 {
142 objDetection.bEnable = OMX_TRUE;
143 }
144 else
145 {
146 objDetection.bEnable = OMX_FALSE;
147 }
148
149 eError = OMX_SetConfig(mCameraAdapterParameters.mHandleComp,
150 ( OMX_INDEXTYPE ) OMX_IndexConfigImageFaceDetection,
151 &objDetection);
152 if ( OMX_ErrorNone != eError )
153 {
154 CAMHAL_LOGEB("Error while configuring face detection 0x%x", eError);
155 ret = -1;
156 }
157 else
158 {
159 CAMHAL_LOGDA("Face detection configured successfully");
160 }
161 }
162
163 if ( NO_ERROR == ret )
164 {
165 ret = setExtraData(enable, mCameraAdapterParameters.mPrevPortIndex, OMX_FaceDetection);
166
167 if ( NO_ERROR != ret )
168 {
169 CAMHAL_LOGEA("Error while configuring face detection extra data");
170 }
171 else
172 {
173 CAMHAL_LOGDA("Face detection extra data configured successfully");
174 }
175 }
176
177 if ( NO_ERROR == ret )
178 {
179 mFaceDetectionRunning = enable;
180 mFaceDetectionPaused = !enable;
181 }
182
183 LOG_FUNCTION_NAME_EXIT;
184
185 return ret;
186 }
187
detectFaces(OMX_BUFFERHEADERTYPE * pBuffHeader,sp<CameraFDResult> & result,size_t previewWidth,size_t previewHeight)188 status_t OMXCameraAdapter::detectFaces(OMX_BUFFERHEADERTYPE* pBuffHeader,
189 sp<CameraFDResult> &result,
190 size_t previewWidth,
191 size_t previewHeight)
192 {
193 status_t ret = NO_ERROR;
194 OMX_ERRORTYPE eError = OMX_ErrorNone;
195 OMX_TI_FACERESULT *faceResult;
196 OMX_OTHER_EXTRADATATYPE *extraData;
197 OMX_FACEDETECTIONTYPE *faceData;
198 OMX_TI_PLATFORMPRIVATE *platformPrivate;
199 camera_frame_metadata_t *faces;
200
201 LOG_FUNCTION_NAME;
202
203 if ( OMX_StateExecuting != mComponentState ) {
204 CAMHAL_LOGEA("OMX component is not in executing state");
205 return NO_INIT;
206 }
207
208 if ( NULL == pBuffHeader ) {
209 CAMHAL_LOGEA("Invalid Buffer header");
210 return-EINVAL;
211 }
212
213 platformPrivate = (OMX_TI_PLATFORMPRIVATE *) (pBuffHeader->pPlatformPrivate);
214 if ( NULL != platformPrivate ) {
215 if ( sizeof(OMX_TI_PLATFORMPRIVATE) == platformPrivate->nSize ) {
216 CAMHAL_LOGVB("Size = %d, sizeof = %d, pAuxBuf = 0x%x, pAuxBufSize= %d, pMetaDataBufer = 0x%x, nMetaDataSize = %d",
217 platformPrivate->nSize,
218 sizeof(OMX_TI_PLATFORMPRIVATE),
219 platformPrivate->pAuxBuf1,
220 platformPrivate->pAuxBufSize1,
221 platformPrivate->pMetaDataBuffer,
222 platformPrivate->nMetaDataSize);
223 } else {
224 CAMHAL_LOGDB("OMX_TI_PLATFORMPRIVATE size mismatch: expected = %d, received = %d",
225 ( unsigned int ) sizeof(OMX_TI_PLATFORMPRIVATE),
226 ( unsigned int ) platformPrivate->nSize);
227 return -EINVAL;
228 }
229 } else {
230 CAMHAL_LOGDA("Invalid OMX_TI_PLATFORMPRIVATE");
231 return-EINVAL;
232 }
233
234
235 if ( 0 >= platformPrivate->nMetaDataSize ) {
236 CAMHAL_LOGDB("OMX_TI_PLATFORMPRIVATE nMetaDataSize is size is %d",
237 ( unsigned int ) platformPrivate->nMetaDataSize);
238 return -EINVAL;
239 }
240
241 extraData = getExtradata((OMX_OTHER_EXTRADATATYPE *) (platformPrivate->pMetaDataBuffer),
242 (OMX_EXTRADATATYPE)OMX_FaceDetection);
243
244 if ( NULL != extraData ) {
245 CAMHAL_LOGVB("Size = %d, sizeof = %d, eType = 0x%x, nDataSize= %d, nPortIndex = 0x%x, nVersion = 0x%x",
246 extraData->nSize,
247 sizeof(OMX_OTHER_EXTRADATATYPE),
248 extraData->eType,
249 extraData->nDataSize,
250 extraData->nPortIndex,
251 extraData->nVersion);
252 } else {
253 CAMHAL_LOGDA("Invalid OMX_OTHER_EXTRADATATYPE");
254 return -EINVAL;
255 }
256
257 faceData = ( OMX_FACEDETECTIONTYPE * ) extraData->data;
258 if ( NULL != faceData ) {
259 if ( sizeof(OMX_FACEDETECTIONTYPE) == faceData->nSize ) {
260 CAMHAL_LOGVB("Faces detected %d",
261 faceData->ulFaceCount,
262 faceData->nSize,
263 sizeof(OMX_FACEDETECTIONTYPE),
264 faceData->eCameraView,
265 faceData->nPortIndex,
266 faceData->nVersion);
267 } else {
268 CAMHAL_LOGDB("OMX_FACEDETECTIONTYPE size mismatch: expected = %d, received = %d",
269 ( unsigned int ) sizeof(OMX_FACEDETECTIONTYPE),
270 ( unsigned int ) faceData->nSize);
271 return -EINVAL;
272 }
273 } else {
274 CAMHAL_LOGEA("Invalid OMX_FACEDETECTIONTYPE");
275 return -EINVAL;
276 }
277
278 ret = encodeFaceCoordinates(faceData, &faces, previewWidth, previewHeight);
279
280 if ( NO_ERROR == ret ) {
281 result = new CameraFDResult(faces);
282 } else {
283 result.clear();
284 result = NULL;
285 }
286
287 LOG_FUNCTION_NAME_EXIT;
288
289 return ret;
290 }
291
encodeFaceCoordinates(const OMX_FACEDETECTIONTYPE * faceData,camera_frame_metadata_t ** pFaces,size_t previewWidth,size_t previewHeight)292 status_t OMXCameraAdapter::encodeFaceCoordinates(const OMX_FACEDETECTIONTYPE *faceData,
293 camera_frame_metadata_t **pFaces,
294 size_t previewWidth,
295 size_t previewHeight)
296 {
297 status_t ret = NO_ERROR;
298 camera_face_t *faces;
299 camera_frame_metadata_t *faceResult;
300 size_t hRange, vRange;
301 double tmp;
302
303 LOG_FUNCTION_NAME;
304
305 if ( NULL == faceData ) {
306 CAMHAL_LOGEA("Invalid OMX_FACEDETECTIONTYPE parameter");
307 return EINVAL;
308 }
309
310 LOG_FUNCTION_NAME
311
312 hRange = CameraFDResult::RIGHT - CameraFDResult::LEFT;
313 vRange = CameraFDResult::BOTTOM - CameraFDResult::TOP;
314
315 faceResult = ( camera_frame_metadata_t * ) malloc(sizeof(camera_frame_metadata_t));
316 if ( NULL == faceResult ) {
317 return -ENOMEM;
318 }
319
320 if ( 0 < faceData->ulFaceCount ) {
321 int orient_mult;
322 int trans_left, trans_top, trans_right, trans_bot;
323
324 faces = ( camera_face_t * ) malloc(sizeof(camera_face_t)*faceData->ulFaceCount);
325 if ( NULL == faces ) {
326 return -ENOMEM;
327 }
328
329 /**
330 / * When device is 180 degrees oriented to the sensor, need to translate
331 / * the output from Ducati to what Android expects
332 / * Ducati always gives face coordinates in this form, irrespective of
333 / * rotation, i.e (l,t) always represents the point towards the left eye
334 / * and top of hair.
335 / * (l, t)
336 / * ---------------
337 / * - ,,,,,,, -
338 / * - | | -
339 / * - |<a <a| -
340 / * - (| ^ |) -
341 / * - | -=- | -
342 / * - \_____/ -
343 / * ---------------
344 / * (r, b)
345 / *
346 / * However, Android expects the coords to be in respect with what the
347 / * sensor is viewing, i.e Android expects sensor to see this with (l,t)
348 / * and (r,b) like so:
349 / * (l, t)
350 / * ---------------
351 / * - _____ -
352 / * - / \ -
353 / * - | -=- | -
354 / * - (| ^ |) -
355 / * - |a> a>| -
356 / * - | | -
357 / * - ,,,,,,, -
358 / * ---------------
359 / * (r, b)
360 */
361
362 if (mDeviceOrientation == 180) {
363 orient_mult = -1;
364 trans_left = 2; // right is now left
365 trans_top = 3; // bottom is now top
366 trans_right = 0; // left is now right
367 trans_bot = 1; // top is not bottom
368 } else {
369 orient_mult = 1;
370 trans_left = 0; // left
371 trans_top = 1; // top
372 trans_right = 2; // right
373 trans_bot = 3; // bottom
374 }
375
376 int j = 0, i = 0;
377 for ( ; j < faceData->ulFaceCount ; j++)
378 {
379 OMX_S32 nLeft = 0;
380 OMX_S32 nTop = 0;
381 //Face filtering
382 //For real faces, it is seen that the h/w passes a score >=80
383 //For false faces, we seem to get even a score of 70 sometimes.
384 //In order to avoid any issue at application level, we filter
385 //<=70 score here.
386 if(faceData->tFacePosition[j].nScore <= FACE_DETECTION_THRESHOLD)
387 continue;
388
389 if (mDeviceOrientation == 180) {
390 // from sensor pov, the left pos is the right corner of the face in pov of frame
391 nLeft = faceData->tFacePosition[j].nLeft + faceData->tFacePosition[j].nWidth;
392 nTop = faceData->tFacePosition[j].nTop + faceData->tFacePosition[j].nHeight;
393 } else {
394 nLeft = faceData->tFacePosition[j].nLeft;
395 nTop = faceData->tFacePosition[j].nTop;
396 }
397
398 tmp = ( double ) nLeft / ( double ) previewWidth;
399 tmp *= hRange;
400 tmp -= hRange/2;
401 faces[i].rect[trans_left] = tmp;
402
403 tmp = ( double ) nTop / ( double )previewHeight;
404 tmp *= vRange;
405 tmp -= vRange/2;
406 faces[i].rect[trans_top] = tmp;
407
408 tmp = ( double ) faceData->tFacePosition[j].nWidth / ( double ) previewWidth;
409 tmp *= hRange;
410 tmp *= orient_mult;
411 faces[i].rect[trans_right] = faces[i].rect[trans_left] + tmp;
412
413 tmp = ( double ) faceData->tFacePosition[j].nHeight / ( double ) previewHeight;
414 tmp *= vRange;
415 tmp *= orient_mult;
416 faces[i].rect[trans_bot] = faces[i].rect[trans_top] + tmp;
417
418 faces[i].score = faceData->tFacePosition[j].nScore;
419 faces[i].id = 0;
420 faces[i].left_eye[0] = CameraFDResult::INVALID_DATA;
421 faces[i].left_eye[1] = CameraFDResult::INVALID_DATA;
422 faces[i].right_eye[0] = CameraFDResult::INVALID_DATA;
423 faces[i].right_eye[1] = CameraFDResult::INVALID_DATA;
424 faces[i].mouth[0] = CameraFDResult::INVALID_DATA;
425 faces[i].mouth[1] = CameraFDResult::INVALID_DATA;
426 i++;
427 }
428
429 faceResult->number_of_faces = i;
430 faceResult->faces = faces;
431
432 for (int i = 0; i < faceResult->number_of_faces; i++)
433 {
434 int centerX = (faces[i].rect[trans_left] + faces[i].rect[trans_right] ) / 2;
435 int centerY = (faces[i].rect[trans_top] + faces[i].rect[trans_bot] ) / 2;
436
437 int sizeX = (faces[i].rect[trans_right] - faces[i].rect[trans_left] ) ;
438 int sizeY = (faces[i].rect[trans_bot] - faces[i].rect[trans_top] ) ;
439
440 for (int j = 0; j < faceDetectionNumFacesLastOutput; j++)
441 {
442 int tempCenterX = (faceDetectionLastOutput[j].rect[trans_left] +
443 faceDetectionLastOutput[j].rect[trans_right] ) / 2;
444 int tempCenterY = (faceDetectionLastOutput[j].rect[trans_top] +
445 faceDetectionLastOutput[j].rect[trans_bot] ) / 2;
446 int tempSizeX = (faceDetectionLastOutput[j].rect[trans_right] -
447 faceDetectionLastOutput[j].rect[trans_left] ) ;
448 int tempSizeY = (faceDetectionLastOutput[j].rect[trans_bot] -
449 faceDetectionLastOutput[j].rect[trans_top] ) ;
450
451 if ( (abs(tempCenterX - centerX) < HorizontalFilterThreshold) &&
452 (abs(tempCenterY - centerY) < VerticalFilterThreshold) )
453 {
454 // Found Face. It did not move too far.
455 // Now check size of rectangle compare to last output
456 if ( (abs (tempSizeX -sizeX) < HorizontalFaceSizeThreshold) &&
457 (abs (tempSizeY -sizeY) < VerticalFaceSizeThreshold) )
458 {
459 // Rectangle is almost same as last time
460 // Output exactly what was done for this face last time.
461 faces[i] = faceDetectionLastOutput[j];
462 }
463 else
464 {
465 // TODO(XXX): Rectangle size changed but position is same.
466 // Possibly we can apply just positional correctness.
467 }
468 }
469 }
470 }
471
472 // Save this output for next iteration
473 for (int i = 0; i < faceResult->number_of_faces; i++)
474 {
475 faceDetectionLastOutput[i] = faces[i];
476 }
477 faceDetectionNumFacesLastOutput = faceResult->number_of_faces;
478 } else {
479 faceResult->number_of_faces = 0;
480 faceResult->faces = NULL;
481 }
482
483 *pFaces = faceResult;
484
485 LOG_FUNCTION_NAME_EXIT;
486
487 return ret;
488 }
489
490 };
491