1# Copyright 2014 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies 3 faces with different skin tones are detected."""
15
16
17import logging
18import os.path
19
20import cv2
21from mobly import test_runner
22
23import its_base_test
24import camera_properties_utils
25import capture_request_utils
26import image_processing_utils
27import its_session_utils
28import opencv_processing_utils
29
30_CV2_FACE_SCALE_FACTOR = 1.05  # 5% step for resizing image to find face
31_CV2_FACE_MIN_NEIGHBORS = 4  # recommended 3-6: higher for less faces
32_CV2_GREEN = (0, 1, 0)
33_FD_MODE_OFF, _FD_MODE_SIMPLE, _FD_MODE_FULL = 0, 1, 2
34_NAME = os.path.splitext(os.path.basename(__file__))[0]
35_NUM_FACES = 3
36_NUM_TEST_FRAMES = 20
37_TEST_REQUIRED_MPC = 34
38_W, _H = 640, 480
39
40
41def check_face_bounding_box(rect, aw, ah, index):
42  """Checks face bounding box is within the active array area.
43
44  Args:
45    rect: dict; with face bounding box information
46    aw: int; active array width
47    ah: int; active array height
48    index: int to designate face number
49  """
50  logging.debug('Checking bounding box in face %d: %s', index, str(rect))
51  if (rect['top'] >= rect['bottom'] or
52      rect['left'] >= rect['right']):
53    raise AssertionError('Face coordinates incorrect! '
54                         f" t: {rect['top']}, b: {rect['bottom']}, "
55                         f" l: {rect['left']}, r: {rect['right']}")
56  if (not 0 <= rect['top'] <= ah or
57      not 0 <= rect['bottom'] <= ah):
58    raise AssertionError('Face top/bottom outside of image height! '
59                         f"t: {rect['top']}, b: {rect['bottom']}, "
60                         f"h: {ah}")
61  if (not 0 <= rect['left'] <= aw or
62      not 0 <= rect['right'] <= aw):
63    raise AssertionError('Face left/right outside of image width! '
64                         f"l: {rect['left']}, r: {rect['right']}, "
65                         f" w: {aw}")
66
67
68def check_face_landmarks(face, fd_mode, index):
69  """Checks face landmarks fall within face bounding box.
70
71  Face ID should be -1 for SIMPLE and unique for FULL
72  Args:
73    face: dict from face detection algorithm
74    fd_mode: int of face detection mode
75    index: int to designate face number
76  """
77  logging.debug('Checking landmarks in face %d: %s', index, str(face))
78  if fd_mode == _FD_MODE_SIMPLE:
79    if 'leftEye' in face or 'rightEye' in face:
80      raise AssertionError('Eyes not supported in FD_MODE_SIMPLE.')
81    if 'mouth' in face:
82      raise AssertionError('Mouth not supported in FD_MODE_SIMPLE.')
83    if face['id'] != -1:
84      raise AssertionError('face_id should be -1 in FD_MODE_SIMPLE.')
85  elif fd_mode == _FD_MODE_FULL:
86    l, r = face['bounds']['left'], face['bounds']['right']
87    t, b = face['bounds']['top'], face['bounds']['bottom']
88    l_eye_x, l_eye_y = face['leftEye']['x'], face['leftEye']['y']
89    r_eye_x, r_eye_y = face['rightEye']['x'], face['rightEye']['y']
90    mouth_x, mouth_y = face['mouth']['x'], face['mouth']['y']
91    if not l <= l_eye_x <= r:
92      raise AssertionError(f'Face l: {l}, r: {r}, left eye x: {l_eye_x}')
93    if not t <= l_eye_y <= b:
94      raise AssertionError(f'Face t: {t}, b: {b}, left eye y: {l_eye_y}')
95    if not l <= r_eye_x <= r:
96      raise AssertionError(f'Face l: {l}, r: {r}, right eye x: {r_eye_x}')
97    if not t <= r_eye_y <= b:
98      raise AssertionError(f'Face t: {t}, b: {b}, right eye y: {r_eye_y}')
99    if not l <= mouth_x <= r:
100      raise AssertionError(f'Face l: {l}, r: {r}, mouth x: {mouth_x}')
101    if not t <= mouth_y <= b:
102      raise AssertionError(f'Face t: {t}, b: {b}, mouth y: {mouth_y}')
103  else:
104    raise AssertionError(f'Unknown face detection mode: {fd_mode}.')
105
106
107class NumFacesTest(its_base_test.ItsBaseTest):
108  """Test face detection with different skin tones.
109  """
110
111  def test_num_faces(self):
112    """Test face detection."""
113    with its_session_utils.ItsSession(
114        device_id=self.dut.serial,
115        camera_id=self.camera_id,
116        hidden_physical_id=self.hidden_physical_id) as cam:
117      props = cam.get_camera_properties()
118      props = cam.override_with_hidden_physical_camera_props(props)
119
120      # Load chart for scene
121      its_session_utils.load_scene(
122          cam, props, self.scene, self.tablet, self.chart_distance,
123          log_path=self.log_path)
124
125      # Check media performance class
126      should_run = camera_properties_utils.face_detect(props)
127      media_performance_class = its_session_utils.get_media_performance_class(
128          self.dut.serial)
129      if media_performance_class >= _TEST_REQUIRED_MPC and not should_run:
130        its_session_utils.raise_mpc_assertion_error(
131            _TEST_REQUIRED_MPC, _NAME, media_performance_class)
132
133      # Check skip conditions
134      camera_properties_utils.skip_unless(should_run)
135      mono_camera = camera_properties_utils.mono_camera(props)
136      fd_modes = props['android.statistics.info.availableFaceDetectModes']
137      a = props['android.sensor.info.activeArraySize']
138      aw, ah = a['right'] - a['left'], a['bottom'] - a['top']
139      logging.debug('active array size: %s', str(a))
140      file_name_stem = os.path.join(self.log_path, _NAME)
141
142      cam.do_3a(mono_camera=mono_camera)
143
144      for fd_mode in fd_modes:
145        logging.debug('face detection mode: %d', fd_mode)
146        if not _FD_MODE_OFF <= fd_mode <= _FD_MODE_FULL:
147          raise AssertionError(f'FD mode {fd_mode} not in MODES! '
148                               f'OFF: {_FD_MODE_OFF}, FULL: {_FD_MODE_FULL}')
149        req = capture_request_utils.auto_capture_request()
150        req['android.statistics.faceDetectMode'] = fd_mode
151        fmt = {'format': 'yuv', 'width': _W, 'height': _H}
152        caps = cam.do_capture([req]*_NUM_TEST_FRAMES, fmt)
153        for i, cap in enumerate(caps):
154          fd_mode_cap = cap['metadata']['android.statistics.faceDetectMode']
155          if fd_mode_cap != fd_mode:
156            raise AssertionError(f'metadata {fd_mode_cap} != req {fd_mode}')
157
158          faces = cap['metadata']['android.statistics.faces']
159          # 0 faces should be returned for OFF mode
160          if fd_mode == _FD_MODE_OFF:
161            if faces:
162              raise AssertionError(f'Error: faces detected in OFF: {faces}')
163            continue
164          # Face detection could take several frames to warm up,
165          # but should detect the correct number of faces in last frame
166          if i == _NUM_TEST_FRAMES - 1:
167            img = image_processing_utils.convert_capture_to_rgb_image(
168                cap, props=props)
169            fnd_faces = len(faces)
170            logging.debug('Found %d face(s), expected %d.',
171                          fnd_faces, _NUM_FACES)
172
173            # draw boxes around faces in green
174            crop_region = cap['metadata']['android.scaler.cropRegion']
175            faces_cropped = opencv_processing_utils.correct_faces_for_crop(
176                faces, img, crop_region)
177            for (l, r, t, b) in faces_cropped:
178              cv2.rectangle(img, (l, t), (r, b), _CV2_GREEN, 2)
179
180            # Save image with green rectangles
181            img_name = f'{file_name_stem}_fd_mode_{fd_mode}.jpg'
182            image_processing_utils.write_image(img, img_name)
183            if fnd_faces != _NUM_FACES:
184              raise AssertionError('Wrong num of faces found! Found: '
185                                   f'{fnd_faces}, expected: {_NUM_FACES}')
186            # Reasonable scores for faces
187            face_scores = [face['score'] for face in faces]
188            for score in face_scores:
189              if not 1 <= score <= 100:
190                raise AssertionError(f'score not between [1:100]! {score}')
191
192            # Face bounds should be within active array
193            face_rectangles = [face['bounds'] for face in faces]
194            for j, rect in enumerate(face_rectangles):
195              check_face_bounding_box(rect, aw, ah, j)
196
197            # Face landmarks (if provided) are within face bounding box
198            if (its_session_utils.get_first_api_level(self.dut.serial) >=
199                its_session_utils.ANDROID14_API_LEVEL):
200              for k, face in enumerate(faces):
201                check_face_landmarks(face, fd_mode, k)
202
203            # Match location of opencv and face detection mode faces
204            if self.scene == 'scene2_d':
205              logging.debug('Match face centers between opencv & faces')
206              faces_opencv = opencv_processing_utils.find_opencv_faces(
207                  img, _CV2_FACE_SCALE_FACTOR, _CV2_FACE_MIN_NEIGHBORS)
208              if fd_mode:  # non-zero value for ON
209                opencv_processing_utils.match_face_locations(
210                    faces_cropped, faces_opencv, img, img_name)
211
212          if not faces:
213            continue
214          logging.debug('Frame %d face metadata:', i)
215          logging.debug(' Faces: %s', str(faces))
216
217
218if __name__ == '__main__':
219  test_runner.main()
220