1# Copyright 2014 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies faces are detected and landmarks in bounding boxes."""
15
16
17import logging
18import os.path
19from mobly import test_runner
20
21import its_base_test
22import camera_properties_utils
23import capture_request_utils
24import image_processing_utils
25import its_session_utils
26
27NAME = os.path.splitext(os.path.basename(__file__))[0]
28NUM_TEST_FRAMES = 20
29FD_MODE_OFF = 0
30FD_MODE_SIMPLE = 1
31FD_MODE_FULL = 2
32W, H = 640, 480
33
34
35def check_face_bounding_box(rect, aa_w, aa_h):
36  """Check that face bounding box is within the active array area."""
37  rect_t = rect['top']
38  rect_b = rect['bottom']
39  rect_l = rect['left']
40  rect_r = rect['right']
41  if rect_t > rect_b:
42    raise AssertionError(f'Face top > bottom! t: {rect_t}, b: {rect_b}')
43  if rect_l > rect_r:
44    raise AssertionError(f'Face left > right! l: {rect_l}, r: {rect_r}')
45
46  if not 0 <= rect_l <= aa_w:
47    raise AssertionError(f'Face l: {rect_l} outside of active W: 0,{aa_w}')
48  if not 0 <= rect_r <= aa_w:
49    raise AssertionError(f'Face r: {rect_r} outside of active W: 0,{aa_w}')
50  if not 0 <= rect_t <= aa_h:
51    raise AssertionError(f'Face t: {rect_t} outside active H: 0,{aa_h}')
52  if not 0 <= rect_b <= aa_h:
53    raise AssertionError(f'Face b: {rect_b} outside active H: 0,{aa_h}')
54
55
56def check_face_landmarks(face):
57  """Check that face landmarks fall within face bounding box."""
58  l, r = face['bounds']['left'], face['bounds']['right']
59  t, b = face['bounds']['top'], face['bounds']['bottom']
60  l_eye_x, l_eye_y = face['leftEye']['x'], face['leftEye']['y']
61  r_eye_x, r_eye_y = face['rightEye']['x'], face['rightEye']['y']
62  mouth_x, mouth_y = face['mouth']['x'], face['mouth']['y']
63  if not l <= l_eye_x <= r:
64    raise AssertionError(f'Face l: {l}, r: {r}, left eye x: {l_eye_x}')
65  if not t <= l_eye_y <= b:
66    raise AssertionError(f'Face t: {t}, b: {b}, left eye y: {l_eye_y}')
67  if not l <= r_eye_x <= r:
68    raise AssertionError(f'Face l: {l}, r: {r}, right eye x: {r_eye_x}')
69  if not t <= r_eye_y <= b:
70    raise AssertionError(f'Face t: {t}, b: {b}, right eye y: {r_eye_y}')
71  if not l <= mouth_x <= r:
72    raise AssertionError(f'Face l: {l}, r: {r}, mouth x: {mouth_x}')
73  if not t <= mouth_y <= b:
74    raise AssertionError(f'Face t: {t}, b: {b}, mouth y: {mouth_y}')
75
76
77class FacesTest(its_base_test.ItsBaseTest):
78  """Tests face detection algorithms.
79
80  Allows NUM_TEST_FRAMES for face detection algorithm to find all faces.
81  Tests OFF, SIMPLE, and FULL modes if available.
82    OFF --> no faces should be found.
83    SIMPLE --> face(s) should be found, but no landmarks.
84    FULL --> face(s) should be found and face landmarks reported.
85  """
86
87  def test_faces(self):
88    logging.debug('Starting %s', NAME)
89    with its_session_utils.ItsSession(
90        device_id=self.dut.serial,
91        camera_id=self.camera_id,
92        hidden_physical_id=self.hidden_physical_id) as cam:
93      props = cam.get_camera_properties()
94      props = cam.override_with_hidden_physical_camera_props(props)
95
96      # Load chart for scene.
97      its_session_utils.load_scene(
98          cam, props, self.scene, self.tablet, self.chart_distance)
99
100      camera_properties_utils.skip_unless(
101          camera_properties_utils.face_detect(props))
102      mono_camera = camera_properties_utils.mono_camera(props)
103      fd_modes = props['android.statistics.info.availableFaceDetectModes']
104      a = props['android.sensor.info.activeArraySize']
105      aw, ah = a['right'] - a['left'], a['bottom'] - a['top']
106      if camera_properties_utils.read_3a(props):
107        gain, exp, _, _, focus = cam.do_3a(
108            get_results=True, mono_camera=mono_camera)
109        logging.debug('iso = %d', gain)
110        logging.debug('exp = %.2fms', (exp * 1.0E-6))
111        if focus == 0.0:
112          logging.debug('fd = infinity')
113        else:
114          logging.debug('fd = %.2fcm', (1.0E2 / focus))
115      for fd_mode in fd_modes:
116        if not FD_MODE_OFF <= fd_mode <= FD_MODE_FULL:
117          raise AssertionError(f'fd_mode undefined: {fd_mode}')
118        req = capture_request_utils.auto_capture_request()
119        req['android.statistics.faceDetectMode'] = fd_mode
120        fmt = {'format': 'yuv', 'width': W, 'height': H}
121        caps = cam.do_capture([req] * NUM_TEST_FRAMES, fmt)
122        for i, cap in enumerate(caps):
123          fd_mode_md = cap['metadata']['android.statistics.faceDetectMode']
124          if fd_mode_md != fd_mode:
125            raise AssertionError('Metadata does not match request! '
126                                 f'Request: {fd_mode} metadata: {fd_mode_md}.')
127          faces = cap['metadata']['android.statistics.faces']
128
129          # 0 faces should be returned for OFF mode
130          if fd_mode == FD_MODE_OFF:
131            if faces:
132              raise AssertionError('Faces found in OFF mode.')
133            continue
134          # Save last frame.
135          if i == NUM_TEST_FRAMES - 1:
136            img = image_processing_utils.convert_capture_to_rgb_image(
137                cap, props=props)
138            img = image_processing_utils.rotate_img_per_argv(img)
139            img_name = '%s_fd_mode_%s.jpg' % (os.path.join(self.log_path,
140                                                           NAME), fd_mode)
141            image_processing_utils.write_image(img, img_name)
142            if not faces:
143              raise AssertionError(f'No face detected in mode {fd_mode}.')
144          if not faces:
145            continue
146
147          logging.debug('Frame %d face metadata:', i)
148          logging.debug('Faces: %s', faces)
149
150          face_scores = [face['score'] for face in faces]
151          face_rectangles = [face['bounds'] for face in faces]
152          for score in face_scores:
153            if not 1 <= score <= 100:
154              raise AssertionError(f'Face score not valid! score: {score}.')
155          # Face bounds should be within active array.
156          for j, rect in enumerate(face_rectangles):
157            logging.debug('Checking face rectangle %d', j)
158            check_face_bounding_box(rect, aw, ah)
159
160          # Face ID should be -1 for SIMPLE and unique for FULL
161          if fd_mode == FD_MODE_SIMPLE:
162            for face in faces:
163              if 'leftEye' in face or 'rightEye' in face:
164                raise AssertionError('Eyes not supported in FD_MODE_SIMPLE.')
165              if 'mouth' in face:
166                raise AssertionError('Mouth not supported in FD_MODE_SIMPLE.')
167              if face['id'] != -1:
168                raise AssertionError('face_id should be -1 in FD_MODE_SIMPLE.')
169          elif fd_mode == FD_MODE_FULL:
170            face_ids = [face['id'] for face in faces]
171            if len(face_ids) != len(set(face_ids)):
172              raise AssertionError('Same face detected more than 1x.')
173            # Face landmarks should be within face bounds
174            for k, face in enumerate(faces):
175              logging.debug('Checking landmarks in face %d: %s', k, str(face))
176              check_face_landmarks(face)
177
178if __name__ == '__main__':
179  test_runner.main()
180