1# Copyright 2023 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verify HDR is activated correctly for extension captures."""
15
16
17import logging
18import os.path
19import time
20
21import cv2
22from mobly import test_runner
23import numpy as np
24from scipy import ndimage
25
26import its_base_test
27import camera_properties_utils
28import capture_request_utils
29import error_util
30import image_processing_utils
31import its_session_utils
32import lighting_control_utils
33import opencv_processing_utils
34
35_NAME = os.path.splitext(os.path.basename(__file__))[0]
36_EXTENSION_HDR = 3
37_TABLET_BRIGHTNESS = '12'  # Highest minimum brightness on a supported tablet
38
39_FMT_NAME = 'jpg'
40_WIDTH = 1920
41_HEIGHT = 1080
42
43_MIN_QRCODE_AREA = 0.01  # Reject squares smaller than 1% of image
44_QR_CODE_VALUE = 'CameraITS'
45_CONTRAST_ARANGE = (1, 10, 0.01)
46_CONTOUR_INDEX = -1  # Draw all contours as per opencv convention
47_BGR_RED = (0, 0, 255)
48_CONTOUR_LINE_THICKNESS = 3
49
50_DURATION_DIFF_TOL = 0.5  # HDR ON captures must take 0.5 seconds longer
51_GRADIENT_TOL = 0.15  # Largest HDR gradient must be at most 15% of non-HDR
52
53
54def extract_tile(img, file_stem_with_suffix):
55  """Extracts a white square from an image and processes it for analysis.
56
57  Args:
58    img: An RGB image
59    file_stem_with_suffix: Filename describing image format and HDR activation.
60  Returns:
61    openCV image representing the QR code
62  """
63  img *= 255  # openCV needs [0:255] images
64  square = opencv_processing_utils.find_white_square(
65      img, _MIN_QRCODE_AREA)
66  tile = image_processing_utils.get_image_patch(
67      img,
68      square['left']/img.shape[1],
69      square['top']/img.shape[0],
70      square['w']/img.shape[1],
71      square['h']/img.shape[0]
72  )
73  tile = tile.astype(np.uint8)
74  tile = tile[:, :, ::-1]  # RGB --> BGR for cv2
75  tile = cv2.cvtColor(tile, cv2.COLOR_BGR2GRAY)  # Convert to grayscale
76
77  # Rotate tile to reduce scene variation
78  h, w = tile.shape[:2]
79  center_x, center_y = w // 2, h // 2
80  rotation_matrix = cv2.getRotationMatrix2D((center_x, center_y),
81                                            square['angle'], 1.0)
82  tile = cv2.warpAffine(tile, rotation_matrix, (w, h))
83  cv2.imwrite(f'{file_stem_with_suffix}_tile.png', tile)
84  return tile
85
86
87def analyze_qr_code(img, file_stem_with_suffix):
88  """Analyze gradient across ROI and detect/decode its QR code from an image.
89
90  Attempts to detect and decode a QR code from the image represented by img,
91  after converting to grayscale and rotating the code to be in line with
92  the x and y axes. Then, if even detection fails, modifies the contrast of
93  the image until the QR code is detectable. Measures the gradient across
94  the code by finding the length of the largest contour found by openCV.
95
96  Args:
97    img: An RGB image
98    file_stem_with_suffix: Filename describing image format and HDR activation.
99
100  Returns:
101    detection_object: Union[str, bool], describes decoded data or detection
102    lowest_successful_alpha: float, contrast where QR code was detected/decoded
103    contour_length: int, length of largest contour in gradient image
104  """
105  tile = extract_tile(img, file_stem_with_suffix)
106
107  # Find gradient
108  sobel_x = ndimage.sobel(tile, axis=0, mode='constant')
109  sobel_y = ndimage.sobel(tile, axis=1, mode='constant')
110  sobel = np.float32(np.hypot(sobel_x, sobel_y))
111
112  # Find largest contour in gradient image
113  contour = max(
114      opencv_processing_utils.find_all_contours(np.uint8(sobel)), key=len)
115  contour_length = len(contour)
116
117  # Draw contour (need a color image for visibility)
118  sobel_bgr = cv2.cvtColor(sobel, cv2.COLOR_GRAY2BGR)
119  contour_image = cv2.drawContours(sobel_bgr, contour, _CONTOUR_INDEX,
120                                   _BGR_RED, _CONTOUR_LINE_THICKNESS)
121  cv2.imwrite(f'{file_stem_with_suffix}_sobel_contour.png', contour_image)
122
123  # Try to detect QR code
124  detection_object = None
125  lowest_successful_alpha = None
126  qr_detector = cv2.QRCodeDetector()
127
128  # See if original tile is detectable
129  qr_code, _, _ = qr_detector.detectAndDecode(tile)
130  if qr_code and qr_code == _QR_CODE_VALUE:
131    logging.debug('Decoded correct QR code: %s without contrast changes',
132                  _QR_CODE_VALUE)
133    return qr_code, 0.0, contour_length
134  else:
135    qr_code, _ = qr_detector.detect(tile)
136    if qr_code:
137      detection_object = qr_code
138      lowest_successful_alpha = 0.0
139      logging.debug('Detected QR code without contrast changes')
140
141  # Modify contrast (not brightness) to see if QR code detectable/decodable
142  for a in np.arange(*_CONTRAST_ARANGE):
143    qr_tile = cv2.convertScaleAbs(tile, alpha=a, beta=0)
144    qr_code, _, _ = qr_detector.detectAndDecode(qr_tile)
145    if qr_code and qr_code == _QR_CODE_VALUE:
146      logging.debug('Decoded correct QR code: %s at alpha of %.2f',
147                    _QR_CODE_VALUE, a)
148      return qr_code, a, contour_length
149    elif qr_code:
150      logging.debug('Decoded other QR code: %s', qr_code)
151    else:
152      # If QR code already detected, only try to decode.
153      if detection_object:
154        continue
155      qr_code, _ = qr_detector.detect(qr_tile)
156      if qr_code:
157        logging.debug('Detected QR code at alpha of %.2f', a)
158        detection_object = qr_code
159        lowest_successful_alpha = a
160
161  return detection_object, lowest_successful_alpha, contour_length
162
163
164class HdrExtensionTest(its_base_test.ItsBaseTest):
165  """Tests HDR extension under dark lighting conditions.
166
167  Takes capture with and without HDR extension activated.
168  Verifies that QR code on the right is lit evenly,
169  or can be decoded/detected with the HDR extension on.
170  """
171
172  def test_hdr(self):
173    # Handle subdirectory
174    self.scene = 'scene_hdr'
175    with its_session_utils.ItsSession(
176        device_id=self.dut.serial,
177        camera_id=self.camera_id,
178        hidden_physical_id=self.hidden_physical_id) as cam:
179      props = cam.get_camera_properties()
180      props = cam.override_with_hidden_physical_camera_props(props)
181      test_name = os.path.join(self.log_path, _NAME)
182
183      # Determine camera supported extensions
184      supported_extensions = cam.get_supported_extensions(self.camera_id)
185      logging.debug('Supported extensions: %s', supported_extensions)
186
187      # Check SKIP conditions
188      first_api_level = its_session_utils.get_first_api_level(self.dut.serial)
189      camera_properties_utils.skip_unless(
190          _EXTENSION_HDR in supported_extensions and
191          first_api_level >= its_session_utils.ANDROID14_API_LEVEL)
192
193      # Establish connection with lighting controller
194      arduino_serial_port = lighting_control_utils.lighting_control(
195          self.lighting_cntl, self.lighting_ch)
196
197      # Turn OFF lights to darken scene
198      lighting_control_utils.set_lighting_state(
199          arduino_serial_port, self.lighting_ch, 'OFF')
200
201      # Check that tablet is connected and turn it off to validate lighting
202      self.turn_off_tablet()
203
204      # Validate lighting
205      cam.do_3a(do_af=False)
206      cap = cam.do_capture(
207          capture_request_utils.auto_capture_request(), cam.CAP_YUV)
208      y_plane, _, _ = image_processing_utils.convert_capture_to_planes(cap)
209      its_session_utils.validate_lighting(
210          y_plane, self.scene, state='OFF', log_path=self.log_path,
211          tablet_state='OFF')
212
213      self.setup_tablet()
214      self.set_screen_brightness(_TABLET_BRIGHTNESS)
215
216      its_session_utils.load_scene(
217          cam, props, self.scene, self.tablet, self.chart_distance,
218          lighting_check=False, log_path=self.log_path)
219
220      file_stem = f'{test_name}_{_FMT_NAME}_{_WIDTH}x{_HEIGHT}'
221
222      # Take capture without HDR extension activated as baseline
223      logging.debug('Taking capture without HDR extension')
224      out_surfaces = {'format': _FMT_NAME, 'width': _WIDTH, 'height': _HEIGHT}
225      try:
226        cam.do_3a()
227      except error_util.CameraItsError:
228        logging.error('Could not converge 3A in HDR scene')
229      req = capture_request_utils.auto_capture_request()
230      no_hdr_start_of_capture = time.time()
231      no_hdr_cap = cam.do_capture(req, out_surfaces)
232      no_hdr_end_of_capture = time.time()
233      no_hdr_capture_duration = no_hdr_end_of_capture - no_hdr_start_of_capture
234      logging.debug('no HDR cap duration: %.2f', no_hdr_capture_duration)
235      logging.debug('no HDR cap metadata: %s', no_hdr_cap['metadata'])
236      no_hdr_img = image_processing_utils.convert_capture_to_rgb_image(
237          no_hdr_cap)
238      image_processing_utils.write_image(
239          no_hdr_img, f'{file_stem}_no_HDR.jpg')
240
241      # Take capture with HDR extension
242      logging.debug('Taking capture with HDR extension')
243      out_surfaces = {'format': _FMT_NAME, 'width': _WIDTH, 'height': _HEIGHT}
244      req = capture_request_utils.auto_capture_request()
245      hdr_start_of_capture = time.time()
246      hdr_cap = cam.do_capture_with_extensions(
247          req, _EXTENSION_HDR, out_surfaces)
248      hdr_end_of_capture = time.time()
249      hdr_capture_duration = hdr_end_of_capture - hdr_start_of_capture
250      logging.debug('HDR cap duration: %.2f', hdr_capture_duration)
251      logging.debug('HDR cap metadata: %s', hdr_cap['metadata'])
252      hdr_img = image_processing_utils.convert_capture_to_rgb_image(
253          hdr_cap)
254      image_processing_utils.write_image(hdr_img, f'{file_stem}_HDR.jpg')
255
256      # Attempt to decode QR code with and without HDR
257      format_optional_float = lambda x: f'{x:.2f}' if x is not None else 'None'
258      logging.debug('Attempting to detect and decode QR code without HDR')
259      no_hdr_detection_object, no_hdr_alpha, no_hdr_length = analyze_qr_code(
260          no_hdr_img, f'{file_stem}_no_HDR')
261      logging.debug('No HDR code: %s, No HDR alpha: %s, '
262                    'No HDR contour length: %d',
263                    no_hdr_detection_object,
264                    format_optional_float(no_hdr_alpha),
265                    no_hdr_length)
266      logging.debug('Attempting to detect and decode QR code with HDR')
267      hdr_detection_object, hdr_alpha, hdr_length = analyze_qr_code(
268          hdr_img, f'{file_stem}_HDR')
269      logging.debug('HDR code: %s, HDR alpha: %s, HDR contour length: %d',
270                    hdr_detection_object,
271                    format_optional_float(hdr_alpha),
272                    hdr_length)
273
274      # Assert correct behavior
275      failure_messages = []
276      # Decoding QR code with HDR -> PASS
277      if hdr_detection_object != _QR_CODE_VALUE:
278        if hdr_alpha is None:  # Allow hdr_alpha to be falsy (0.0)
279          failure_messages.append(
280              'Unable to detect QR code with HDR extension')
281        if (no_hdr_alpha is not None and
282            hdr_alpha is not None and
283            no_hdr_alpha < hdr_alpha):
284          failure_messages.append('QR code was found at a lower contrast with '
285                                  f'HDR off ({no_hdr_alpha}) than with HDR on '
286                                  f'({hdr_alpha})')
287        if no_hdr_length > 0 and hdr_length / no_hdr_length > _GRADIENT_TOL:
288          failure_messages.append(
289              ('HDR gradient was not significantly '
290               'smaller than gradient without HDR. '
291               'Largest HDR gradient contour perimeter was '
292               f'{hdr_length / no_hdr_length} of '
293               'the size of largest non-HDR contour length, '
294               f'expected to be at least {_GRADIENT_TOL}')
295          )
296        else:
297          # If HDR gradient is better, allow PASS to account for cv2 flakiness
298          if failure_messages:
299            logging.error('\n'.join(failure_messages))
300            failure_messages = []
301
302      # Compare capture durations
303      duration_diff = hdr_capture_duration - no_hdr_capture_duration
304      if duration_diff < _DURATION_DIFF_TOL:
305        failure_messages.append('Capture with HDR did not take '
306                                'significantly more time than '
307                                'capture without HDR! '
308                                f'Difference: {duration_diff:.2f}, '
309                                f'Expected: {_DURATION_DIFF_TOL}')
310
311      if failure_messages:
312        raise AssertionError('\n'.join(failure_messages))
313
314
315if __name__ == '__main__':
316  test_runner.main()
317