1# Copyright 2024 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Verifies changing AE/AWB regions changes images AE/AWB results."""
15
16
17import logging
18import os.path
19
20from mobly import test_runner
21import numpy
22
23import camera_properties_utils
24import capture_request_utils
25import image_processing_utils
26import its_base_test
27import its_session_utils
28import opencv_processing_utils
29import video_processing_utils
30
31_AE_CHANGE_THRESH = 1  # Incorrect behavior is empirically < 0.5 percent
32_AWB_CHANGE_THRESH = 2  # Incorrect behavior is empirically < 1.5 percent
33_AE_AWB_METER_WEIGHT = 1000  # 1 - 1000 with 1000 as the highest
34_ARUCO_MARKERS_COUNT = 4
35_AE_AWB_REGIONS_AVAILABLE = 1  # Valid range is >= 0, and unavailable if 0
36_MIRRORED_PREVIEW_SENSOR_ORIENTATIONS = (0, 180)
37_NAME = os.path.splitext(os.path.basename(__file__))[0]
38_NUM_AE_AWB_REGIONS = 4
39_PERCENTAGE = 100
40_REGION_DURATION_MS = 1800  # 1.8 seconds
41_TAP_COORDINATES = (500, 500)  # Location to tap tablet screen via adb
42
43
44def _convert_image_coords_to_sensor_coords(
45    aa_width, aa_height, coords, img_width, img_height):
46  """Transform image coordinates to sensor coordinate system.
47
48  Calculate the difference between sensor active array and image aspect ratio.
49  Taking the difference into account, figure out if the width or height has been
50  cropped. Using this information, transform the image coordinates to sensor
51  coordinates.
52
53  Args:
54    aa_width: int; active array width.
55    aa_height: int; active array height.
56    coords: coordinates; defined by aruco markers from camera capture.
57    img_width: int; width of image.
58    img_height: int; height of image.
59  Returns:
60    sensor_coords: coordinates; corresponding coorediates on
61      sensor coordinate system.
62  """
63  # TODO: b/330382627 - find out if distortion correction is ON/OFF
64  aa_aspect_ratio = aa_width / aa_height
65  image_aspect_ratio = img_width / img_height
66  if aa_aspect_ratio >= image_aspect_ratio:
67    # If aa aspect ratio is greater than image aspect ratio, then
68    # sensor width is being cropped
69    aspect_ratio_multiplication_factor = aa_height / img_height
70    crop_width = img_width * aspect_ratio_multiplication_factor
71    buffer = (aa_width - crop_width) / 2
72    sensor_coords = (coords[0] * aspect_ratio_multiplication_factor + buffer,
73                     coords[1] * aspect_ratio_multiplication_factor)
74  else:
75    # If aa aspect ratio is less than image aspect ratio, then
76    # sensor height is being cropped
77    aspect_ratio_multiplication_factor = aa_width / img_width
78    crop_height = img_height * aspect_ratio_multiplication_factor
79    buffer = (aa_height - crop_height) / 2
80    sensor_coords = (coords[0] * aspect_ratio_multiplication_factor,
81                     coords[1] * aspect_ratio_multiplication_factor + buffer)
82  logging.debug('Sensor coordinates: %s', sensor_coords)
83  return sensor_coords
84
85
86def _define_metering_regions(img, img_path, chart_path, props, width, height):
87  """Define 4 metering rectangles for AE/AWB regions based on ArUco markers.
88
89  Args:
90    img: numpy array; RGB image.
91    img_path: str; image file location.
92    chart_path: str; chart file location.
93    props: dict; camera properties object.
94    width: int; preview's width in pixels.
95    height: int; preview's height in pixels.
96  Returns:
97    ae_awb_regions: metering rectangles; AE/AWB metering regions.
98  """
99  # Extract chart coordinates from aruco markers
100  # TODO: b/330382627 - get chart boundary from 4 aruco markers instead of 2
101  aruco_corners, aruco_ids, _ = opencv_processing_utils.find_aruco_markers(
102      img, img_path)
103  tl, tr, br, bl = (
104      opencv_processing_utils.get_chart_boundary_from_aruco_markers(
105          aruco_corners, aruco_ids, img, chart_path))
106
107  # Convert image coordinates to sensor coordinates for metering rectangles
108  aa = props['android.sensor.info.activeArraySize']
109  aa_width, aa_height = aa['right'] - aa['left'], aa['bottom'] - aa['top']
110  logging.debug('Active array size: %s', aa)
111  sc_tl = _convert_image_coords_to_sensor_coords(
112      aa_width, aa_height, tl, width, height)
113  sc_tr = _convert_image_coords_to_sensor_coords(
114      aa_width, aa_height, tr, width, height)
115  sc_br = _convert_image_coords_to_sensor_coords(
116      aa_width, aa_height, br, width, height)
117  sc_bl = _convert_image_coords_to_sensor_coords(
118      aa_width, aa_height, bl, width, height)
119
120  # Define AE/AWB regions through ArUco markers' positions
121  region_blue, region_light, region_dark, region_yellow = (
122      opencv_processing_utils.define_metering_rectangle_values(
123          props, sc_tl, sc_tr, sc_br, sc_bl, aa_width, aa_height))
124
125  # Create a dictionary of AE/AWB regions for testing
126  ae_awb_regions = {
127      'aeAwbRegionOne': region_blue,
128      'aeAwbRegionTwo': region_light,
129      'aeAwbRegionThree': region_dark,
130      'aeAwbRegionFour': region_yellow,
131  }
132  return ae_awb_regions
133
134
135def _do_ae_check(light, dark, file_name_with_path):
136  """Checks luma change between two images is above threshold.
137
138  Checks that the Y-average of image with darker metering region
139  is higher than the Y-average of image with lighter metering
140  region. Y stands for brightness, or "luma".
141
142  Args:
143    light: RGB image; metering light region.
144    dark: RGB image; metering dark region.
145    file_name_with_path: str; path to preview recording.
146  """
147  # Converts img to YUV and returns Y-average
148  light_y = opencv_processing_utils.convert_to_y(light, 'RGB')
149  light_y_avg = numpy.average(light_y)
150  dark_y = opencv_processing_utils.convert_to_y(dark, 'RGB')
151  dark_y_avg = numpy.average(dark_y)
152  logging.debug('Light image Y-average: %.4f', light_y_avg)
153  logging.debug('Dark image Y-average: %.4f', dark_y_avg)
154  # Checks average change in Y-average between two images
155  y_avg_change = (
156      (dark_y_avg-light_y_avg)/light_y_avg)*_PERCENTAGE
157  logging.debug('Y-average percentage change: %.4f', y_avg_change)
158
159  # Don't change print to logging. Used for KPI.
160  print(f'{_NAME}_ae_y_change: ', y_avg_change)
161
162  if y_avg_change < _AE_CHANGE_THRESH:
163    raise AssertionError(
164        f'Luma change {y_avg_change} is less than the threshold: '
165        f'{_AE_CHANGE_THRESH}')
166  else:
167    its_session_utils.remove_mp4_file(file_name_with_path)
168
169
170def _do_awb_check(blue, yellow):
171  """Checks the ratio of red over blue between two RGB images.
172
173  Checks that the R/B of image with blue metering region
174  is higher than the R/B of image with yellow metering
175  region.
176
177  Args:
178    blue: RGB image; metering blue region.
179    yellow: RGB image; metering yellow region.
180  Returns:
181    failure_messages: (list of strings) of error messages.
182  """
183  # Calculates average red value over average blue value in images
184  blue_r_b_ratio = _get_red_blue_ratio(blue)
185  yellow_r_b_ratio = _get_red_blue_ratio(yellow)
186  logging.debug('Blue image R/B ratio: %s', blue_r_b_ratio)
187  logging.debug('Yellow image R/B ratio: %s', yellow_r_b_ratio)
188  # Calculates change in red over blue values between two images
189  r_b_ratio_change = (
190      (blue_r_b_ratio-yellow_r_b_ratio)/yellow_r_b_ratio)*_PERCENTAGE
191  logging.debug('R/B ratio change in percentage: %.4f', r_b_ratio_change)
192
193  # Don't change print to logging. Used for KPI.
194  print(f'{_NAME}_awb_rb_change: ', r_b_ratio_change)
195
196  if r_b_ratio_change < _AWB_CHANGE_THRESH:
197    raise AssertionError(
198        f'R/B ratio change {r_b_ratio_change} is less than the'
199        f' threshold: {_AWB_CHANGE_THRESH}')
200
201
202def _extract_and_process_key_frames_from_recording(log_path, file_name):
203  """Extract key frames (1 frame/second) from recordings.
204
205  Args:
206    log_path: str; file location.
207    file_name: str; file name for saved video.
208  Returns:
209    dictionary of images.
210  """
211  # TODO: b/330382627 - Add function to preview_processing_utils
212  # Extract key frames from video
213  key_frame_files = video_processing_utils.extract_key_frames_from_video(
214      log_path, file_name)
215
216  # Process key frame files
217  key_frames = []
218  for file in key_frame_files:
219    img = image_processing_utils.convert_image_to_numpy_array(
220        os.path.join(log_path, file))
221    key_frames.append(img)
222  logging.debug('Frame size %d x %d', key_frames[0].shape[1],
223                key_frames[0].shape[0])
224  return key_frames
225
226
227def _get_largest_common_aspect_ratio_preview_size(cam, camera_id):
228  """Get largest, supported preview size that matches sensor's aspect ratio.
229
230  Args:
231    cam: obj; camera object.
232    camera_id: int; device id.
233  Returns:
234    preview_size: str; largest, supported preview size w/ 4:3, or 16:9
235        aspect ratio.
236  """
237  preview_sizes = cam.get_supported_preview_sizes(camera_id)
238  dimensions = lambda s: (int(s.split('x')[0]), int(s.split('x')[1]))
239  for size in reversed(preview_sizes):
240    if capture_request_utils.is_common_aspect_ratio(dimensions(size)):
241      preview_size = size
242      logging.debug('Largest common aspect ratio preview size: %s',
243                    preview_size)
244      break
245  return preview_size
246
247
248def _get_red_blue_ratio(img):
249  """Computes the ratios of average red over blue in img.
250
251  Args:
252    img: numpy array; RGB image.
253  Returns:
254    r_b_ratio: float; ratio of R and B channel means.
255  """
256  img_means = image_processing_utils.compute_image_means(img)
257  r_b_ratio = img_means[0]/img_means[2]
258  return r_b_ratio
259
260
261class AeAwbRegions(its_base_test.ItsBaseTest):
262  """Tests that changing AE and AWB regions changes image's RGB values.
263
264  Test records an 8 seconds preview recording, and meters a different
265  AE/AWB region (blue, light, dark, yellow) for every 2 seconds.
266  Extracts a frame from each second of recording with a total of 8 frames
267  (2 from each region). For AE check, a frame from light is compared to the
268  dark region. For AWB check, a frame from blue is compared to the yellow
269  region.
270
271  """
272
273  def test_ae_awb_regions(self):
274    """Test AE and AWB regions."""
275
276    with its_session_utils.ItsSession(
277        device_id=self.dut.serial,
278        camera_id=self.camera_id,
279        hidden_physical_id=self.hidden_physical_id) as cam:
280      props = cam.get_camera_properties()
281      props = cam.override_with_hidden_physical_camera_props(props)
282      log_path = self.log_path
283      test_name_with_log_path = os.path.join(log_path, _NAME)
284
285      # Load chart for scene
286      its_session_utils.load_scene(
287          cam, props, self.scene, self.tablet, self.chart_distance,
288          log_path)
289
290      # Tap tablet to remove gallery buttons
291      if self.tablet:
292        self.tablet.adb.shell(
293            f'input tap {_TAP_COORDINATES[0]} {_TAP_COORDINATES[1]}')
294
295      # Check skip conditions
296      max_ae_regions = props['android.control.maxRegionsAe']
297      max_awb_regions = props['android.control.maxRegionsAwb']
298      first_api_level = its_session_utils.get_first_api_level(self.dut.serial)
299      camera_properties_utils.skip_unless(
300          first_api_level >= its_session_utils.ANDROID15_API_LEVEL and
301          camera_properties_utils.ae_regions(props) and
302          (max_awb_regions >= _AE_AWB_REGIONS_AVAILABLE or
303           max_ae_regions >= _AE_AWB_REGIONS_AVAILABLE))
304      logging.debug('maximum AE regions: %d', max_ae_regions)
305      logging.debug('maximum AWB regions: %d', max_awb_regions)
306
307      # Find largest preview size to define capture size to find aruco markers
308      preview_size = _get_largest_common_aspect_ratio_preview_size(
309          cam, self.camera_id)
310      width = int(preview_size.split('x')[0])
311      height = int(preview_size.split('x')[1])
312      req = capture_request_utils.auto_capture_request()
313      fmt = {'format': 'yuv', 'width': width, 'height': height}
314      cam.do_3a()
315      cap = cam.do_capture(req, fmt)
316
317      # Save image and convert to numpy array
318      img = image_processing_utils.convert_capture_to_rgb_image(
319          cap, props=props)
320      img_path = f'{test_name_with_log_path}_aruco_markers.jpg'
321      image_processing_utils.write_image(img, img_path)
322      img = image_processing_utils.convert_image_to_uint8(img)
323
324      # Define AE/AWB metering regions
325      chart_path = f'{test_name_with_log_path}_chart_boundary.jpg'
326      ae_awb_regions = _define_metering_regions(
327          img, img_path, chart_path, props, width, height)
328
329      # Do preview recording with pre-defined AE/AWB regions
330      recording_obj = cam.do_preview_recording_with_dynamic_ae_awb_region(
331          preview_size, ae_awb_regions, _REGION_DURATION_MS)
332      logging.debug('Tested quality: %s', recording_obj['quality'])
333
334      # Grab the video from the save location on DUT
335      self.dut.adb.pull([recording_obj['recordedOutputPath'], log_path])
336      file_name = recording_obj['recordedOutputPath'].split('/')[-1]
337      file_name_with_path = os.path.join(log_path, file_name)
338      logging.debug('file_name: %s', file_name)
339
340      # Extract 8 key frames per 8 seconds of preview recording
341      # Meters each region of 4 (blue, light, dark, yellow) for 2 seconds
342      # Unpack frames based on metering region's color
343      # If testing front camera with preview mirrored, reverse order.
344      # pylint: disable=unbalanced-tuple-unpacking
345      if ((props['android.lens.facing'] ==
346           camera_properties_utils.LENS_FACING['FRONT']) and
347          props['android.sensor.orientation'] in
348          _MIRRORED_PREVIEW_SENSOR_ORIENTATIONS):
349        _, yellow, _, dark, _, light, _, blue = (
350            _extract_and_process_key_frames_from_recording(
351                log_path, file_name))
352      else:
353        _, blue, _, light, _, dark, _, yellow = (
354            _extract_and_process_key_frames_from_recording(
355                log_path, file_name))
356
357      # AWB Check : Verify R/B ratio change is greater than threshold
358      if max_awb_regions >= _AE_AWB_REGIONS_AVAILABLE:
359        _do_awb_check(blue, yellow)
360
361      # AE Check: Extract the Y component from rectangle patch
362      if max_ae_regions >= _AE_AWB_REGIONS_AVAILABLE:
363        _do_ae_check(light, dark, file_name_with_path)
364
365if __name__ == '__main__':
366  test_runner.main()
367