1# Copyright 2023 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Utility functions for zoom capture. 15""" 16 17from collections.abc import Iterable 18import dataclasses 19import logging 20import math 21import cv2 22import matplotlib.pyplot as plt 23import numpy 24 25import camera_properties_utils 26import capture_request_utils 27import image_processing_utils 28import opencv_processing_utils 29 30_CIRCLE_COLOR = 0 # [0: black, 255: white] 31_CIRCLE_AR_RTOL = 0.15 # contour width vs height (aspect ratio) 32_CIRCLISH_RTOL = 0.05 # contour area vs ideal circle area pi*((w+h)/4)**2 33_CONTOUR_AREA_LOGGING_THRESH = 0.8 # logging tol to cut down spam in log file 34_CV2_LINE_THICKNESS = 3 # line thickness for drawing on images 35_CV2_RED = (255, 0, 0) # color in cv2 to draw lines 36_DEFAULT_FOV_RATIO = 1 # ratio of sub camera's fov over logical camera's fov 37_MIN_AREA_RATIO = 0.00013 # Found empirically with partners 38_MIN_CIRCLE_PTS = 25 39_MIN_FOCUS_DIST_TOL = 0.80 # allow charts a little closer than min 40_OFFSET_ATOL = 10 # number of pixels 41_OFFSET_RTOL_MIN_FD = 0.30 42_RADIUS_RTOL_MIN_FD = 0.15 43OFFSET_RTOL = 1.0 # TODO: b/342176245 - enable offset check w/ marker identity 44RADIUS_RTOL = 0.10 45ZOOM_MIN_THRESH = 2.0 46ZOOM_MAX_THRESH = 10.0 47ZOOM_RTOL = 0.01 # variation of zoom ratio due to floating point 48PRV_Z_RTOL = 0.02 # 2% variation of zoom ratio between request and result 49PREFERRED_BASE_ZOOM_RATIO = 1 # Preferred base image for zoom data verification 50JPEG_STR = 'jpg' 51 52 53@dataclasses.dataclass 54class ZoomTestData: 55 """Class to store zoom-related metadata for a capture.""" 56 result_zoom: float 57 circle: Iterable[float] # TODO: b/338638040 - create a dataclass for circles 58 radius_tol: float 59 offset_tol: float 60 focal_length: float 61 physical_id: int = dataclasses.field(default=None) 62 63 64def get_test_tols_and_cap_size(cam, props, chart_distance, debug): 65 """Determine the tolerance per camera based on test rig and camera params. 66 67 Cameras are pre-filtered to only include supportable cameras. 68 Supportable cameras are: YUV(RGB) 69 70 Args: 71 cam: camera object 72 props: dict; physical camera properties dictionary 73 chart_distance: float; distance to chart in cm 74 debug: boolean; log additional data 75 76 Returns: 77 dict of TOLs with camera focal length as key 78 largest common size across all cameras 79 """ 80 ids = camera_properties_utils.logical_multi_camera_physical_ids(props) 81 physical_props = {} 82 physical_ids = [] 83 for i in ids: 84 physical_props[i] = cam.get_camera_properties_by_id(i) 85 # find YUV capable physical cameras 86 if camera_properties_utils.backward_compatible(physical_props[i]): 87 physical_ids.append(i) 88 89 # find physical camera focal lengths that work well with rig 90 chart_distance_m = abs(chart_distance)/100 # convert CM to M 91 test_tols = {} 92 test_yuv_sizes = [] 93 for i in physical_ids: 94 yuv_sizes = capture_request_utils.get_available_output_sizes( 95 'yuv', physical_props[i]) 96 test_yuv_sizes.append(yuv_sizes) 97 if debug: 98 logging.debug('cam[%s] yuv sizes: %s', i, str(yuv_sizes)) 99 100 # determine if minimum focus distance is less than rig depth 101 min_fd = physical_props[i]['android.lens.info.minimumFocusDistance'] 102 for fl in physical_props[i]['android.lens.info.availableFocalLengths']: 103 logging.debug('cam[%s] min_fd: %.3f (diopters), fl: %.2f', i, min_fd, fl) 104 if (math.isclose(min_fd, 0.0, rel_tol=1E-6) or # fixed focus 105 (1.0/min_fd < chart_distance_m*_MIN_FOCUS_DIST_TOL)): 106 test_tols[fl] = (RADIUS_RTOL, OFFSET_RTOL) 107 else: 108 test_tols[fl] = (_RADIUS_RTOL_MIN_FD, _OFFSET_RTOL_MIN_FD) 109 logging.debug('loosening RTOL for cam[%s]: ' 110 'min focus distance too large.', i) 111 # find intersection of formats for max common format 112 common_sizes = list(set.intersection(*[set(list) for list in test_yuv_sizes])) 113 if debug: 114 logging.debug('common_fmt: %s', max(common_sizes)) 115 116 return test_tols, max(common_sizes) 117 118 119def find_center_circle( 120 img, img_name, size, zoom_ratio, min_zoom_ratio, 121 expected_color=_CIRCLE_COLOR, circle_ar_rtol=_CIRCLE_AR_RTOL, 122 circlish_rtol=_CIRCLISH_RTOL, min_circle_pts=_MIN_CIRCLE_PTS, 123 fov_ratio=_DEFAULT_FOV_RATIO, debug=False, draw_color=_CV2_RED, 124 write_img=True): 125 """Find circle closest to image center for scene with multiple circles. 126 127 Finds all contours in the image. Rejects those too small and not enough 128 points to qualify as a circle. The remaining contours must have center 129 point of color=color and are sorted based on distance from the center 130 of the image. The contour closest to the center of the image is returned. 131 If circle is not found due to zoom ratio being larger than ZOOM_MAX_THRESH 132 or the circle being cropped, None is returned. 133 134 Note: hierarchy is not used as the hierarchy for black circles changes 135 as the zoom level changes. 136 137 Args: 138 img: numpy img array with pixel values in [0,255] 139 img_name: str file name for saved image 140 size: [width, height] of the image 141 zoom_ratio: zoom_ratio for the particular capture 142 min_zoom_ratio: min_zoom_ratio supported by the camera device 143 expected_color: int 0 --> black, 255 --> white 144 circle_ar_rtol: float aspect ratio relative tolerance 145 circlish_rtol: float contour area vs ideal circle area pi*((w+h)/4)**2 146 min_circle_pts: int minimum number of points to define a circle 147 fov_ratio: ratio of sub camera over logical camera's field of view 148 debug: bool to save extra data 149 draw_color: cv2 color in RGB to draw circle and circle center on the image 150 write_img: bool: True - save image with circle and center 151 False - don't save image. 152 153 Returns: 154 circle: [center_x, center_y, radius] 155 """ 156 157 width, height = size 158 min_area = ( 159 _MIN_AREA_RATIO * width * height * zoom_ratio * zoom_ratio * fov_ratio) 160 161 # create a copy of image to avoid modification on the original image since 162 # image_processing_utils.convert_image_to_uint8 uses mutable np array methods 163 if debug: 164 img = numpy.ndarray.copy(img) 165 166 # convert [0, 1] image to [0, 255] and cast as uint8 167 if img.dtype != numpy.uint8: 168 img = image_processing_utils.convert_image_to_uint8(img) 169 170 # gray scale & otsu threshold to binarize the image 171 gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) 172 _, img_bw = cv2.threshold( 173 numpy.uint8(gray), 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU) 174 175 # use OpenCV to find contours (connected components) 176 contours = opencv_processing_utils.find_all_contours(255-img_bw) 177 178 # write copy of image for debug purposes 179 if debug: 180 img_copy_name = img_name.split('.')[0] + '_copy.jpg' 181 image_processing_utils.write_image(numpy.expand_dims( 182 (255-img_bw).astype(numpy.float)/255.0, axis=2), img_copy_name) 183 184 # check contours and find the best circle candidates 185 circles = [] 186 img_ctr = [gray.shape[1] // 2, gray.shape[0] // 2] 187 logging.debug('img center x,y: %d, %d', img_ctr[0], img_ctr[1]) 188 logging.debug('min area: %d, min circle pts: %d', min_area, min_circle_pts) 189 logging.debug('circlish_rtol: %.3f', circlish_rtol) 190 191 for contour in contours: 192 area = cv2.contourArea(contour) 193 if area > min_area * _CONTOUR_AREA_LOGGING_THRESH: # skip tiny contours 194 logging.debug('area: %d, min_area: %d, num_pts: %d, min_circle_pts: %d', 195 area, min_area, len(contour), min_circle_pts) 196 if area > min_area and len(contour) >= min_circle_pts: 197 shape = opencv_processing_utils.component_shape(contour) 198 radius = (shape['width'] + shape['height']) / 4 199 circle_color = img_bw[shape['cty']][shape['ctx']] 200 circlish = round((math.pi * radius**2) / area, 4) 201 logging.debug('color: %s, circlish: %.2f, WxH: %dx%d', 202 circle_color, circlish, shape['width'], shape['height']) 203 if (circle_color == expected_color and 204 math.isclose(1, circlish, rel_tol=circlish_rtol) and 205 math.isclose(shape['width'], shape['height'], 206 rel_tol=circle_ar_rtol)): 207 logging.debug('circle found: r: %.2f, area: %.2f\n', radius, area) 208 circles.append([shape['ctx'], shape['cty'], radius, circlish, area]) 209 else: 210 logging.debug('circle rejected: bad color, circlish or aspect ratio\n') 211 212 if not circles: 213 zoom_ratio_value = zoom_ratio / min_zoom_ratio 214 if zoom_ratio_value >= ZOOM_MAX_THRESH: 215 logging.debug('No circle was detected, but zoom %.2f exceeds' 216 ' maximum zoom threshold', zoom_ratio_value) 217 return None 218 else: 219 raise AssertionError( 220 'No circle detected for zoom ratio <= ' 221 f'{ZOOM_MAX_THRESH}. ' 222 'Take pictures according to instructions carefully!') 223 else: 224 logging.debug('num of circles found: %s', len(circles)) 225 226 if debug: 227 logging.debug('circles [x, y, r, pi*r**2/area, area]: %s', str(circles)) 228 229 # find circle closest to center 230 circle = min( 231 circles, key=lambda x: math.hypot(x[0] - img_ctr[0], x[1] - img_ctr[1])) 232 233 # check if circle is cropped because of zoom factor 234 if opencv_processing_utils.is_circle_cropped(circle, size): 235 logging.debug('zoom %.2f is too large! Skip further captures', zoom_ratio) 236 return None 237 238 # mark image center 239 size = gray.shape 240 m_x, m_y = size[1] // 2, size[0] // 2 241 marker_size = _CV2_LINE_THICKNESS * 10 242 cv2.drawMarker(img, (m_x, m_y), draw_color, markerType=cv2.MARKER_CROSS, 243 markerSize=marker_size, thickness=_CV2_LINE_THICKNESS) 244 245 # add circle to saved image 246 center_i = (int(round(circle[0], 0)), int(round(circle[1], 0))) 247 radius_i = int(round(circle[2], 0)) 248 cv2.circle(img, center_i, radius_i, draw_color, _CV2_LINE_THICKNESS) 249 if write_img: 250 image_processing_utils.write_image(img / 255.0, img_name) 251 252 return circle 253 254 255def preview_zoom_data_to_string(test_data): 256 """Returns formatted string from test_data. 257 258 Floats are capped at 2 floating points. 259 260 Args: 261 test_data: ZoomTestData with relevant test data. 262 263 Returns: 264 Formatted String 265 """ 266 output = [] 267 for key, value in dataclasses.asdict(test_data).items(): 268 if isinstance(value, float): 269 output.append(f'{key}: {value:.2f}') 270 elif isinstance(value, list): 271 output.append( 272 f"{key}: [{', '.join([f'{item:.2f}' for item in value])}]") 273 else: 274 output.append(f'{key}: {value}') 275 276 return ', '.join(output) 277 278 279def verify_zoom_results(test_data, size, z_max, z_min): 280 """Verify that the output images' zoom level reflects the correct zoom ratios. 281 282 This test verifies that the center and radius of the circles in the output 283 images reflects the zoom ratios being set. The larger the zoom ratio, the 284 larger the circle. And the distance from the center of the circle to the 285 center of the image is proportional to the zoom ratio as well. 286 287 Args: 288 test_data: Iterable[ZoomTestData] 289 size: array; the width and height of the images 290 z_max: float; the maximum zoom ratio being tested 291 z_min: float; the minimum zoom ratio being tested 292 293 Returns: 294 Boolean whether the test passes (True) or not (False) 295 """ 296 # assert some range is tested before circles get too big 297 test_success = True 298 299 zoom_max_thresh = ZOOM_MAX_THRESH 300 z_max_ratio = z_max / z_min 301 if z_max_ratio < ZOOM_MAX_THRESH: 302 zoom_max_thresh = z_max_ratio 303 304 # handle capture orders like [1, 0.5, 1.5, 2...] 305 test_data_zoom_values = [v.result_zoom for v in test_data] 306 test_data_max_z = max(test_data_zoom_values) / min(test_data_zoom_values) 307 logging.debug('test zoom ratio max: %.2f vs threshold %.2f', 308 test_data_max_z, zoom_max_thresh) 309 310 if not math.isclose(test_data_max_z, zoom_max_thresh, rel_tol=ZOOM_RTOL): 311 test_success = False 312 e_msg = (f'Max zoom ratio tested: {test_data_max_z:.4f}, ' 313 f'range advertised min: {z_min}, max: {z_max} ' 314 f'THRESH: {zoom_max_thresh + ZOOM_RTOL}') 315 logging.error(e_msg) 316 317 return test_success and verify_zoom_data(test_data, size) 318 319 320def verify_zoom_data(test_data, size, plot_name_stem=None): 321 """Verify that the output images' zoom level reflects the correct zoom ratios. 322 323 This test verifies that the center and radius of the circles in the output 324 images reflects the zoom ratios being set. The larger the zoom ratio, the 325 larger the circle. And the distance from the center of the circle to the 326 center of the image is proportional to the zoom ratio as well. 327 328 Args: 329 test_data: Iterable[ZoomTestData] 330 size: array; the width and height of the images 331 plot_name_stem: str; log path and name of the plot 332 333 Returns: 334 Boolean whether the test passes (True) or not (False) 335 """ 336 # assert some range is tested before circles get too big 337 test_success = True 338 339 # initialize relative size w/ zoom[0] for diff zoom ratio checks 340 radius_0 = float(test_data[0].circle[2]) 341 z_0 = float(test_data[0].result_zoom) 342 343 # use 1x ~ 1.1x data as base image if available 344 if z_0 < PREFERRED_BASE_ZOOM_RATIO: 345 for i, data in enumerate(test_data): 346 if (test_data[i].result_zoom >= PREFERRED_BASE_ZOOM_RATIO and 347 math.isclose(test_data[i].result_zoom, PREFERRED_BASE_ZOOM_RATIO, 348 rel_tol=0.1)): 349 radius_0 = float(test_data[i].circle[2]) 350 z_0 = float(test_data[i].result_zoom) 351 break 352 logging.debug('z_0: %.3f, radius_0: %.3f', z_0, radius_0) 353 if plot_name_stem: 354 frame_numbers = [] 355 z_variations = [] 356 rel_variations = [] 357 radius_tols = [] 358 for i, data in enumerate(test_data): 359 logging.debug(' ') # add blank line between frames 360 logging.debug('Frame# %d {%s}', i, preview_zoom_data_to_string(data)) 361 logging.debug('Zoom: %.2f, fl: %.2f', data.result_zoom, data.focal_length) 362 offset_xy = [(data.circle[0] - size[0] // 2), 363 (data.circle[1] - size[1] // 2)] 364 logging.debug('Circle r: %.1f, center offset x, y: %d, %d', 365 data.circle[2], offset_xy[0], offset_xy[1]) 366 z_ratio = data.result_zoom / z_0 367 368 # check relative size against zoom[0] 369 radius_ratio = data.circle[2] / radius_0 370 371 # Calculate variations 372 z_variation = z_ratio - radius_ratio 373 relative_variation = abs(z_variation) / max(abs(z_ratio), abs(radius_ratio)) 374 375 # Store values for plotting 376 if plot_name_stem: 377 frame_numbers.append(i) 378 z_variations.append(z_variation) 379 rel_variations.append(relative_variation) 380 radius_tols.append(data.radius_tol) 381 382 logging.debug('r ratio req: %.3f, measured: %.3f', 383 z_ratio, radius_ratio) 384 msg = (f'{i} Circle radius ratio: result({data.result_zoom:.3f}/{z_0:.3f}):' 385 f' {z_ratio:.3f}, circle({data.circle[2]:.3f}/{radius_0:.3f}):' 386 f' {radius_ratio:.3f}, RTOL: {data.radius_tol}' 387 f' z_var: {z_variation:.3f}, rel_var: {relative_variation:.3f}') 388 if not math.isclose(z_ratio, radius_ratio, rel_tol=data.radius_tol): 389 test_success = False 390 logging.error(msg) 391 else: 392 logging.debug(msg) 393 394 # check relative offset against init vals w/ no focal length change 395 # set init values for first capture or change in physical cam focal length 396 if i == 0 or test_data[i-1].focal_length != data.focal_length: 397 z_init = float(data.result_zoom) 398 offset_hypot_init = math.hypot(offset_xy[0], offset_xy[1]) 399 logging.debug('offset_hypot_init: %.3f', offset_hypot_init) 400 d_msg = (f'-- init {i} zoom: {data.result_zoom:.2f}, ' 401 f'offset init: {offset_hypot_init:.1f}, ' 402 f'offset rel: {math.hypot(offset_xy[0], offset_xy[1]):.1f}, ' 403 f'zoom: {z_ratio:.1f} ') 404 logging.debug(d_msg) 405 else: # check 406 z_ratio = data.result_zoom / z_init 407 offset_hypot_rel = math.hypot(offset_xy[0], offset_xy[1]) / z_ratio 408 logging.debug('offset_hypot_rel: %.3f', offset_hypot_rel) 409 410 rel_tol = data.offset_tol 411 if not math.isclose(offset_hypot_init, offset_hypot_rel, 412 rel_tol=rel_tol, abs_tol=_OFFSET_ATOL): 413 test_success = False 414 e_msg = (f'{i} zoom: {data.result_zoom:.2f}, ' 415 f'offset init: {offset_hypot_init:.4f}, ' 416 f'offset rel: {offset_hypot_rel:.4f}, ' 417 f'Zoom: {z_ratio:.1f}, ' 418 f'RTOL: {rel_tol}, ATOL: {_OFFSET_ATOL}') 419 logging.error(e_msg) 420 else: 421 d_msg = (f'{i} zoom: {data.result_zoom:.2f}, ' 422 f'offset init: {offset_hypot_init:.1f}, ' 423 f'offset rel: {offset_hypot_rel:.1f}, ' 424 f'offset dist: {math.hypot(offset_xy[0], offset_xy[1]):.1f}, ' 425 f'Zoom: {z_ratio:.1f}, ' 426 f'RTOL: {rel_tol}, ATOL: {_OFFSET_ATOL}') 427 logging.debug(d_msg) 428 429 if plot_name_stem: 430 plot_variation(frame_numbers, z_variations, None, 431 f'{plot_name_stem}_variations.png', 'Zoom Variation') 432 plot_variation(frame_numbers, rel_variations, radius_tols, 433 f'{plot_name_stem}_relative.png', 'Relative Variation') 434 435 return test_success 436 437 438def verify_preview_zoom_results(test_data, size, z_max, z_min, z_step_size, 439 plot_name_stem): 440 """Verify that the output images' zoom level reflects the correct zoom ratios. 441 442 This test verifies that the center and radius of the circles in the output 443 images reflects the zoom ratios being set. The larger the zoom ratio, the 444 larger the circle. And the distance from the center of the circle to the 445 center of the image is proportional to the zoom ratio as well. Verifies 446 that circles are detected throughout the zoom range. 447 448 Args: 449 test_data: Iterable[ZoomTestData] 450 size: array; the width and height of the images 451 z_max: float; the maximum zoom ratio being tested 452 z_min: float; the minimum zoom ratio being tested 453 z_step_size: float; zoom step size to zoom from z_min to z_max 454 plot_name_stem: str; log path and name of the plot 455 456 Returns: 457 Boolean whether the test passes (True) or not (False) 458 """ 459 test_success = True 460 461 test_data_zoom_values = [v.result_zoom for v in test_data] 462 results_z_max = max(test_data_zoom_values) 463 results_z_min = min(test_data_zoom_values) 464 logging.debug('capture result: min zoom: %.2f vs max zoom: %.2f', 465 results_z_min, results_z_max) 466 467 # check if max zoom in capture result close to requested zoom range 468 if (math.isclose(results_z_max, z_max, rel_tol=PRV_Z_RTOL) or 469 math.isclose(results_z_max, z_max - z_step_size, rel_tol=PRV_Z_RTOL)): 470 d_msg = (f'results_z_max = {results_z_max:.2f} is close to requested ' 471 f'z_max = {z_max:.2f} or z_max-step = {z_max-z_step_size:.2f} ' 472 f'by {PRV_Z_RTOL:.2f} Tol') 473 logging.debug(d_msg) 474 else: 475 test_success = False 476 e_msg = (f'Max zoom ratio {results_z_max:.4f} in capture results ' 477 f'not close to {z_max:.2f} and ' 478 f'z_max-step = {z_max-z_step_size:.2f} by {PRV_Z_RTOL:.2f} ' 479 f'tolerance.') 480 logging.error(e_msg) 481 482 if math.isclose(results_z_min, z_min, rel_tol=PRV_Z_RTOL): 483 d_msg = (f'results_z_min = {results_z_min:.2f} is close to requested ' 484 f'z_min = {z_min:.2f} by {PRV_Z_RTOL:.2f} Tol') 485 logging.debug(d_msg) 486 else: 487 test_success = False 488 e_msg = (f'Min zoom ratio {results_z_min:.4f} in capture results ' 489 f'not close to {z_min:.2f} by {PRV_Z_RTOL:.2f} tolerance.') 490 logging.error(e_msg) 491 492 return test_success and verify_zoom_data(test_data, size, plot_name_stem) 493 494 495def get_preview_zoom_params(zoom_range, steps): 496 """Returns zoom min, max, step_size based on zoom range and steps. 497 498 Determine zoom min, max, step_size based on zoom range, steps. 499 Zoom max is capped due to current ITS box size limitation. 500 501 Args: 502 zoom_range: [float,float]; Camera's zoom range 503 steps: int; number of steps 504 505 Returns: 506 zoom_min: minimum zoom 507 zoom_max: maximum zoom 508 zoom_step_size: size of zoom steps 509 """ 510 # Determine test zoom range 511 logging.debug('z_range = %s', str(zoom_range)) 512 zoom_min, zoom_max = float(zoom_range[0]), float(zoom_range[1]) 513 zoom_max = min(zoom_max, ZOOM_MAX_THRESH * zoom_min) 514 515 zoom_step_size = (zoom_max-zoom_min) / (steps-1) 516 logging.debug('zoomRatioRange = %s z_min = %f z_max = %f z_stepSize = %f', 517 str(zoom_range), zoom_min, zoom_max, zoom_step_size) 518 519 return zoom_min, zoom_max, zoom_step_size 520 521 522def plot_variation(frame_numbers, variations, tolerances, plot_name, ylabel): 523 """Plots a variation against frame numbers with corresponding tolerances. 524 525 Args: 526 frame_numbers: List of frame numbers. 527 variations: List of variations. 528 tolerances: List of tolerances corresponding to each variation. 529 plot_name: Name for the plot file. 530 ylabel: Label for the y-axis. 531 """ 532 533 plt.figure(figsize=(40, 10)) 534 535 plt.scatter(frame_numbers, variations, marker='o', linestyle='-', 536 color='blue', label=ylabel) 537 538 if tolerances: 539 plt.plot(frame_numbers, tolerances, linestyle='--', color='red', 540 label='Tolerance') 541 542 plt.xlabel('Frame Number', fontsize=12) 543 plt.ylabel(ylabel, fontsize=12) 544 plt.title(f'{ylabel} vs. Frame Number', fontsize=14) 545 546 plt.legend() 547 548 plt.grid(axis='y', linestyle='--') 549 plt.savefig(plot_name) 550 plt.close() 551 552