1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import os 16import unittest 17 18import cv2 19import its.caps 20import its.device 21import its.error 22import its.image 23import numpy 24 25VGA_HEIGHT = 480 26VGA_WIDTH = 640 27 28 29def scale_img(img, scale=1.0): 30 """Scale and image based on a real number scale factor.""" 31 dim = (int(img.shape[1]*scale), int(img.shape[0]*scale)) 32 return cv2.resize(img.copy(), dim, interpolation=cv2.INTER_AREA) 33 34 35def gray_scale_img(img): 36 """Return gray scale version of image.""" 37 if len(img.shape) == 2: 38 img_gray = img.copy() 39 elif len(img.shape) == 3: 40 if img.shape[2] == 1: 41 img_gray = img[:, :, 0].copy() 42 else: 43 img_gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) 44 return img_gray 45 46 47class Chart(object): 48 """Definition for chart object. 49 50 Defines PNG reference file, chart size and distance, and scaling range. 51 """ 52 53 def __init__(self, chart_file, height, distance, scale_start, scale_stop, 54 scale_step): 55 """Initial constructor for class. 56 57 Args: 58 chart_file: str; absolute path to png file of chart 59 height: float; height in cm of displayed chart 60 distance: float; distance in cm from camera of displayed chart 61 scale_start: float; start value for scaling for chart search 62 scale_stop: float; stop value for scaling for chart search 63 scale_step: float; step value for scaling for chart search 64 """ 65 self._file = chart_file 66 self._height = height 67 self._distance = distance 68 self._scale_start = scale_start 69 self._scale_stop = scale_stop 70 self._scale_step = scale_step 71 self.xnorm, self.ynorm, self.wnorm, self.hnorm, self.scale = its.image.chart_located_per_argv() 72 if not self.xnorm: 73 with its.device.ItsSession() as cam: 74 props = cam.get_camera_properties() 75 if its.caps.read_3a(props): 76 self.locate(cam, props) 77 else: 78 print 'Chart locator skipped.' 79 self._set_scale_factors_to_one() 80 81 def _set_scale_factors_to_one(self): 82 """Set scale factors to 1.0 for skipped tests.""" 83 self.wnorm = 1.0 84 self.hnorm = 1.0 85 self.xnorm = 0.0 86 self.ynorm = 0.0 87 self.scale = 1.0 88 89 def _calc_scale_factors(self, cam, props, fmt, s, e, fd): 90 """Take an image with s, e, & fd to find the chart location. 91 92 Args: 93 cam: An open device session. 94 props: Properties of cam 95 fmt: Image format for the capture 96 s: Sensitivity for the AF request as defined in 97 android.sensor.sensitivity 98 e: Exposure time for the AF request as defined in 99 android.sensor.exposureTime 100 fd: float; autofocus lens position 101 Returns: 102 template: numpy array; chart template for locator 103 img_3a: numpy array; RGB image for chart location 104 scale_factor: float; scaling factor for chart search 105 """ 106 req = its.objects.manual_capture_request(s, e) 107 req['android.lens.focusDistance'] = fd 108 cap_chart = its.image.stationary_lens_cap(cam, req, fmt) 109 img_3a = its.image.convert_capture_to_rgb_image(cap_chart, props) 110 img_3a = its.image.rotate_img_per_argv(img_3a) 111 its.image.write_image(img_3a, 'af_scene.jpg') 112 template = cv2.imread(self._file, cv2.IMREAD_ANYDEPTH) 113 focal_l = cap_chart['metadata']['android.lens.focalLength'] 114 pixel_pitch = (props['android.sensor.info.physicalSize']['height'] / 115 img_3a.shape[0]) 116 print ' Chart distance: %.2fcm' % self._distance 117 print ' Chart height: %.2fcm' % self._height 118 print ' Focal length: %.2fmm' % focal_l 119 print ' Pixel pitch: %.2fum' % (pixel_pitch*1E3) 120 print ' Template height: %dpixels' % template.shape[0] 121 chart_pixel_h = self._height * focal_l / (self._distance * pixel_pitch) 122 scale_factor = template.shape[0] / chart_pixel_h 123 print 'Chart/image scale factor = %.2f' % scale_factor 124 return template, img_3a, scale_factor 125 126 def locate(self, cam, props): 127 """Find the chart in the image, and append location to chart object. 128 129 The values appended are: 130 xnorm: float; [0, 1] left loc of chart in scene 131 ynorm: float; [0, 1] top loc of chart in scene 132 wnorm: float; [0, 1] width of chart in scene 133 hnorm: float; [0, 1] height of chart in scene 134 scale: float; scale factor to extract chart 135 136 Args: 137 cam: An open device session 138 props: Camera properties 139 """ 140 if its.caps.read_3a(props): 141 s, e, _, _, fd = cam.do_3a(get_results=True) 142 fmt = {'format': 'yuv', 'width': VGA_WIDTH, 'height': VGA_HEIGHT} 143 chart, scene, s_factor = self._calc_scale_factors(cam, props, fmt, 144 s, e, fd) 145 else: 146 print 'Chart locator skipped.' 147 self._set_scale_factors_to_one() 148 return 149 scale_start = self._scale_start * s_factor 150 scale_stop = self._scale_stop * s_factor 151 scale_step = self._scale_step * s_factor 152 self.scale = s_factor 153 max_match = [] 154 # check for normalized image 155 if numpy.amax(scene) <= 1.0: 156 scene = (scene * 255.0).astype(numpy.uint8) 157 scene_gray = gray_scale_img(scene) 158 print 'Finding chart in scene...' 159 for scale in numpy.arange(scale_start, scale_stop, scale_step): 160 scene_scaled = scale_img(scene_gray, scale) 161 result = cv2.matchTemplate(scene_scaled, chart, cv2.TM_CCOEFF) 162 _, opt_val, _, top_left_scaled = cv2.minMaxLoc(result) 163 # print out scale and match 164 print ' scale factor: %.3f, opt val: %.f' % (scale, opt_val) 165 max_match.append((opt_val, top_left_scaled)) 166 167 # determine if optimization results are valid 168 opt_values = [x[0] for x in max_match] 169 if 2.0*min(opt_values) > max(opt_values): 170 estring = ('Warning: unable to find chart in scene!\n' 171 'Check camera distance and self-reported ' 172 'pixel pitch, focal length and hyperfocal distance.') 173 print estring 174 self._set_scale_factors_to_one() 175 else: 176 if (max(opt_values) == opt_values[0] or 177 max(opt_values) == opt_values[len(opt_values)-1]): 178 estring = ('Warning: chart is at extreme range of locator ' 179 'check.\n') 180 print estring 181 # find max and draw bbox 182 match_index = max_match.index(max(max_match, key=lambda x: x[0])) 183 self.scale = scale_start + scale_step * match_index 184 print 'Optimum scale factor: %.3f' % self.scale 185 top_left_scaled = max_match[match_index][1] 186 h, w = chart.shape 187 bottom_right_scaled = (top_left_scaled[0] + w, 188 top_left_scaled[1] + h) 189 top_left = (int(top_left_scaled[0]/self.scale), 190 int(top_left_scaled[1]/self.scale)) 191 bottom_right = (int(bottom_right_scaled[0]/self.scale), 192 int(bottom_right_scaled[1]/self.scale)) 193 self.wnorm = float((bottom_right[0]) - top_left[0]) / scene.shape[1] 194 self.hnorm = float((bottom_right[1]) - top_left[1]) / scene.shape[0] 195 self.xnorm = float(top_left[0]) / scene.shape[1] 196 self.ynorm = float(top_left[1]) / scene.shape[0] 197 198 199def get_angle(input_img): 200 """Computes anglular inclination of chessboard in input_img. 201 202 Angle estimation algoritm description: 203 Input: 2D grayscale image of chessboard. 204 Output: Angle of rotation of chessboard perpendicular to 205 chessboard. Assumes chessboard and camera are parallel to 206 each other. 207 208 1) Use adaptive threshold to make image binary 209 2) Find countours 210 3) Filter out small contours 211 4) Filter out all non-square contours 212 5) Compute most common square shape. 213 The assumption here is that the most common square instances 214 are the chessboard squares. We've shown that with our current 215 tuning, we can robustly identify the squares on the sensor fusion 216 chessboard. 217 6) Return median angle of most common square shape. 218 219 USAGE NOTE: This function has been tuned to work for the chessboard used in 220 the sensor_fusion tests. See images in test_images/rotated_chessboard/ for 221 sample captures. If this function is used with other chessboards, it may not 222 work as expected. 223 224 TODO: Make algorithm more robust so it works on any type of 225 chessboard. 226 227 Args: 228 input_img (2D numpy.ndarray): Grayscale image stored as a 2D 229 numpy array. 230 231 Returns: 232 Median angle of squares in degrees identified in the image. 233 """ 234 # Tuning parameters 235 min_square_area = (float)(input_img.shape[1] * 0.05) 236 237 # Creates copy of image to avoid modifying original. 238 img = numpy.array(input_img, copy=True) 239 240 # Scale pixel values from 0-1 to 0-255 241 img *= 255 242 img = img.astype(numpy.uint8) 243 244 thresh = cv2.adaptiveThreshold( 245 img, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 201, 2) 246 247 # Find all contours 248 contours = [] 249 cv2_version = cv2.__version__ 250 if cv2_version.startswith('2.4.'): 251 contours, _ = cv2.findContours( 252 thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) 253 elif cv2_version.startswith('3.2.'): 254 _, contours, _ = cv2.findContours( 255 thresh, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) 256 257 # Filter contours to squares only. 258 square_contours = [] 259 260 for contour in contours: 261 rect = cv2.minAreaRect(contour) 262 _, (width, height), angle = rect 263 264 # Skip non-squares (with 0.1 tolerance) 265 tolerance = 0.1 266 if width < height * (1 - tolerance) or width > height * (1 + tolerance): 267 continue 268 269 # Remove very small contours. 270 # These are usually just tiny dots due to noise. 271 area = cv2.contourArea(contour) 272 if area < min_square_area: 273 continue 274 275 if cv2_version.startswith('2.4.'): 276 box = numpy.int0(cv2.cv.BoxPoints(rect)) 277 elif cv2_version.startswith('3.2.'): 278 box = numpy.int0(cv2.boxPoints(rect)) 279 square_contours.append(contour) 280 281 areas = [] 282 for contour in square_contours: 283 area = cv2.contourArea(contour) 284 areas.append(area) 285 286 median_area = numpy.median(areas) 287 288 filtered_squares = [] 289 filtered_angles = [] 290 for square in square_contours: 291 area = cv2.contourArea(square) 292 if area < median_area * 0.90 or area > median_area * 1.10: 293 continue 294 295 filtered_squares.append(square) 296 _, (width, height), angle = cv2.minAreaRect(square) 297 filtered_angles.append(angle) 298 299 if len(filtered_angles) < 10: 300 return None 301 302 return numpy.median(filtered_angles) 303 304 305class __UnitTest(unittest.TestCase): 306 """Run a suite of unit tests on this module. 307 """ 308 309 def test_compute_image_sharpness(self): 310 """Unit test for compute_img_sharpness. 311 312 Test by using PNG of ISO12233 chart and blurring intentionally. 313 'sharpness' should drop off by sqrt(2) for 2x blur of image. 314 315 We do one level of blur as PNG image is not perfect. 316 """ 317 yuv_full_scale = 1023.0 318 chart_file = os.path.join(os.environ['CAMERA_ITS_TOP'], 'pymodules', 319 'its', 'test_images', 'ISO12233.png') 320 chart = cv2.imread(chart_file, cv2.IMREAD_ANYDEPTH) 321 white_level = numpy.amax(chart).astype(float) 322 sharpness = {} 323 for j in [2, 4, 8]: 324 blur = cv2.blur(chart, (j, j)) 325 blur = blur[:, :, numpy.newaxis] 326 sharpness[j] = (yuv_full_scale * 327 its.image.compute_image_sharpness(blur / 328 white_level)) 329 self.assertTrue(numpy.isclose(sharpness[2]/sharpness[4], 330 numpy.sqrt(2), atol=0.1)) 331 self.assertTrue(numpy.isclose(sharpness[4]/sharpness[8], 332 numpy.sqrt(2), atol=0.1)) 333 334 def test_get_angle_identify_unrotated_chessboard_angle(self): 335 basedir = os.path.join( 336 os.path.dirname(__file__), 'test_images/rotated_chessboards/') 337 338 normal_img_path = os.path.join(basedir, 'normal.jpg') 339 wide_img_path = os.path.join(basedir, 'wide.jpg') 340 341 normal_img = cv2.cvtColor( 342 cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY) 343 wide_img = cv2.cvtColor( 344 cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY) 345 346 assert get_angle(normal_img) == 0 347 assert get_angle(wide_img) == 0 348 349 def test_get_angle_identify_rotated_chessboard_angle(self): 350 basedir = os.path.join( 351 os.path.dirname(__file__), 'test_images/rotated_chessboards/') 352 353 # Array of the image files and angles containing rotated chessboards. 354 test_cases = [ 355 ('_15_ccw', 15), 356 ('_30_ccw', 30), 357 ('_45_ccw', 45), 358 ('_60_ccw', 60), 359 ('_75_ccw', 75), 360 ('_90_ccw', 90) 361 ] 362 363 # For each rotated image pair (normal, wide). Check if angle is 364 # identified as expected. 365 for suffix, angle in test_cases: 366 # Define image paths 367 normal_img_path = os.path.join( 368 basedir, 'normal{}.jpg'.format(suffix)) 369 wide_img_path = os.path.join( 370 basedir, 'wide{}.jpg'.format(suffix)) 371 372 # Load and color convert images 373 normal_img = cv2.cvtColor( 374 cv2.imread(normal_img_path), cv2.COLOR_BGR2GRAY) 375 wide_img = cv2.cvtColor( 376 cv2.imread(wide_img_path), cv2.COLOR_BGR2GRAY) 377 378 # Assert angle is as expected up to 2.0 degrees of accuracy. 379 assert numpy.isclose( 380 abs(get_angle(normal_img)), angle, 2.0) 381 assert numpy.isclose( 382 abs(get_angle(wide_img)), angle, 2.0) 383 384 385if __name__ == '__main__': 386 unittest.main() 387