1# Copyright 2013 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Image processing utility functions.""" 15 16 17import copy 18import io 19import logging 20import math 21import os 22import random 23import sys 24import unittest 25 26import numpy 27from PIL import Image 28 29 30import cv2 31import capture_request_utils 32import error_util 33 34# The matrix is from JFIF spec 35DEFAULT_YUV_TO_RGB_CCM = numpy.matrix([[1.000, 0.000, 1.402], 36 [1.000, -0.344, -0.714], 37 [1.000, 1.772, 0.000]]) 38 39DEFAULT_YUV_OFFSETS = numpy.array([0, 128, 128]) 40MAX_LUT_SIZE = 65536 41DEFAULT_GAMMA_LUT = numpy.array([ 42 math.floor((MAX_LUT_SIZE-1) * math.pow(i/(MAX_LUT_SIZE-1), 1/2.2) + 0.5) 43 for i in range(MAX_LUT_SIZE)]) 44NUM_TRIES = 2 45NUM_FRAMES = 4 46TEST_IMG_DIR = os.path.join(os.environ['CAMERA_ITS_TOP'], 'test_images') 47 48 49# pylint: disable=unused-argument 50def convert_capture_to_rgb_image(cap, 51 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 52 yuv_off=DEFAULT_YUV_OFFSETS, 53 props=None): 54 """Convert a captured image object to a RGB image. 55 56 Args: 57 cap: A capture object as returned by its_session_utils.do_capture. 58 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 59 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 60 props: (Optional) camera properties object (of static values); 61 required for processing raw images. 62 63 Returns: 64 RGB float-3 image array, with pixel values in [0.0, 1.0]. 65 """ 66 w = cap['width'] 67 h = cap['height'] 68 if cap['format'] == 'raw10': 69 assert props is not None 70 cap = unpack_raw10_capture(cap) 71 72 if cap['format'] == 'raw12': 73 assert props is not None 74 cap = unpack_raw12_capture(cap) 75 76 if cap['format'] == 'yuv': 77 y = cap['data'][0: w * h] 78 u = cap['data'][w * h: w * h * 5//4] 79 v = cap['data'][w * h * 5//4: w * h * 6//4] 80 return convert_yuv420_planar_to_rgb_image(y, u, v, w, h) 81 elif cap['format'] == 'jpeg': 82 return decompress_jpeg_to_rgb_image(cap['data']) 83 elif cap['format'] == 'raw' or cap['format'] == 'rawStats': 84 assert props is not None 85 r, gr, gb, b = convert_capture_to_planes(cap, props) 86 return convert_raw_to_rgb_image(r, gr, gb, b, props, cap['metadata']) 87 elif cap['format'] == 'y8': 88 y = cap['data'][0: w * h] 89 return convert_y8_to_rgb_image(y, w, h) 90 else: 91 raise error_util.CameraItsError('Invalid format %s' % (cap['format'])) 92 93 94def unpack_raw10_capture(cap): 95 """Unpack a raw-10 capture to a raw-16 capture. 96 97 Args: 98 cap: A raw-10 capture object. 99 100 Returns: 101 New capture object with raw-16 data. 102 """ 103 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 104 # the MSBs of the pixels, and the 5th byte holding 4x2b LSBs. 105 w, h = cap['width'], cap['height'] 106 if w % 4 != 0: 107 raise error_util.CameraItsError('Invalid raw-10 buffer width') 108 cap = copy.deepcopy(cap) 109 cap['data'] = unpack_raw10_image(cap['data'].reshape(h, w * 5 // 4)) 110 cap['format'] = 'raw' 111 return cap 112 113 114def unpack_raw10_image(img): 115 """Unpack a raw-10 image to a raw-16 image. 116 117 Output image will have the 10 LSBs filled in each 16b word, and the 6 MSBs 118 will be set to zero. 119 120 Args: 121 img: A raw-10 image, as a uint8 numpy array. 122 123 Returns: 124 Image as a uint16 numpy array, with all row padding stripped. 125 """ 126 if img.shape[1] % 5 != 0: 127 raise error_util.CameraItsError('Invalid raw-10 buffer width') 128 w = img.shape[1] * 4 // 5 129 h = img.shape[0] 130 # Cut out the 4x8b MSBs and shift to bits [9:2] in 16b words. 131 msbs = numpy.delete(img, numpy.s_[4::5], 1) 132 msbs = msbs.astype(numpy.uint16) 133 msbs = numpy.left_shift(msbs, 2) 134 msbs = msbs.reshape(h, w) 135 # Cut out the 4x2b LSBs and put each in bits [1:0] of their own 8b words. 136 lsbs = img[::, 4::5].reshape(h, w // 4) 137 lsbs = numpy.right_shift( 138 numpy.packbits(numpy.unpackbits(lsbs).reshape(h, w // 4, 4, 2), 3), 6) 139 # Pair the LSB bits group to 0th pixel instead of 3rd pixel 140 lsbs = lsbs.reshape(h, w // 4, 4)[:, :, ::-1] 141 lsbs = lsbs.reshape(h, w) 142 # Fuse the MSBs and LSBs back together 143 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h, w) 144 return img16 145 146 147def unpack_raw12_capture(cap): 148 """Unpack a raw-12 capture to a raw-16 capture. 149 150 Args: 151 cap: A raw-12 capture object. 152 153 Returns: 154 New capture object with raw-16 data. 155 """ 156 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 157 # the MSBs of the pixels, and the 5th byte holding 4x2b LSBs. 158 w, h = cap['width'], cap['height'] 159 if w % 2 != 0: 160 raise error_util.CameraItsError('Invalid raw-12 buffer width') 161 cap = copy.deepcopy(cap) 162 cap['data'] = unpack_raw12_image(cap['data'].reshape(h, w * 3 // 2)) 163 cap['format'] = 'raw' 164 return cap 165 166 167def unpack_raw12_image(img): 168 """Unpack a raw-12 image to a raw-16 image. 169 170 Output image will have the 12 LSBs filled in each 16b word, and the 4 MSBs 171 will be set to zero. 172 173 Args: 174 img: A raw-12 image, as a uint8 numpy array. 175 176 Returns: 177 Image as a uint16 numpy array, with all row padding stripped. 178 """ 179 if img.shape[1] % 3 != 0: 180 raise error_util.CameraItsError('Invalid raw-12 buffer width') 181 w = img.shape[1] * 2 // 3 182 h = img.shape[0] 183 # Cut out the 2x8b MSBs and shift to bits [11:4] in 16b words. 184 msbs = numpy.delete(img, numpy.s_[2::3], 1) 185 msbs = msbs.astype(numpy.uint16) 186 msbs = numpy.left_shift(msbs, 4) 187 msbs = msbs.reshape(h, w) 188 # Cut out the 2x4b LSBs and put each in bits [3:0] of their own 8b words. 189 lsbs = img[::, 2::3].reshape(h, w // 2) 190 lsbs = numpy.right_shift( 191 numpy.packbits(numpy.unpackbits(lsbs).reshape(h, w // 2, 2, 4), 3), 4) 192 # Pair the LSB bits group to pixel 0 instead of pixel 1 193 lsbs = lsbs.reshape(h, w // 2, 2)[:, :, ::-1] 194 lsbs = lsbs.reshape(h, w) 195 # Fuse the MSBs and LSBs back together 196 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h, w) 197 return img16 198 199 200def convert_yuv420_planar_to_rgb_image(y_plane, u_plane, v_plane, 201 w, h, 202 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 203 yuv_off=DEFAULT_YUV_OFFSETS): 204 """Convert a YUV420 8-bit planar image to an RGB image. 205 206 Args: 207 y_plane: The packed 8-bit Y plane. 208 u_plane: The packed 8-bit U plane. 209 v_plane: The packed 8-bit V plane. 210 w: The width of the image. 211 h: The height of the image. 212 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 213 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 214 215 Returns: 216 RGB float-3 image array, with pixel values in [0.0, 1.0]. 217 """ 218 y = numpy.subtract(y_plane, yuv_off[0]) 219 u = numpy.subtract(u_plane, yuv_off[1]).view(numpy.int8) 220 v = numpy.subtract(v_plane, yuv_off[2]).view(numpy.int8) 221 u = u.reshape(h // 2, w // 2).repeat(2, axis=1).repeat(2, axis=0) 222 v = v.reshape(h // 2, w // 2).repeat(2, axis=1).repeat(2, axis=0) 223 yuv = numpy.dstack([y, u.reshape(w * h), v.reshape(w * h)]) 224 flt = numpy.empty([h, w, 3], dtype=numpy.float32) 225 flt.reshape(w * h * 3)[:] = yuv.reshape(h * w * 3)[:] 226 flt = numpy.dot(flt.reshape(w * h, 3), ccm_yuv_to_rgb.T).clip(0, 255) 227 rgb = numpy.empty([h, w, 3], dtype=numpy.uint8) 228 rgb.reshape(w * h * 3)[:] = flt.reshape(w * h * 3)[:] 229 return rgb.astype(numpy.float32) / 255.0 230 231 232def decompress_jpeg_to_rgb_image(jpeg_buffer): 233 """Decompress a JPEG-compressed image, returning as an RGB image. 234 235 Args: 236 jpeg_buffer: The JPEG stream. 237 238 Returns: 239 A numpy array for the RGB image, with pixels in [0,1]. 240 """ 241 img = Image.open(io.BytesIO(jpeg_buffer)) 242 w = img.size[0] 243 h = img.size[1] 244 return numpy.array(img).reshape(h, w, 3) / 255.0 245 246 247def convert_capture_to_planes(cap, props=None): 248 """Convert a captured image object to separate image planes. 249 250 Decompose an image into multiple images, corresponding to different planes. 251 252 For YUV420 captures ("yuv"): 253 Returns Y,U,V planes, where the Y plane is full-res and the U,V planes 254 are each 1/2 x 1/2 of the full res. 255 256 For Bayer captures ("raw", "raw10", "raw12", or "rawStats"): 257 Returns planes in the order R,Gr,Gb,B, regardless of the Bayer pattern 258 layout. For full-res raw images ("raw", "raw10", "raw12"), each plane 259 is 1/2 x 1/2 of the full res. For "rawStats" images, the mean image 260 is returned. 261 262 For JPEG captures ("jpeg"): 263 Returns R,G,B full-res planes. 264 265 Args: 266 cap: A capture object as returned by its_session_utils.do_capture. 267 props: (Optional) camera properties object (of static values); 268 required for processing raw images. 269 270 Returns: 271 A tuple of float numpy arrays (one per plane), consisting of pixel values 272 in the range [0.0, 1.0]. 273 """ 274 w = cap['width'] 275 h = cap['height'] 276 if cap['format'] == 'raw10': 277 assert props is not None 278 cap = unpack_raw10_capture(cap) 279 if cap['format'] == 'raw12': 280 assert props is not None 281 cap = unpack_raw12_capture(cap) 282 if cap['format'] == 'yuv': 283 y = cap['data'][0:w * h] 284 u = cap['data'][w * h:w * h * 5 // 4] 285 v = cap['data'][w * h * 5 // 4:w * h * 6 // 4] 286 return ((y.astype(numpy.float32) / 255.0).reshape(h, w, 1), 287 (u.astype(numpy.float32) / 255.0).reshape(h // 2, w // 2, 1), 288 (v.astype(numpy.float32) / 255.0).reshape(h // 2, w // 2, 1)) 289 elif cap['format'] == 'jpeg': 290 rgb = decompress_jpeg_to_rgb_image(cap['data']).reshape(w * h * 3) 291 return (rgb[::3].reshape(h, w, 1), rgb[1::3].reshape(h, w, 1), 292 rgb[2::3].reshape(h, w, 1)) 293 elif cap['format'] == 'raw': 294 assert props is not None 295 white_level = float(props['android.sensor.info.whiteLevel']) 296 img = numpy.ndarray( 297 shape=(h * w,), dtype='<u2', buffer=cap['data'][0:w * h * 2]) 298 img = img.astype(numpy.float32).reshape(h, w) / white_level 299 # Crop the raw image to the active array region. 300 if (props.get('android.sensor.info.preCorrectionActiveArraySize') is 301 not None and 302 props.get('android.sensor.info.pixelArraySize') is not None): 303 # Note that the Rect class is defined such that the left,top values 304 # are "inside" while the right,bottom values are "outside"; that is, 305 # it's inclusive of the top,left sides only. So, the width is 306 # computed as right-left, rather than right-left+1, etc. 307 wfull = props['android.sensor.info.pixelArraySize']['width'] 308 hfull = props['android.sensor.info.pixelArraySize']['height'] 309 xcrop = props['android.sensor.info.preCorrectionActiveArraySize']['left'] 310 ycrop = props['android.sensor.info.preCorrectionActiveArraySize']['top'] 311 wcrop = props['android.sensor.info.preCorrectionActiveArraySize'][ 312 'right'] - xcrop 313 hcrop = props['android.sensor.info.preCorrectionActiveArraySize'][ 314 'bottom'] - ycrop 315 assert wfull >= wcrop >= 0 316 assert hfull >= hcrop >= 0 317 assert wfull - wcrop >= xcrop >= 0 318 assert hfull - hcrop >= ycrop >= 0 319 if w == wfull and h == hfull: 320 # Crop needed; extract the center region. 321 img = img[ycrop:ycrop + hcrop, xcrop:xcrop + wcrop] 322 w = wcrop 323 h = hcrop 324 elif w == wcrop and h == hcrop: 325 logging.debug('Image is already cropped.No cropping needed.') 326 # pylint: disable=pointless-statement 327 None 328 else: 329 raise error_util.CameraItsError('Invalid image size metadata') 330 # Separate the image planes. 331 imgs = [ 332 img[::2].reshape(w * h // 2)[::2].reshape(h // 2, w // 2, 1), 333 img[::2].reshape(w * h // 2)[1::2].reshape(h // 2, w // 2, 1), 334 img[1::2].reshape(w * h // 2)[::2].reshape(h // 2, w // 2, 1), 335 img[1::2].reshape(w * h // 2)[1::2].reshape(h // 2, w // 2, 1) 336 ] 337 idxs = get_canonical_cfa_order(props) 338 return [imgs[i] for i in idxs] 339 elif cap['format'] == 'rawStats': 340 assert props is not None 341 white_level = float(props['android.sensor.info.whiteLevel']) 342 # pylint: disable=unused-variable 343 mean_image, var_image = unpack_rawstats_capture(cap) 344 idxs = get_canonical_cfa_order(props) 345 return [mean_image[:, :, i] / white_level for i in idxs] 346 else: 347 raise error_util.CameraItsError('Invalid format %s' % (cap['format'])) 348 349 350def convert_raw_to_rgb_image(r_plane, gr_plane, gb_plane, b_plane, props, 351 cap_res): 352 """Convert a Bayer raw-16 image to an RGB image. 353 354 Includes some extremely rudimentary demosaicking and color processing 355 operations; the output of this function shouldn't be used for any image 356 quality analysis. 357 358 Args: 359 r_plane: 360 gr_plane: 361 gb_plane: 362 b_plane: Numpy arrays for each color plane 363 in the Bayer image, with pixels in the [0.0, 1.0] range. 364 props: Camera properties object. 365 cap_res: Capture result (metadata) object. 366 367 Returns: 368 RGB float-3 image array, with pixel values in [0.0, 1.0] 369 """ 370 # Values required for the RAW to RGB conversion. 371 assert props is not None 372 white_level = float(props['android.sensor.info.whiteLevel']) 373 black_levels = props['android.sensor.blackLevelPattern'] 374 gains = cap_res['android.colorCorrection.gains'] 375 ccm = cap_res['android.colorCorrection.transform'] 376 377 # Reorder black levels and gains to R,Gr,Gb,B, to match the order 378 # of the planes. 379 black_levels = [get_black_level(i, props, cap_res) for i in range(4)] 380 gains = get_gains_in_canonical_order(props, gains) 381 382 # Convert CCM from rational to float, as numpy arrays. 383 ccm = numpy.array(capture_request_utils.rational_to_float(ccm)).reshape(3, 3) 384 385 # Need to scale the image back to the full [0,1] range after subtracting 386 # the black level from each pixel. 387 scale = white_level / (white_level - max(black_levels)) 388 389 # Three-channel black levels, normalized to [0,1] by white_level. 390 black_levels = numpy.array( 391 [b / white_level for b in [black_levels[i] for i in [0, 1, 3]]]) 392 393 # Three-channel gains. 394 gains = numpy.array([gains[i] for i in [0, 1, 3]]) 395 396 h, w = r_plane.shape[:2] 397 img = numpy.dstack([r_plane, (gr_plane + gb_plane) / 2.0, b_plane]) 398 img = (((img.reshape(h, w, 3) - black_levels) * scale) * gains).clip(0.0, 1.0) 399 img = numpy.dot(img.reshape(w * h, 3), ccm.T).reshape(h, w, 3).clip(0.0, 1.0) 400 return img 401 402 403def convert_y8_to_rgb_image(y_plane, w, h): 404 """Convert a Y 8-bit image to an RGB image. 405 406 Args: 407 y_plane: The packed 8-bit Y plane. 408 w: The width of the image. 409 h: The height of the image. 410 411 Returns: 412 RGB float-3 image array, with pixel values in [0.0, 1.0]. 413 """ 414 y3 = numpy.dstack([y_plane, y_plane, y_plane]) 415 rgb = numpy.empty([h, w, 3], dtype=numpy.uint8) 416 rgb.reshape(w * h * 3)[:] = y3.reshape(w * h * 3)[:] 417 return rgb.astype(numpy.float32) / 255.0 418 419 420def write_image(img, fname, apply_gamma=False): 421 """Save a float-3 numpy array image to a file. 422 423 Supported formats: PNG, JPEG, and others; see PIL docs for more. 424 425 Image can be 3-channel, which is interpreted as RGB, or can be 1-channel, 426 which is greyscale. 427 428 Can optionally specify that the image should be gamma-encoded prior to 429 writing it out; this should be done if the image contains linear pixel 430 values, to make the image look "normal". 431 432 Args: 433 img: Numpy image array data. 434 fname: Path of file to save to; the extension specifies the format. 435 apply_gamma: (Optional) apply gamma to the image prior to writing it. 436 """ 437 if apply_gamma: 438 img = apply_lut_to_image(img, DEFAULT_GAMMA_LUT) 439 (h, w, chans) = img.shape 440 if chans == 3: 441 Image.fromarray((img * 255.0).astype(numpy.uint8), 'RGB').save(fname) 442 elif chans == 1: 443 img3 = (img * 255.0).astype(numpy.uint8).repeat(3).reshape(h, w, 3) 444 Image.fromarray(img3, 'RGB').save(fname) 445 else: 446 raise error_util.CameraItsError('Unsupported image type') 447 448 449def read_image(fname): 450 """Read image function to match write_image() above.""" 451 return Image.open(fname) 452 453 454def apply_lut_to_image(img, lut): 455 """Applies a LUT to every pixel in a float image array. 456 457 Internally converts to a 16b integer image, since the LUT can work with up 458 to 16b->16b mappings (i.e. values in the range [0,65535]). The lut can also 459 have fewer than 65536 entries, however it must be sized as a power of 2 460 (and for smaller luts, the scale must match the bitdepth). 461 462 For a 16b lut of 65536 entries, the operation performed is: 463 464 lut[r * 65535] / 65535 -> r' 465 lut[g * 65535] / 65535 -> g' 466 lut[b * 65535] / 65535 -> b' 467 468 For a 10b lut of 1024 entries, the operation becomes: 469 470 lut[r * 1023] / 1023 -> r' 471 lut[g * 1023] / 1023 -> g' 472 lut[b * 1023] / 1023 -> b' 473 474 Args: 475 img: Numpy float image array, with pixel values in [0,1]. 476 lut: Numpy table encoding a LUT, mapping 16b integer values. 477 478 Returns: 479 Float image array after applying LUT to each pixel. 480 """ 481 n = len(lut) 482 if n <= 0 or n > MAX_LUT_SIZE or (n & (n - 1)) != 0: 483 raise error_util.CameraItsError('Invalid arg LUT size: %d' % (n)) 484 m = float(n - 1) 485 return (lut[(img * m).astype(numpy.uint16)] / m).astype(numpy.float32) 486 487 488def get_gains_in_canonical_order(props, gains): 489 """Reorders the gains tuple to the canonical R,Gr,Gb,B order. 490 491 Args: 492 props: Camera properties object. 493 gains: List of 4 values, in R,G_even,G_odd,B order. 494 495 Returns: 496 List of gains values, in R,Gr,Gb,B order. 497 """ 498 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 499 if cfa_pat in [0, 1]: 500 # RGGB or GRBG, so G_even is Gr 501 return gains 502 elif cfa_pat in [2, 3]: 503 # GBRG or BGGR, so G_even is Gb 504 return [gains[0], gains[2], gains[1], gains[3]] 505 else: 506 raise error_util.CameraItsError('Not supported') 507 508 509def get_black_level(chan, props, cap_res=None): 510 """Return the black level to use for a given capture. 511 512 Uses a dynamic value from the capture result if available, else falls back 513 to the static global value in the camera characteristics. 514 515 Args: 516 chan: The channel index, in canonical order (R, Gr, Gb, B). 517 props: The camera properties object. 518 cap_res: A capture result object. 519 520 Returns: 521 The black level value for the specified channel. 522 """ 523 if (cap_res is not None and 524 'android.sensor.dynamicBlackLevel' in cap_res and 525 cap_res['android.sensor.dynamicBlackLevel'] is not None): 526 black_levels = cap_res['android.sensor.dynamicBlackLevel'] 527 else: 528 black_levels = props['android.sensor.blackLevelPattern'] 529 idxs = get_canonical_cfa_order(props) 530 ordered_black_levels = [black_levels[i] for i in idxs] 531 return ordered_black_levels[chan] 532 533 534def get_canonical_cfa_order(props): 535 """Returns a mapping to the standard order R,Gr,Gb,B. 536 537 Returns a mapping from the Bayer 2x2 top-left grid in the CFA to the standard 538 order R,Gr,Gb,B. 539 540 Args: 541 props: Camera properties object. 542 543 Returns: 544 List of 4 integers, corresponding to the positions in the 2x2 top- 545 left Bayer grid of R,Gr,Gb,B, where the 2x2 grid is labeled as 546 0,1,2,3 in row major order. 547 """ 548 # Note that raw streams aren't croppable, so the cropRegion doesn't need 549 # to be considered when determining the top-left pixel color. 550 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 551 if cfa_pat == 0: 552 # RGGB 553 return [0, 1, 2, 3] 554 elif cfa_pat == 1: 555 # GRBG 556 return [1, 0, 3, 2] 557 elif cfa_pat == 2: 558 # GBRG 559 return [2, 3, 0, 1] 560 elif cfa_pat == 3: 561 # BGGR 562 return [3, 2, 1, 0] 563 else: 564 raise error_util.CameraItsError('Not supported') 565 566 567def unpack_rawstats_capture(cap): 568 """Unpack a rawStats capture to the mean and variance images. 569 570 Args: 571 cap: A capture object as returned by its_session_utils.do_capture. 572 573 Returns: 574 Tuple (mean_image var_image) of float-4 images, with non-normalized 575 pixel values computed from the RAW16 images on the device 576 """ 577 assert cap['format'] == 'rawStats' 578 w = cap['width'] 579 h = cap['height'] 580 img = numpy.ndarray(shape=(2 * h * w * 4,), dtype='<f', buffer=cap['data']) 581 analysis_image = img.reshape((2, h, w, 4)) 582 mean_image = analysis_image[0, :, :, :].reshape(h, w, 4) 583 var_image = analysis_image[1, :, :, :].reshape(h, w, 4) 584 return mean_image, var_image 585 586 587def get_image_patch(img, xnorm, ynorm, wnorm, hnorm): 588 """Get a patch (tile) of an image. 589 590 Args: 591 img: Numpy float image array, with pixel values in [0,1]. 592 xnorm: 593 ynorm: 594 wnorm: 595 hnorm: Normalized (in [0,1]) coords for the tile. 596 597 Returns: 598 Numpy float image array of the patch. 599 """ 600 hfull = img.shape[0] 601 wfull = img.shape[1] 602 xtile = int(math.ceil(xnorm * wfull)) 603 ytile = int(math.ceil(ynorm * hfull)) 604 wtile = int(math.floor(wnorm * wfull)) 605 htile = int(math.floor(hnorm * hfull)) 606 if len(img.shape) == 2: 607 return img[ytile:ytile + htile, xtile:xtile + wtile].copy() 608 else: 609 return img[ytile:ytile + htile, xtile:xtile + wtile, :].copy() 610 611 612def compute_image_means(img): 613 """Calculate the mean of each color channel in the image. 614 615 Args: 616 img: Numpy float image array, with pixel values in [0,1]. 617 618 Returns: 619 A list of mean values, one per color channel in the image. 620 """ 621 means = [] 622 chans = img.shape[2] 623 for i in range(chans): 624 means.append(numpy.mean(img[:, :, i], dtype=numpy.float64)) 625 return means 626 627 628def compute_image_variances(img): 629 """Calculate the variance of each color channel in the image. 630 631 Args: 632 img: Numpy float image array, with pixel values in [0,1]. 633 634 Returns: 635 A list of variance values, one per color channel in the image. 636 """ 637 variances = [] 638 chans = img.shape[2] 639 for i in range(chans): 640 variances.append(numpy.var(img[:, :, i], dtype=numpy.float64)) 641 return variances 642 643 644def compute_image_sharpness(img): 645 """Calculate the sharpness of input image. 646 647 Args: 648 img: numpy float RGB/luma image array, with pixel values in [0,1]. 649 650 Returns: 651 Sharpness estimation value based on the average of gradient magnitude. 652 Larger value means the image is sharper. 653 """ 654 chans = img.shape[2] 655 assert chans == 1 or chans == 3 656 if chans == 1: 657 luma = img[:, :, 0] 658 else: 659 luma = convert_rgb_to_grayscale(img) 660 gy, gx = numpy.gradient(luma) 661 return numpy.average(numpy.sqrt(gy*gy + gx*gx)) 662 663 664def compute_image_max_gradients(img): 665 """Calculate the maximum gradient of each color channel in the image. 666 667 Args: 668 img: Numpy float image array, with pixel values in [0,1]. 669 670 Returns: 671 A list of gradient max values, one per color channel in the image. 672 """ 673 grads = [] 674 chans = img.shape[2] 675 for i in range(chans): 676 grads.append(numpy.amax(numpy.gradient(img[:, :, i]))) 677 return grads 678 679 680def compute_image_snrs(img): 681 """Calculate the SNR (dB) of each color channel in the image. 682 683 Args: 684 img: Numpy float image array, with pixel values in [0,1]. 685 686 Returns: 687 A list of SNR values in dB, one per color channel in the image. 688 """ 689 means = compute_image_means(img) 690 variances = compute_image_variances(img) 691 std_devs = [math.sqrt(v) for v in variances] 692 snrs = [20 * math.log10(m/s) for m, s in zip(means, std_devs)] 693 return snrs 694 695 696def convert_rgb_to_grayscale(img): 697 """Convert and 3-D array RGB image to grayscale image. 698 699 Args: 700 img: numpy float RGB/luma image array, with pixel values in [0,1]. 701 702 Returns: 703 2-D grayscale image 704 """ 705 assert img.shape[2] == 3, 'Not an RGB image' 706 return 0.299*img[:, :, 0] + 0.587*img[:, :, 1] + 0.114*img[:, :, 2] 707 708 709def normalize_img(img): 710 """Normalize the image values to between 0 and 1. 711 712 Args: 713 img: 2-D numpy array of image values 714 Returns: 715 Normalized image 716 """ 717 return (img - numpy.amin(img))/(numpy.amax(img) - numpy.amin(img)) 718 719 720def rotate_img_per_argv(img): 721 """Rotate an image 180 degrees if "rotate" is in argv. 722 723 Args: 724 img: 2-D numpy array of image values 725 Returns: 726 Rotated image 727 """ 728 img_out = img 729 if 'rotate180' in sys.argv: 730 img_out = numpy.fliplr(numpy.flipud(img_out)) 731 return img_out 732 733 734def chart_located_per_argv(chart_loc_arg): 735 """Determine if chart already located outside of test. 736 737 If chart info provided, return location and size. If not, return None. 738 Args: 739 chart_loc_arg: chart_loc arg value. 740 741 Returns: 742 chart_loc: float converted xnorm,ynorm,wnorm,hnorm,scale from argv 743 text.argv is of form 'chart_loc=0.45,0.45,0.1,0.1,1.0' 744 """ 745 if chart_loc_arg: 746 return map(float, chart_loc_arg) 747 return None, None, None, None, None 748 749 750def stationary_lens_cap(cam, req, fmt): 751 """Take up to NUM_TRYS caps and save the 1st one with lens stationary. 752 753 Args: 754 cam: open device session 755 req: capture request 756 fmt: format for capture 757 758 Returns: 759 capture 760 """ 761 tries = 0 762 done = False 763 reqs = [req] * NUM_FRAMES 764 while not done: 765 logging.debug('Waiting for lens to move to correct location.') 766 cap = cam.do_capture(reqs, fmt) 767 done = (cap[NUM_FRAMES - 1]['metadata']['android.lens.state'] == 0) 768 logging.debug('status: %s', done) 769 tries += 1 770 if tries == NUM_TRIES: 771 raise error_util.CameraItsError('Cannot settle lens after %d tries!' % 772 tries) 773 return cap[NUM_FRAMES - 1] 774 775 776def compute_image_rms_difference(rgb_x, rgb_y): 777 """Calculate the RMS difference between 2 RBG images. 778 779 Args: 780 rgb_x: image array 781 rgb_y: image array 782 783 Returns: 784 rms_diff 785 """ 786 len_rgb_x = len(rgb_x) 787 assert len(rgb_y) == len_rgb_x, 'The images have different number of planes.' 788 return math.sqrt(sum([pow(rgb_x[i] - rgb_y[i], 2.0) 789 for i in range(len_rgb_x)]) / len_rgb_x) 790 791 792class ImageProcessingUtilsTest(unittest.TestCase): 793 """Unit tests for this module.""" 794 _SQRT_2 = numpy.sqrt(2) 795 _YUV_FULL_SCALE = 1023 796 797 def test_unpack_raw10_image(self): 798 """Unit test for unpack_raw10_image. 799 800 RAW10 bit packing format 801 bit 7 bit 6 bit 5 bit 4 bit 3 bit 2 bit 1 bit 0 802 Byte 0: P0[9] P0[8] P0[7] P0[6] P0[5] P0[4] P0[3] P0[2] 803 Byte 1: P1[9] P1[8] P1[7] P1[6] P1[5] P1[4] P1[3] P1[2] 804 Byte 2: P2[9] P2[8] P2[7] P2[6] P2[5] P2[4] P2[3] P2[2] 805 Byte 3: P3[9] P3[8] P3[7] P3[6] P3[5] P3[4] P3[3] P3[2] 806 Byte 4: P3[1] P3[0] P2[1] P2[0] P1[1] P1[0] P0[1] P0[0] 807 """ 808 # Test using a random 4x4 10-bit image 809 img_w, img_h = 4, 4 810 check_list = random.sample(range(0, 1024), img_h*img_w) 811 img_check = numpy.array(check_list).reshape(img_h, img_w) 812 813 # Pack bits 814 for row_start in range(0, len(check_list), img_w): 815 msbs = [] 816 lsbs = '' 817 for pixel in range(img_w): 818 val = numpy.binary_repr(check_list[row_start+pixel], 10) 819 msbs.append(int(val[:8], base=2)) 820 lsbs = val[8:] + lsbs 821 packed = msbs 822 packed.append(int(lsbs, base=2)) 823 chunk_raw10 = numpy.array(packed, dtype='uint8').reshape(1, 5) 824 if row_start == 0: 825 img_raw10 = chunk_raw10 826 else: 827 img_raw10 = numpy.vstack((img_raw10, chunk_raw10)) 828 829 # Unpack and check against original 830 self.assertTrue(numpy.array_equal(unpack_raw10_image(img_raw10), 831 img_check)) 832 833 def test_compute_image_sharpness(self): 834 """Unit test for compute_img_sharpness. 835 836 Tests by using PNG of ISO12233 chart and blurring intentionally. 837 'sharpness' should drop off by sqrt(2) for 2x blur of image. 838 839 We do one level of initial blur as PNG image is not perfect. 840 """ 841 blur_levels = [2, 4, 8] 842 chart_file = os.path.join(TEST_IMG_DIR, 'ISO12233.png') 843 chart = cv2.imread(chart_file, cv2.IMREAD_ANYDEPTH) 844 white_level = numpy.amax(chart).astype(float) 845 sharpness = {} 846 for blur in blur_levels: 847 chart_blurred = cv2.blur(chart, (blur, blur)) 848 chart_blurred = chart_blurred[:, :, numpy.newaxis] 849 sharpness[blur] = self._YUV_FULL_SCALE * compute_image_sharpness( 850 chart_blurred / white_level) 851 852 for i in range(len(blur_levels)-1): 853 self.assertTrue(numpy.isclose( 854 sharpness[blur_levels[i]]/sharpness[blur_levels[i+1]], self._SQRT_2, 855 atol=0.1)) 856 857 def test_apply_lut_to_image(self): 858 """Unit test for apply_lut_to_image. 859 860 Test by using a canned set of values on a 1x1 pixel image. 861 The look-up table should double the value of the index: lut[x] = x*2 862 """ 863 ref_image = [0.1, 0.2, 0.3] 864 lut_max = 65536 865 lut = numpy.array([i*2 for i in range(lut_max)]) 866 x = numpy.array(ref_image).reshape(1, 1, 3) 867 y = apply_lut_to_image(x, lut).reshape(3).tolist() 868 y_ref = [i*2 for i in ref_image] 869 self.assertTrue(numpy.allclose(y, y_ref, atol=1/lut_max)) 870 871 872if __name__ == '__main__': 873 unittest.main() 874