1# Copyright 2013 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Image processing utility functions.""" 15 16 17import copy 18import io 19import logging 20import math 21import os 22import random 23import sys 24import unittest 25 26import capture_request_utils 27import cv2 28import error_util 29import numpy 30from PIL import Image 31 32 33# The matrix is from JFIF spec 34DEFAULT_YUV_TO_RGB_CCM = numpy.matrix([[1.000, 0.000, 1.402], 35 [1.000, -0.344, -0.714], 36 [1.000, 1.772, 0.000]]) 37 38DEFAULT_YUV_OFFSETS = numpy.array([0, 128, 128]) 39MAX_LUT_SIZE = 65536 40DEFAULT_GAMMA_LUT = numpy.array([ 41 math.floor((MAX_LUT_SIZE-1) * math.pow(i/(MAX_LUT_SIZE-1), 1/2.2) + 0.5) 42 for i in range(MAX_LUT_SIZE)]) 43NUM_TRIES = 2 44NUM_FRAMES = 4 45TEST_IMG_DIR = os.path.join(os.environ['CAMERA_ITS_TOP'], 'test_images') 46 47 48# pylint: disable=unused-argument 49def convert_capture_to_rgb_image(cap, 50 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 51 yuv_off=DEFAULT_YUV_OFFSETS, 52 props=None, 53 apply_ccm_raw_to_rgb=True): 54 """Convert a captured image object to a RGB image. 55 56 Args: 57 cap: A capture object as returned by its_session_utils.do_capture. 58 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 59 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 60 props: (Optional) camera properties object (of static values); 61 required for processing raw images. 62 apply_ccm_raw_to_rgb: (Optional) boolean to apply color correction matrix. 63 64 Returns: 65 RGB float-3 image array, with pixel values in [0.0, 1.0]. 66 """ 67 w = cap['width'] 68 h = cap['height'] 69 if cap['format'] == 'raw10': 70 assert props is not None 71 cap = unpack_raw10_capture(cap) 72 73 if cap['format'] == 'raw12': 74 assert props is not None 75 cap = unpack_raw12_capture(cap) 76 77 if cap['format'] == 'yuv': 78 y = cap['data'][0: w * h] 79 u = cap['data'][w * h: w * h * 5//4] 80 v = cap['data'][w * h * 5//4: w * h * 6//4] 81 return convert_yuv420_planar_to_rgb_image(y, u, v, w, h) 82 elif cap['format'] == 'jpeg': 83 return decompress_jpeg_to_rgb_image(cap['data']) 84 elif cap['format'] == 'raw' or cap['format'] == 'rawStats': 85 assert props is not None 86 r, gr, gb, b = convert_capture_to_planes(cap, props) 87 return convert_raw_to_rgb_image( 88 r, gr, gb, b, props, cap['metadata'], apply_ccm_raw_to_rgb) 89 elif cap['format'] == 'y8': 90 y = cap['data'][0: w * h] 91 return convert_y8_to_rgb_image(y, w, h) 92 else: 93 raise error_util.CameraItsError('Invalid format %s' % (cap['format'])) 94 95 96def unpack_raw10_capture(cap): 97 """Unpack a raw-10 capture to a raw-16 capture. 98 99 Args: 100 cap: A raw-10 capture object. 101 102 Returns: 103 New capture object with raw-16 data. 104 """ 105 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 106 # the MSBs of the pixels, and the 5th byte holding 4x2b LSBs. 107 w, h = cap['width'], cap['height'] 108 if w % 4 != 0: 109 raise error_util.CameraItsError('Invalid raw-10 buffer width') 110 cap = copy.deepcopy(cap) 111 cap['data'] = unpack_raw10_image(cap['data'].reshape(h, w * 5 // 4)) 112 cap['format'] = 'raw' 113 return cap 114 115 116def unpack_raw10_image(img): 117 """Unpack a raw-10 image to a raw-16 image. 118 119 Output image will have the 10 LSBs filled in each 16b word, and the 6 MSBs 120 will be set to zero. 121 122 Args: 123 img: A raw-10 image, as a uint8 numpy array. 124 125 Returns: 126 Image as a uint16 numpy array, with all row padding stripped. 127 """ 128 if img.shape[1] % 5 != 0: 129 raise error_util.CameraItsError('Invalid raw-10 buffer width') 130 w = img.shape[1] * 4 // 5 131 h = img.shape[0] 132 # Cut out the 4x8b MSBs and shift to bits [9:2] in 16b words. 133 msbs = numpy.delete(img, numpy.s_[4::5], 1) 134 msbs = msbs.astype(numpy.uint16) 135 msbs = numpy.left_shift(msbs, 2) 136 msbs = msbs.reshape(h, w) 137 # Cut out the 4x2b LSBs and put each in bits [1:0] of their own 8b words. 138 lsbs = img[::, 4::5].reshape(h, w // 4) 139 lsbs = numpy.right_shift( 140 numpy.packbits(numpy.unpackbits(lsbs).reshape(h, w // 4, 4, 2), 3), 6) 141 # Pair the LSB bits group to 0th pixel instead of 3rd pixel 142 lsbs = lsbs.reshape(h, w // 4, 4)[:, :, ::-1] 143 lsbs = lsbs.reshape(h, w) 144 # Fuse the MSBs and LSBs back together 145 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h, w) 146 return img16 147 148 149def unpack_raw12_capture(cap): 150 """Unpack a raw-12 capture to a raw-16 capture. 151 152 Args: 153 cap: A raw-12 capture object. 154 155 Returns: 156 New capture object with raw-16 data. 157 """ 158 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 159 # the MSBs of the pixels, and the 5th byte holding 4x2b LSBs. 160 w, h = cap['width'], cap['height'] 161 if w % 2 != 0: 162 raise error_util.CameraItsError('Invalid raw-12 buffer width') 163 cap = copy.deepcopy(cap) 164 cap['data'] = unpack_raw12_image(cap['data'].reshape(h, w * 3 // 2)) 165 cap['format'] = 'raw' 166 return cap 167 168 169def unpack_raw12_image(img): 170 """Unpack a raw-12 image to a raw-16 image. 171 172 Output image will have the 12 LSBs filled in each 16b word, and the 4 MSBs 173 will be set to zero. 174 175 Args: 176 img: A raw-12 image, as a uint8 numpy array. 177 178 Returns: 179 Image as a uint16 numpy array, with all row padding stripped. 180 """ 181 if img.shape[1] % 3 != 0: 182 raise error_util.CameraItsError('Invalid raw-12 buffer width') 183 w = img.shape[1] * 2 // 3 184 h = img.shape[0] 185 # Cut out the 2x8b MSBs and shift to bits [11:4] in 16b words. 186 msbs = numpy.delete(img, numpy.s_[2::3], 1) 187 msbs = msbs.astype(numpy.uint16) 188 msbs = numpy.left_shift(msbs, 4) 189 msbs = msbs.reshape(h, w) 190 # Cut out the 2x4b LSBs and put each in bits [3:0] of their own 8b words. 191 lsbs = img[::, 2::3].reshape(h, w // 2) 192 lsbs = numpy.right_shift( 193 numpy.packbits(numpy.unpackbits(lsbs).reshape(h, w // 2, 2, 4), 3), 4) 194 # Pair the LSB bits group to pixel 0 instead of pixel 1 195 lsbs = lsbs.reshape(h, w // 2, 2)[:, :, ::-1] 196 lsbs = lsbs.reshape(h, w) 197 # Fuse the MSBs and LSBs back together 198 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h, w) 199 return img16 200 201 202def convert_yuv420_planar_to_rgb_image(y_plane, u_plane, v_plane, 203 w, h, 204 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 205 yuv_off=DEFAULT_YUV_OFFSETS): 206 """Convert a YUV420 8-bit planar image to an RGB image. 207 208 Args: 209 y_plane: The packed 8-bit Y plane. 210 u_plane: The packed 8-bit U plane. 211 v_plane: The packed 8-bit V plane. 212 w: The width of the image. 213 h: The height of the image. 214 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 215 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 216 217 Returns: 218 RGB float-3 image array, with pixel values in [0.0, 1.0]. 219 """ 220 y = numpy.subtract(y_plane, yuv_off[0]) 221 u = numpy.subtract(u_plane, yuv_off[1]).view(numpy.int8) 222 v = numpy.subtract(v_plane, yuv_off[2]).view(numpy.int8) 223 u = u.reshape(h // 2, w // 2).repeat(2, axis=1).repeat(2, axis=0) 224 v = v.reshape(h // 2, w // 2).repeat(2, axis=1).repeat(2, axis=0) 225 yuv = numpy.dstack([y, u.reshape(w * h), v.reshape(w * h)]) 226 flt = numpy.empty([h, w, 3], dtype=numpy.float32) 227 flt.reshape(w * h * 3)[:] = yuv.reshape(h * w * 3)[:] 228 flt = numpy.dot(flt.reshape(w * h, 3), ccm_yuv_to_rgb.T).clip(0, 255) 229 rgb = numpy.empty([h, w, 3], dtype=numpy.uint8) 230 rgb.reshape(w * h * 3)[:] = flt.reshape(w * h * 3)[:] 231 return rgb.astype(numpy.float32) / 255.0 232 233 234def decompress_jpeg_to_rgb_image(jpeg_buffer): 235 """Decompress a JPEG-compressed image, returning as an RGB image. 236 237 Args: 238 jpeg_buffer: The JPEG stream. 239 240 Returns: 241 A numpy array for the RGB image, with pixels in [0,1]. 242 """ 243 img = Image.open(io.BytesIO(jpeg_buffer)) 244 w = img.size[0] 245 h = img.size[1] 246 return numpy.array(img).reshape(h, w, 3) / 255.0 247 248 249def convert_capture_to_planes(cap, props=None): 250 """Convert a captured image object to separate image planes. 251 252 Decompose an image into multiple images, corresponding to different planes. 253 254 For YUV420 captures ("yuv"): 255 Returns Y,U,V planes, where the Y plane is full-res and the U,V planes 256 are each 1/2 x 1/2 of the full res. 257 258 For Bayer captures ("raw", "raw10", "raw12", or "rawStats"): 259 Returns planes in the order R,Gr,Gb,B, regardless of the Bayer pattern 260 layout. For full-res raw images ("raw", "raw10", "raw12"), each plane 261 is 1/2 x 1/2 of the full res. For "rawStats" images, the mean image 262 is returned. 263 264 For JPEG captures ("jpeg"): 265 Returns R,G,B full-res planes. 266 267 Args: 268 cap: A capture object as returned by its_session_utils.do_capture. 269 props: (Optional) camera properties object (of static values); 270 required for processing raw images. 271 272 Returns: 273 A tuple of float numpy arrays (one per plane), consisting of pixel values 274 in the range [0.0, 1.0]. 275 """ 276 w = cap['width'] 277 h = cap['height'] 278 if cap['format'] == 'raw10': 279 assert props is not None 280 cap = unpack_raw10_capture(cap) 281 if cap['format'] == 'raw12': 282 assert props is not None 283 cap = unpack_raw12_capture(cap) 284 if cap['format'] == 'yuv': 285 y = cap['data'][0:w * h] 286 u = cap['data'][w * h:w * h * 5 // 4] 287 v = cap['data'][w * h * 5 // 4:w * h * 6 // 4] 288 return ((y.astype(numpy.float32) / 255.0).reshape(h, w, 1), 289 (u.astype(numpy.float32) / 255.0).reshape(h // 2, w // 2, 1), 290 (v.astype(numpy.float32) / 255.0).reshape(h // 2, w // 2, 1)) 291 elif cap['format'] == 'jpeg': 292 rgb = decompress_jpeg_to_rgb_image(cap['data']).reshape(w * h * 3) 293 return (rgb[::3].reshape(h, w, 1), rgb[1::3].reshape(h, w, 1), 294 rgb[2::3].reshape(h, w, 1)) 295 elif cap['format'] == 'raw': 296 assert props is not None 297 white_level = float(props['android.sensor.info.whiteLevel']) 298 img = numpy.ndarray( 299 shape=(h * w,), dtype='<u2', buffer=cap['data'][0:w * h * 2]) 300 img = img.astype(numpy.float32).reshape(h, w) / white_level 301 # Crop the raw image to the active array region. 302 if (props.get('android.sensor.info.preCorrectionActiveArraySize') is 303 not None and 304 props.get('android.sensor.info.pixelArraySize') is not None): 305 # Note that the Rect class is defined such that the left,top values 306 # are "inside" while the right,bottom values are "outside"; that is, 307 # it's inclusive of the top,left sides only. So, the width is 308 # computed as right-left, rather than right-left+1, etc. 309 wfull = props['android.sensor.info.pixelArraySize']['width'] 310 hfull = props['android.sensor.info.pixelArraySize']['height'] 311 xcrop = props['android.sensor.info.preCorrectionActiveArraySize']['left'] 312 ycrop = props['android.sensor.info.preCorrectionActiveArraySize']['top'] 313 wcrop = props['android.sensor.info.preCorrectionActiveArraySize'][ 314 'right'] - xcrop 315 hcrop = props['android.sensor.info.preCorrectionActiveArraySize'][ 316 'bottom'] - ycrop 317 assert wfull >= wcrop >= 0 318 assert hfull >= hcrop >= 0 319 assert wfull - wcrop >= xcrop >= 0 320 assert hfull - hcrop >= ycrop >= 0 321 if w == wfull and h == hfull: 322 # Crop needed; extract the center region. 323 img = img[ycrop:ycrop + hcrop, xcrop:xcrop + wcrop] 324 w = wcrop 325 h = hcrop 326 elif w == wcrop and h == hcrop: 327 logging.debug('Image is already cropped.No cropping needed.') 328 # pylint: disable=pointless-statement 329 None 330 else: 331 raise error_util.CameraItsError('Invalid image size metadata') 332 # Separate the image planes. 333 imgs = [ 334 img[::2].reshape(w * h // 2)[::2].reshape(h // 2, w // 2, 1), 335 img[::2].reshape(w * h // 2)[1::2].reshape(h // 2, w // 2, 1), 336 img[1::2].reshape(w * h // 2)[::2].reshape(h // 2, w // 2, 1), 337 img[1::2].reshape(w * h // 2)[1::2].reshape(h // 2, w // 2, 1) 338 ] 339 idxs = get_canonical_cfa_order(props) 340 return [imgs[i] for i in idxs] 341 elif cap['format'] == 'rawStats': 342 assert props is not None 343 white_level = float(props['android.sensor.info.whiteLevel']) 344 # pylint: disable=unused-variable 345 mean_image, var_image = unpack_rawstats_capture(cap) 346 idxs = get_canonical_cfa_order(props) 347 return [mean_image[:, :, i] / white_level for i in idxs] 348 else: 349 raise error_util.CameraItsError('Invalid format %s' % (cap['format'])) 350 351 352def downscale_image(img, f): 353 """Shrink an image by a given integer factor. 354 355 This function computes output pixel values by averaging over rectangular 356 regions of the input image; it doesn't skip or sample pixels, and all input 357 image pixels are evenly weighted. 358 359 If the downscaling factor doesn't cleanly divide the width and/or height, 360 then the remaining pixels on the right or bottom edge are discarded prior 361 to the downscaling. 362 363 Args: 364 img: The input image as an ndarray. 365 f: The downscaling factor, which should be an integer. 366 367 Returns: 368 The new (downscaled) image, as an ndarray. 369 """ 370 h, w, chans = img.shape 371 f = int(f) 372 assert f >= 1 373 h = (h//f)*f 374 w = (w//f)*f 375 img = img[0:h:, 0:w:, ::] 376 chs = [] 377 for i in range(chans): 378 ch = img.reshape(h*w*chans)[i::chans].reshape(h, w) 379 ch = ch.reshape(h, w//f, f).mean(2).reshape(h, w//f) 380 ch = ch.T.reshape(w//f, h//f, f).mean(2).T.reshape(h//f, w//f) 381 chs.append(ch.reshape(h*w//(f*f))) 382 img = numpy.vstack(chs).T.reshape(h//f, w//f, chans) 383 return img 384 385 386def convert_raw_to_rgb_image(r_plane, gr_plane, gb_plane, b_plane, props, 387 cap_res, apply_ccm_raw_to_rgb=True): 388 """Convert a Bayer raw-16 image to an RGB image. 389 390 Includes some extremely rudimentary demosaicking and color processing 391 operations; the output of this function shouldn't be used for any image 392 quality analysis. 393 394 Args: 395 r_plane: 396 gr_plane: 397 gb_plane: 398 b_plane: Numpy arrays for each color plane 399 in the Bayer image, with pixels in the [0.0, 1.0] range. 400 props: Camera properties object. 401 cap_res: Capture result (metadata) object. 402 apply_ccm_raw_to_rgb: (Optional) boolean to apply color correction matrix. 403 404 Returns: 405 RGB float-3 image array, with pixel values in [0.0, 1.0] 406 """ 407 # Values required for the RAW to RGB conversion. 408 assert props is not None 409 white_level = float(props['android.sensor.info.whiteLevel']) 410 black_levels = props['android.sensor.blackLevelPattern'] 411 gains = cap_res['android.colorCorrection.gains'] 412 ccm = cap_res['android.colorCorrection.transform'] 413 414 # Reorder black levels and gains to R,Gr,Gb,B, to match the order 415 # of the planes. 416 black_levels = [get_black_level(i, props, cap_res) for i in range(4)] 417 gains = get_gains_in_canonical_order(props, gains) 418 419 # Convert CCM from rational to float, as numpy arrays. 420 ccm = numpy.array(capture_request_utils.rational_to_float(ccm)).reshape(3, 3) 421 422 # Need to scale the image back to the full [0,1] range after subtracting 423 # the black level from each pixel. 424 scale = white_level / (white_level - max(black_levels)) 425 426 # Three-channel black levels, normalized to [0,1] by white_level. 427 black_levels = numpy.array( 428 [b / white_level for b in [black_levels[i] for i in [0, 1, 3]]]) 429 430 # Three-channel gains. 431 gains = numpy.array([gains[i] for i in [0, 1, 3]]) 432 433 h, w = r_plane.shape[:2] 434 img = numpy.dstack([r_plane, (gr_plane + gb_plane) / 2.0, b_plane]) 435 img = (((img.reshape(h, w, 3) - black_levels) * scale) * gains).clip(0.0, 1.0) 436 if apply_ccm_raw_to_rgb: 437 img = numpy.dot( 438 img.reshape(w * h, 3), ccm.T).reshape(h, w, 3).clip(0.0, 1.0) 439 return img 440 441 442def convert_y8_to_rgb_image(y_plane, w, h): 443 """Convert a Y 8-bit image to an RGB image. 444 445 Args: 446 y_plane: The packed 8-bit Y plane. 447 w: The width of the image. 448 h: The height of the image. 449 450 Returns: 451 RGB float-3 image array, with pixel values in [0.0, 1.0]. 452 """ 453 y3 = numpy.dstack([y_plane, y_plane, y_plane]) 454 rgb = numpy.empty([h, w, 3], dtype=numpy.uint8) 455 rgb.reshape(w * h * 3)[:] = y3.reshape(w * h * 3)[:] 456 return rgb.astype(numpy.float32) / 255.0 457 458 459def write_image(img, fname, apply_gamma=False): 460 """Save a float-3 numpy array image to a file. 461 462 Supported formats: PNG, JPEG, and others; see PIL docs for more. 463 464 Image can be 3-channel, which is interpreted as RGB, or can be 1-channel, 465 which is greyscale. 466 467 Can optionally specify that the image should be gamma-encoded prior to 468 writing it out; this should be done if the image contains linear pixel 469 values, to make the image look "normal". 470 471 Args: 472 img: Numpy image array data. 473 fname: Path of file to save to; the extension specifies the format. 474 apply_gamma: (Optional) apply gamma to the image prior to writing it. 475 """ 476 if apply_gamma: 477 img = apply_lut_to_image(img, DEFAULT_GAMMA_LUT) 478 (h, w, chans) = img.shape 479 if chans == 3: 480 Image.fromarray((img * 255.0).astype(numpy.uint8), 'RGB').save(fname) 481 elif chans == 1: 482 img3 = (img * 255.0).astype(numpy.uint8).repeat(3).reshape(h, w, 3) 483 Image.fromarray(img3, 'RGB').save(fname) 484 else: 485 raise error_util.CameraItsError('Unsupported image type') 486 487 488def read_image(fname): 489 """Read image function to match write_image() above.""" 490 return Image.open(fname) 491 492 493def apply_lut_to_image(img, lut): 494 """Applies a LUT to every pixel in a float image array. 495 496 Internally converts to a 16b integer image, since the LUT can work with up 497 to 16b->16b mappings (i.e. values in the range [0,65535]). The lut can also 498 have fewer than 65536 entries, however it must be sized as a power of 2 499 (and for smaller luts, the scale must match the bitdepth). 500 501 For a 16b lut of 65536 entries, the operation performed is: 502 503 lut[r * 65535] / 65535 -> r' 504 lut[g * 65535] / 65535 -> g' 505 lut[b * 65535] / 65535 -> b' 506 507 For a 10b lut of 1024 entries, the operation becomes: 508 509 lut[r * 1023] / 1023 -> r' 510 lut[g * 1023] / 1023 -> g' 511 lut[b * 1023] / 1023 -> b' 512 513 Args: 514 img: Numpy float image array, with pixel values in [0,1]. 515 lut: Numpy table encoding a LUT, mapping 16b integer values. 516 517 Returns: 518 Float image array after applying LUT to each pixel. 519 """ 520 n = len(lut) 521 if n <= 0 or n > MAX_LUT_SIZE or (n & (n - 1)) != 0: 522 raise error_util.CameraItsError('Invalid arg LUT size: %d' % (n)) 523 m = float(n - 1) 524 return (lut[(img * m).astype(numpy.uint16)] / m).astype(numpy.float32) 525 526 527def get_gains_in_canonical_order(props, gains): 528 """Reorders the gains tuple to the canonical R,Gr,Gb,B order. 529 530 Args: 531 props: Camera properties object. 532 gains: List of 4 values, in R,G_even,G_odd,B order. 533 534 Returns: 535 List of gains values, in R,Gr,Gb,B order. 536 """ 537 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 538 if cfa_pat in [0, 1]: 539 # RGGB or GRBG, so G_even is Gr 540 return gains 541 elif cfa_pat in [2, 3]: 542 # GBRG or BGGR, so G_even is Gb 543 return [gains[0], gains[2], gains[1], gains[3]] 544 else: 545 raise error_util.CameraItsError('Not supported') 546 547 548def get_black_level(chan, props, cap_res=None): 549 """Return the black level to use for a given capture. 550 551 Uses a dynamic value from the capture result if available, else falls back 552 to the static global value in the camera characteristics. 553 554 Args: 555 chan: The channel index, in canonical order (R, Gr, Gb, B). 556 props: The camera properties object. 557 cap_res: A capture result object. 558 559 Returns: 560 The black level value for the specified channel. 561 """ 562 if (cap_res is not None and 563 'android.sensor.dynamicBlackLevel' in cap_res and 564 cap_res['android.sensor.dynamicBlackLevel'] is not None): 565 black_levels = cap_res['android.sensor.dynamicBlackLevel'] 566 else: 567 black_levels = props['android.sensor.blackLevelPattern'] 568 idxs = get_canonical_cfa_order(props) 569 ordered_black_levels = [black_levels[i] for i in idxs] 570 return ordered_black_levels[chan] 571 572 573def get_canonical_cfa_order(props): 574 """Returns a mapping to the standard order R,Gr,Gb,B. 575 576 Returns a mapping from the Bayer 2x2 top-left grid in the CFA to the standard 577 order R,Gr,Gb,B. 578 579 Args: 580 props: Camera properties object. 581 582 Returns: 583 List of 4 integers, corresponding to the positions in the 2x2 top- 584 left Bayer grid of R,Gr,Gb,B, where the 2x2 grid is labeled as 585 0,1,2,3 in row major order. 586 """ 587 # Note that raw streams aren't croppable, so the cropRegion doesn't need 588 # to be considered when determining the top-left pixel color. 589 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 590 if cfa_pat == 0: 591 # RGGB 592 return [0, 1, 2, 3] 593 elif cfa_pat == 1: 594 # GRBG 595 return [1, 0, 3, 2] 596 elif cfa_pat == 2: 597 # GBRG 598 return [2, 3, 0, 1] 599 elif cfa_pat == 3: 600 # BGGR 601 return [3, 2, 1, 0] 602 else: 603 raise error_util.CameraItsError('Not supported') 604 605 606def unpack_rawstats_capture(cap): 607 """Unpack a rawStats capture to the mean and variance images. 608 609 Args: 610 cap: A capture object as returned by its_session_utils.do_capture. 611 612 Returns: 613 Tuple (mean_image var_image) of float-4 images, with non-normalized 614 pixel values computed from the RAW16 images on the device 615 """ 616 assert cap['format'] == 'rawStats' 617 w = cap['width'] 618 h = cap['height'] 619 img = numpy.ndarray(shape=(2 * h * w * 4,), dtype='<f', buffer=cap['data']) 620 analysis_image = img.reshape((2, h, w, 4)) 621 mean_image = analysis_image[0, :, :, :].reshape(h, w, 4) 622 var_image = analysis_image[1, :, :, :].reshape(h, w, 4) 623 return mean_image, var_image 624 625 626def get_image_patch(img, xnorm, ynorm, wnorm, hnorm): 627 """Get a patch (tile) of an image. 628 629 Args: 630 img: Numpy float image array, with pixel values in [0,1]. 631 xnorm: 632 ynorm: 633 wnorm: 634 hnorm: Normalized (in [0,1]) coords for the tile. 635 636 Returns: 637 Numpy float image array of the patch. 638 """ 639 hfull = img.shape[0] 640 wfull = img.shape[1] 641 xtile = int(math.ceil(xnorm * wfull)) 642 ytile = int(math.ceil(ynorm * hfull)) 643 wtile = int(math.floor(wnorm * wfull)) 644 htile = int(math.floor(hnorm * hfull)) 645 if len(img.shape) == 2: 646 return img[ytile:ytile + htile, xtile:xtile + wtile].copy() 647 else: 648 return img[ytile:ytile + htile, xtile:xtile + wtile, :].copy() 649 650 651def compute_image_means(img): 652 """Calculate the mean of each color channel in the image. 653 654 Args: 655 img: Numpy float image array, with pixel values in [0,1]. 656 657 Returns: 658 A list of mean values, one per color channel in the image. 659 """ 660 means = [] 661 chans = img.shape[2] 662 for i in range(chans): 663 means.append(numpy.mean(img[:, :, i], dtype=numpy.float64)) 664 return means 665 666 667def compute_image_variances(img): 668 """Calculate the variance of each color channel in the image. 669 670 Args: 671 img: Numpy float image array, with pixel values in [0,1]. 672 673 Returns: 674 A list of variance values, one per color channel in the image. 675 """ 676 variances = [] 677 chans = img.shape[2] 678 for i in range(chans): 679 variances.append(numpy.var(img[:, :, i], dtype=numpy.float64)) 680 return variances 681 682 683def compute_image_sharpness(img): 684 """Calculate the sharpness of input image. 685 686 Args: 687 img: numpy float RGB/luma image array, with pixel values in [0,1]. 688 689 Returns: 690 Sharpness estimation value based on the average of gradient magnitude. 691 Larger value means the image is sharper. 692 """ 693 chans = img.shape[2] 694 assert chans == 1 or chans == 3 695 if chans == 1: 696 luma = img[:, :, 0] 697 else: 698 luma = convert_rgb_to_grayscale(img) 699 gy, gx = numpy.gradient(luma) 700 return numpy.average(numpy.sqrt(gy*gy + gx*gx)) 701 702 703def compute_image_max_gradients(img): 704 """Calculate the maximum gradient of each color channel in the image. 705 706 Args: 707 img: Numpy float image array, with pixel values in [0,1]. 708 709 Returns: 710 A list of gradient max values, one per color channel in the image. 711 """ 712 grads = [] 713 chans = img.shape[2] 714 for i in range(chans): 715 grads.append(numpy.amax(numpy.gradient(img[:, :, i]))) 716 return grads 717 718 719def compute_image_snrs(img): 720 """Calculate the SNR (dB) of each color channel in the image. 721 722 Args: 723 img: Numpy float image array, with pixel values in [0,1]. 724 725 Returns: 726 A list of SNR values in dB, one per color channel in the image. 727 """ 728 means = compute_image_means(img) 729 variances = compute_image_variances(img) 730 std_devs = [math.sqrt(v) for v in variances] 731 snrs = [20 * math.log10(m/s) for m, s in zip(means, std_devs)] 732 return snrs 733 734 735def convert_rgb_to_grayscale(img): 736 """Convert and 3-D array RGB image to grayscale image. 737 738 Args: 739 img: numpy float RGB/luma image array, with pixel values in [0,1]. 740 741 Returns: 742 2-D grayscale image 743 """ 744 assert img.shape[2] == 3, 'Not an RGB image' 745 return 0.299*img[:, :, 0] + 0.587*img[:, :, 1] + 0.114*img[:, :, 2] 746 747 748def normalize_img(img): 749 """Normalize the image values to between 0 and 1. 750 751 Args: 752 img: 2-D numpy array of image values 753 Returns: 754 Normalized image 755 """ 756 return (img - numpy.amin(img))/(numpy.amax(img) - numpy.amin(img)) 757 758 759def rotate_img_per_argv(img): 760 """Rotate an image 180 degrees if "rotate" is in argv. 761 762 Args: 763 img: 2-D numpy array of image values 764 Returns: 765 Rotated image 766 """ 767 img_out = img 768 if 'rotate180' in sys.argv: 769 img_out = numpy.fliplr(numpy.flipud(img_out)) 770 return img_out 771 772 773def chart_located_per_argv(chart_loc_arg): 774 """Determine if chart already located outside of test. 775 776 If chart info provided, return location and size. If not, return None. 777 Args: 778 chart_loc_arg: chart_loc arg value. 779 780 Returns: 781 chart_loc: float converted xnorm,ynorm,wnorm,hnorm,scale from argv 782 text.argv is of form 'chart_loc=0.45,0.45,0.1,0.1,1.0' 783 """ 784 if chart_loc_arg: 785 return map(float, chart_loc_arg) 786 return None, None, None, None, None 787 788 789def stationary_lens_cap(cam, req, fmt): 790 """Take up to NUM_TRYS caps and save the 1st one with lens stationary. 791 792 Args: 793 cam: open device session 794 req: capture request 795 fmt: format for capture 796 797 Returns: 798 capture 799 """ 800 tries = 0 801 done = False 802 reqs = [req] * NUM_FRAMES 803 while not done: 804 logging.debug('Waiting for lens to move to correct location.') 805 cap = cam.do_capture(reqs, fmt) 806 done = (cap[NUM_FRAMES - 1]['metadata']['android.lens.state'] == 0) 807 logging.debug('status: %s', done) 808 tries += 1 809 if tries == NUM_TRIES: 810 raise error_util.CameraItsError('Cannot settle lens after %d tries!' % 811 tries) 812 return cap[NUM_FRAMES - 1] 813 814 815def compute_image_rms_difference(rgb_x, rgb_y): 816 """Calculate the RMS difference between 2 RBG images. 817 818 Args: 819 rgb_x: image array 820 rgb_y: image array 821 822 Returns: 823 rms_diff 824 """ 825 len_rgb_x = len(rgb_x) 826 assert len(rgb_y) == len_rgb_x, 'The images have different number of planes.' 827 return math.sqrt(sum([pow(rgb_x[i] - rgb_y[i], 2.0) 828 for i in range(len_rgb_x)]) / len_rgb_x) 829 830 831class ImageProcessingUtilsTest(unittest.TestCase): 832 """Unit tests for this module.""" 833 _SQRT_2 = numpy.sqrt(2) 834 _YUV_FULL_SCALE = 1023 835 836 def test_unpack_raw10_image(self): 837 """Unit test for unpack_raw10_image. 838 839 RAW10 bit packing format 840 bit 7 bit 6 bit 5 bit 4 bit 3 bit 2 bit 1 bit 0 841 Byte 0: P0[9] P0[8] P0[7] P0[6] P0[5] P0[4] P0[3] P0[2] 842 Byte 1: P1[9] P1[8] P1[7] P1[6] P1[5] P1[4] P1[3] P1[2] 843 Byte 2: P2[9] P2[8] P2[7] P2[6] P2[5] P2[4] P2[3] P2[2] 844 Byte 3: P3[9] P3[8] P3[7] P3[6] P3[5] P3[4] P3[3] P3[2] 845 Byte 4: P3[1] P3[0] P2[1] P2[0] P1[1] P1[0] P0[1] P0[0] 846 """ 847 # Test using a random 4x4 10-bit image 848 img_w, img_h = 4, 4 849 check_list = random.sample(range(0, 1024), img_h*img_w) 850 img_check = numpy.array(check_list).reshape(img_h, img_w) 851 852 # Pack bits 853 for row_start in range(0, len(check_list), img_w): 854 msbs = [] 855 lsbs = '' 856 for pixel in range(img_w): 857 val = numpy.binary_repr(check_list[row_start+pixel], 10) 858 msbs.append(int(val[:8], base=2)) 859 lsbs = val[8:] + lsbs 860 packed = msbs 861 packed.append(int(lsbs, base=2)) 862 chunk_raw10 = numpy.array(packed, dtype='uint8').reshape(1, 5) 863 if row_start == 0: 864 img_raw10 = chunk_raw10 865 else: 866 img_raw10 = numpy.vstack((img_raw10, chunk_raw10)) 867 868 # Unpack and check against original 869 self.assertTrue(numpy.array_equal(unpack_raw10_image(img_raw10), 870 img_check)) 871 872 def test_compute_image_sharpness(self): 873 """Unit test for compute_img_sharpness. 874 875 Tests by using PNG of ISO12233 chart and blurring intentionally. 876 'sharpness' should drop off by sqrt(2) for 2x blur of image. 877 878 We do one level of initial blur as PNG image is not perfect. 879 """ 880 blur_levels = [2, 4, 8] 881 chart_file = os.path.join(TEST_IMG_DIR, 'ISO12233.png') 882 chart = cv2.imread(chart_file, cv2.IMREAD_ANYDEPTH) 883 white_level = numpy.amax(chart).astype(float) 884 sharpness = {} 885 for blur in blur_levels: 886 chart_blurred = cv2.blur(chart, (blur, blur)) 887 chart_blurred = chart_blurred[:, :, numpy.newaxis] 888 sharpness[blur] = self._YUV_FULL_SCALE * compute_image_sharpness( 889 chart_blurred / white_level) 890 891 for i in range(len(blur_levels)-1): 892 self.assertTrue(numpy.isclose( 893 sharpness[blur_levels[i]]/sharpness[blur_levels[i+1]], self._SQRT_2, 894 atol=0.1)) 895 896 def test_apply_lut_to_image(self): 897 """Unit test for apply_lut_to_image. 898 899 Test by using a canned set of values on a 1x1 pixel image. 900 The look-up table should double the value of the index: lut[x] = x*2 901 """ 902 ref_image = [0.1, 0.2, 0.3] 903 lut_max = 65536 904 lut = numpy.array([i*2 for i in range(lut_max)]) 905 x = numpy.array(ref_image).reshape(1, 1, 3) 906 y = apply_lut_to_image(x, lut).reshape(3).tolist() 907 y_ref = [i*2 for i in ref_image] 908 self.assertTrue(numpy.allclose(y, y_ref, atol=1/lut_max)) 909 910 911if __name__ == '__main__': 912 unittest.main() 913