1# Copyright 2013 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15import matplotlib 16matplotlib.use('Agg') 17 18import its.error 19import sys 20from PIL import Image 21import numpy 22import math 23import unittest 24import cStringIO 25import copy 26import random 27 28DEFAULT_YUV_TO_RGB_CCM = numpy.matrix([ 29 [1.000, 0.000, 1.402], 30 [1.000, -0.344, -0.714], 31 [1.000, 1.772, 0.000]]) 32 33DEFAULT_YUV_OFFSETS = numpy.array([0, 128, 128]) 34 35DEFAULT_GAMMA_LUT = numpy.array( 36 [math.floor(65535 * math.pow(i/65535.0, 1/2.2) + 0.5) 37 for i in xrange(65536)]) 38 39DEFAULT_INVGAMMA_LUT = numpy.array( 40 [math.floor(65535 * math.pow(i/65535.0, 2.2) + 0.5) 41 for i in xrange(65536)]) 42 43MAX_LUT_SIZE = 65536 44 45NUM_TRYS = 2 46NUM_FRAMES = 4 47 48 49def convert_capture_to_rgb_image(cap, 50 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 51 yuv_off=DEFAULT_YUV_OFFSETS, 52 props=None): 53 """Convert a captured image object to a RGB image. 54 55 Args: 56 cap: A capture object as returned by its.device.do_capture. 57 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 58 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 59 props: (Optional) camera properties object (of static values); 60 required for processing raw images. 61 62 Returns: 63 RGB float-3 image array, with pixel values in [0.0, 1.0]. 64 """ 65 w = cap["width"] 66 h = cap["height"] 67 if cap["format"] == "raw10": 68 assert(props is not None) 69 cap = unpack_raw10_capture(cap, props) 70 if cap["format"] == "raw12": 71 assert(props is not None) 72 cap = unpack_raw12_capture(cap, props) 73 if cap["format"] == "yuv": 74 y = cap["data"][0:w*h] 75 u = cap["data"][w*h:w*h*5/4] 76 v = cap["data"][w*h*5/4:w*h*6/4] 77 return convert_yuv420_planar_to_rgb_image(y, u, v, w, h) 78 elif cap["format"] == "jpeg": 79 return decompress_jpeg_to_rgb_image(cap["data"]) 80 elif cap["format"] == "raw" or cap["format"] == "rawStats": 81 assert(props is not None) 82 r,gr,gb,b = convert_capture_to_planes(cap, props) 83 return convert_raw_to_rgb_image(r,gr,gb,b, props, cap["metadata"]) 84 else: 85 raise its.error.Error('Invalid format %s' % (cap["format"])) 86 87 88def unpack_rawstats_capture(cap): 89 """Unpack a rawStats capture to the mean and variance images. 90 91 Args: 92 cap: A capture object as returned by its.device.do_capture. 93 94 Returns: 95 Tuple (mean_image var_image) of float-4 images, with non-normalized 96 pixel values computed from the RAW16 images on the device 97 """ 98 assert(cap["format"] == "rawStats") 99 w = cap["width"] 100 h = cap["height"] 101 img = numpy.ndarray(shape=(2*h*w*4,), dtype='<f', buffer=cap["data"]) 102 analysis_image = img.reshape(2,h,w,4) 103 mean_image = analysis_image[0,:,:,:].reshape(h,w,4) 104 var_image = analysis_image[1,:,:,:].reshape(h,w,4) 105 return mean_image, var_image 106 107 108def unpack_raw10_capture(cap, props): 109 """Unpack a raw-10 capture to a raw-16 capture. 110 111 Args: 112 cap: A raw-10 capture object. 113 props: Camera properties object. 114 115 Returns: 116 New capture object with raw-16 data. 117 """ 118 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 119 # the MSPs of the pixels, and the 5th byte holding 4x2b LSBs. 120 w,h = cap["width"], cap["height"] 121 if w % 4 != 0: 122 raise its.error.Error('Invalid raw-10 buffer width') 123 cap = copy.deepcopy(cap) 124 cap["data"] = unpack_raw10_image(cap["data"].reshape(h,w*5/4)) 125 cap["format"] = "raw" 126 return cap 127 128 129def unpack_raw10_image(img): 130 """Unpack a raw-10 image to a raw-16 image. 131 132 Output image will have the 10 LSBs filled in each 16b word, and the 6 MSBs 133 will be set to zero. 134 135 Args: 136 img: A raw-10 image, as a uint8 numpy array. 137 138 Returns: 139 Image as a uint16 numpy array, with all row padding stripped. 140 """ 141 if img.shape[1] % 5 != 0: 142 raise its.error.Error('Invalid raw-10 buffer width') 143 w = img.shape[1]*4/5 144 h = img.shape[0] 145 # Cut out the 4x8b MSBs and shift to bits [9:2] in 16b words. 146 msbs = numpy.delete(img, numpy.s_[4::5], 1) 147 msbs = msbs.astype(numpy.uint16) 148 msbs = numpy.left_shift(msbs, 2) 149 msbs = msbs.reshape(h,w) 150 # Cut out the 4x2b LSBs and put each in bits [1:0] of their own 8b words. 151 lsbs = img[::, 4::5].reshape(h,w/4) 152 lsbs = numpy.right_shift( 153 numpy.packbits(numpy.unpackbits(lsbs).reshape(h,w/4,4,2),3), 6) 154 # Pair the LSB bits group to pixel 0 instead of pixel 3 155 lsbs = lsbs.reshape(h,w/4,4)[:,:,::-1] 156 lsbs = lsbs.reshape(h,w) 157 # Fuse the MSBs and LSBs back together 158 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h,w) 159 return img16 160 161 162def unpack_raw12_capture(cap, props): 163 """Unpack a raw-12 capture to a raw-16 capture. 164 165 Args: 166 cap: A raw-12 capture object. 167 props: Camera properties object. 168 169 Returns: 170 New capture object with raw-16 data. 171 """ 172 # Data is packed as 4x10b pixels in 5 bytes, with the first 4 bytes holding 173 # the MSBs of the pixels, and the 5th byte holding 4x2b LSBs. 174 w,h = cap["width"], cap["height"] 175 if w % 2 != 0: 176 raise its.error.Error('Invalid raw-12 buffer width') 177 cap = copy.deepcopy(cap) 178 cap["data"] = unpack_raw12_image(cap["data"].reshape(h,w*3/2)) 179 cap["format"] = "raw" 180 return cap 181 182 183def unpack_raw12_image(img): 184 """Unpack a raw-12 image to a raw-16 image. 185 186 Output image will have the 12 LSBs filled in each 16b word, and the 4 MSBs 187 will be set to zero. 188 189 Args: 190 img: A raw-12 image, as a uint8 numpy array. 191 192 Returns: 193 Image as a uint16 numpy array, with all row padding stripped. 194 """ 195 if img.shape[1] % 3 != 0: 196 raise its.error.Error('Invalid raw-12 buffer width') 197 w = img.shape[1]*2/3 198 h = img.shape[0] 199 # Cut out the 2x8b MSBs and shift to bits [11:4] in 16b words. 200 msbs = numpy.delete(img, numpy.s_[2::3], 1) 201 msbs = msbs.astype(numpy.uint16) 202 msbs = numpy.left_shift(msbs, 4) 203 msbs = msbs.reshape(h,w) 204 # Cut out the 2x4b LSBs and put each in bits [3:0] of their own 8b words. 205 lsbs = img[::, 2::3].reshape(h,w/2) 206 lsbs = numpy.right_shift( 207 numpy.packbits(numpy.unpackbits(lsbs).reshape(h,w/2,2,4),3), 4) 208 # Pair the LSB bits group to pixel 0 instead of pixel 1 209 lsbs = lsbs.reshape(h,w/2,2)[:,:,::-1] 210 lsbs = lsbs.reshape(h,w) 211 # Fuse the MSBs and LSBs back together 212 img16 = numpy.bitwise_or(msbs, lsbs).reshape(h,w) 213 return img16 214 215 216def convert_capture_to_planes(cap, props=None): 217 """Convert a captured image object to separate image planes. 218 219 Decompose an image into multiple images, corresponding to different planes. 220 221 For YUV420 captures ("yuv"): 222 Returns Y,U,V planes, where the Y plane is full-res and the U,V planes 223 are each 1/2 x 1/2 of the full res. 224 225 For Bayer captures ("raw", "raw10", "raw12", or "rawStats"): 226 Returns planes in the order R,Gr,Gb,B, regardless of the Bayer pattern 227 layout. For full-res raw images ("raw", "raw10", "raw12"), each plane 228 is 1/2 x 1/2 of the full res. For "rawStats" images, the mean image 229 is returned. 230 231 For JPEG captures ("jpeg"): 232 Returns R,G,B full-res planes. 233 234 Args: 235 cap: A capture object as returned by its.device.do_capture. 236 props: (Optional) camera properties object (of static values); 237 required for processing raw images. 238 239 Returns: 240 A tuple of float numpy arrays (one per plane), consisting of pixel 241 values in the range [0.0, 1.0]. 242 """ 243 w = cap["width"] 244 h = cap["height"] 245 if cap["format"] == "raw10": 246 assert(props is not None) 247 cap = unpack_raw10_capture(cap, props) 248 if cap["format"] == "raw12": 249 assert(props is not None) 250 cap = unpack_raw12_capture(cap, props) 251 if cap["format"] == "yuv": 252 y = cap["data"][0:w*h] 253 u = cap["data"][w*h:w*h*5/4] 254 v = cap["data"][w*h*5/4:w*h*6/4] 255 return ((y.astype(numpy.float32) / 255.0).reshape(h, w, 1), 256 (u.astype(numpy.float32) / 255.0).reshape(h/2, w/2, 1), 257 (v.astype(numpy.float32) / 255.0).reshape(h/2, w/2, 1)) 258 elif cap["format"] == "jpeg": 259 rgb = decompress_jpeg_to_rgb_image(cap["data"]).reshape(w*h*3) 260 return (rgb[::3].reshape(h,w,1), 261 rgb[1::3].reshape(h,w,1), 262 rgb[2::3].reshape(h,w,1)) 263 elif cap["format"] == "raw": 264 assert(props is not None) 265 white_level = float(props['android.sensor.info.whiteLevel']) 266 img = numpy.ndarray(shape=(h*w,), dtype='<u2', 267 buffer=cap["data"][0:w*h*2]) 268 img = img.astype(numpy.float32).reshape(h,w) / white_level 269 # Crop the raw image to the active array region. 270 if props.has_key("android.sensor.info.activeArraySize") \ 271 and props["android.sensor.info.activeArraySize"] is not None \ 272 and props.has_key("android.sensor.info.pixelArraySize") \ 273 and props["android.sensor.info.pixelArraySize"] is not None: 274 # Note that the Rect class is defined such that the left,top values 275 # are "inside" while the right,bottom values are "outside"; that is, 276 # it's inclusive of the top,left sides only. So, the width is 277 # computed as right-left, rather than right-left+1, etc. 278 wfull = props["android.sensor.info.pixelArraySize"]["width"] 279 hfull = props["android.sensor.info.pixelArraySize"]["height"] 280 xcrop = props["android.sensor.info.activeArraySize"]["left"] 281 ycrop = props["android.sensor.info.activeArraySize"]["top"] 282 wcrop = props["android.sensor.info.activeArraySize"]["right"]-xcrop 283 hcrop = props["android.sensor.info.activeArraySize"]["bottom"]-ycrop 284 assert(wfull >= wcrop >= 0) 285 assert(hfull >= hcrop >= 0) 286 assert(wfull - wcrop >= xcrop >= 0) 287 assert(hfull - hcrop >= ycrop >= 0) 288 if w == wfull and h == hfull: 289 # Crop needed; extract the center region. 290 img = img[ycrop:ycrop+hcrop,xcrop:xcrop+wcrop] 291 w = wcrop 292 h = hcrop 293 elif w == wcrop and h == hcrop: 294 # No crop needed; image is already cropped to the active array. 295 None 296 else: 297 raise its.error.Error('Invalid image size metadata') 298 # Separate the image planes. 299 imgs = [img[::2].reshape(w*h/2)[::2].reshape(h/2,w/2,1), 300 img[::2].reshape(w*h/2)[1::2].reshape(h/2,w/2,1), 301 img[1::2].reshape(w*h/2)[::2].reshape(h/2,w/2,1), 302 img[1::2].reshape(w*h/2)[1::2].reshape(h/2,w/2,1)] 303 idxs = get_canonical_cfa_order(props) 304 return [imgs[i] for i in idxs] 305 elif cap["format"] == "rawStats": 306 assert(props is not None) 307 white_level = float(props['android.sensor.info.whiteLevel']) 308 mean_image, var_image = its.image.unpack_rawstats_capture(cap) 309 idxs = get_canonical_cfa_order(props) 310 return [mean_image[:,:,i] / white_level for i in idxs] 311 else: 312 raise its.error.Error('Invalid format %s' % (cap["format"])) 313 314 315def get_canonical_cfa_order(props): 316 """Returns a mapping from the Bayer 2x2 top-left grid in the CFA to 317 the standard order R,Gr,Gb,B. 318 319 Args: 320 props: Camera properties object. 321 322 Returns: 323 List of 4 integers, corresponding to the positions in the 2x2 top- 324 left Bayer grid of R,Gr,Gb,B, where the 2x2 grid is labeled as 325 0,1,2,3 in row major order. 326 """ 327 # Note that raw streams aren't croppable, so the cropRegion doesn't need 328 # to be considered when determining the top-left pixel color. 329 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 330 if cfa_pat == 0: 331 # RGGB 332 return [0,1,2,3] 333 elif cfa_pat == 1: 334 # GRBG 335 return [1,0,3,2] 336 elif cfa_pat == 2: 337 # GBRG 338 return [2,3,0,1] 339 elif cfa_pat == 3: 340 # BGGR 341 return [3,2,1,0] 342 else: 343 raise its.error.Error("Not supported") 344 345 346def get_gains_in_canonical_order(props, gains): 347 """Reorders the gains tuple to the canonical R,Gr,Gb,B order. 348 349 Args: 350 props: Camera properties object. 351 gains: List of 4 values, in R,G_even,G_odd,B order. 352 353 Returns: 354 List of gains values, in R,Gr,Gb,B order. 355 """ 356 cfa_pat = props['android.sensor.info.colorFilterArrangement'] 357 if cfa_pat in [0,1]: 358 # RGGB or GRBG, so G_even is Gr 359 return gains 360 elif cfa_pat in [2,3]: 361 # GBRG or BGGR, so G_even is Gb 362 return [gains[0], gains[2], gains[1], gains[3]] 363 else: 364 raise its.error.Error("Not supported") 365 366 367def convert_raw_to_rgb_image(r_plane, gr_plane, gb_plane, b_plane, 368 props, cap_res): 369 """Convert a Bayer raw-16 image to an RGB image. 370 371 Includes some extremely rudimentary demosaicking and color processing 372 operations; the output of this function shouldn't be used for any image 373 quality analysis. 374 375 Args: 376 r_plane,gr_plane,gb_plane,b_plane: Numpy arrays for each color plane 377 in the Bayer image, with pixels in the [0.0, 1.0] range. 378 props: Camera properties object. 379 cap_res: Capture result (metadata) object. 380 381 Returns: 382 RGB float-3 image array, with pixel values in [0.0, 1.0] 383 """ 384 # Values required for the RAW to RGB conversion. 385 assert(props is not None) 386 white_level = float(props['android.sensor.info.whiteLevel']) 387 black_levels = props['android.sensor.blackLevelPattern'] 388 gains = cap_res['android.colorCorrection.gains'] 389 ccm = cap_res['android.colorCorrection.transform'] 390 391 # Reorder black levels and gains to R,Gr,Gb,B, to match the order 392 # of the planes. 393 black_levels = [get_black_level(i,props,cap_res) for i in range(4)] 394 gains = get_gains_in_canonical_order(props, gains) 395 396 # Convert CCM from rational to float, as numpy arrays. 397 ccm = numpy.array(its.objects.rational_to_float(ccm)).reshape(3,3) 398 399 # Need to scale the image back to the full [0,1] range after subtracting 400 # the black level from each pixel. 401 scale = white_level / (white_level - max(black_levels)) 402 403 # Three-channel black levels, normalized to [0,1] by white_level. 404 black_levels = numpy.array([b/white_level for b in [ 405 black_levels[i] for i in [0,1,3]]]) 406 407 # Three-channel gains. 408 gains = numpy.array([gains[i] for i in [0,1,3]]) 409 410 h,w = r_plane.shape[:2] 411 img = numpy.dstack([r_plane,(gr_plane+gb_plane)/2.0,b_plane]) 412 img = (((img.reshape(h,w,3) - black_levels) * scale) * gains).clip(0.0,1.0) 413 img = numpy.dot(img.reshape(w*h,3), ccm.T).reshape(h,w,3).clip(0.0,1.0) 414 return img 415 416 417def get_black_level(chan, props, cap_res=None): 418 """Return the black level to use for a given capture. 419 420 Uses a dynamic value from the capture result if available, else falls back 421 to the static global value in the camera characteristics. 422 423 Args: 424 chan: The channel index, in canonical order (R, Gr, Gb, B). 425 props: The camera properties object. 426 cap_res: A capture result object. 427 428 Returns: 429 The black level value for the specified channel. 430 """ 431 if (cap_res is not None and cap_res.has_key('android.sensor.dynamicBlackLevel') and 432 cap_res['android.sensor.dynamicBlackLevel'] is not None): 433 black_levels = cap_res['android.sensor.dynamicBlackLevel'] 434 else: 435 black_levels = props['android.sensor.blackLevelPattern'] 436 idxs = its.image.get_canonical_cfa_order(props) 437 ordered_black_levels = [black_levels[i] for i in idxs] 438 return ordered_black_levels[chan] 439 440 441def convert_yuv420_planar_to_rgb_image(y_plane, u_plane, v_plane, 442 w, h, 443 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 444 yuv_off=DEFAULT_YUV_OFFSETS): 445 """Convert a YUV420 8-bit planar image to an RGB image. 446 447 Args: 448 y_plane: The packed 8-bit Y plane. 449 u_plane: The packed 8-bit U plane. 450 v_plane: The packed 8-bit V plane. 451 w: The width of the image. 452 h: The height of the image. 453 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 454 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 455 456 Returns: 457 RGB float-3 image array, with pixel values in [0.0, 1.0]. 458 """ 459 y = numpy.subtract(y_plane, yuv_off[0]) 460 u = numpy.subtract(u_plane, yuv_off[1]).view(numpy.int8) 461 v = numpy.subtract(v_plane, yuv_off[2]).view(numpy.int8) 462 u = u.reshape(h/2, w/2).repeat(2, axis=1).repeat(2, axis=0) 463 v = v.reshape(h/2, w/2).repeat(2, axis=1).repeat(2, axis=0) 464 yuv = numpy.dstack([y, u.reshape(w*h), v.reshape(w*h)]) 465 flt = numpy.empty([h, w, 3], dtype=numpy.float32) 466 flt.reshape(w*h*3)[:] = yuv.reshape(h*w*3)[:] 467 flt = numpy.dot(flt.reshape(w*h,3), ccm_yuv_to_rgb.T).clip(0, 255) 468 rgb = numpy.empty([h, w, 3], dtype=numpy.uint8) 469 rgb.reshape(w*h*3)[:] = flt.reshape(w*h*3)[:] 470 return rgb.astype(numpy.float32) / 255.0 471 472 473def load_rgb_image(fname): 474 """Load a standard image file (JPG, PNG, etc.). 475 476 Args: 477 fname: The path of the file to load. 478 479 Returns: 480 RGB float-3 image array, with pixel values in [0.0, 1.0]. 481 """ 482 img = Image.open(fname) 483 w = img.size[0] 484 h = img.size[1] 485 a = numpy.array(img) 486 if len(a.shape) == 3 and a.shape[2] == 3: 487 # RGB 488 return a.reshape(h,w,3) / 255.0 489 elif len(a.shape) == 2 or len(a.shape) == 3 and a.shape[2] == 1: 490 # Greyscale; convert to RGB 491 return a.reshape(h*w).repeat(3).reshape(h,w,3) / 255.0 492 else: 493 raise its.error.Error('Unsupported image type') 494 495 496def load_yuv420_to_rgb_image(yuv_fname, 497 w, h, 498 layout="planar", 499 ccm_yuv_to_rgb=DEFAULT_YUV_TO_RGB_CCM, 500 yuv_off=DEFAULT_YUV_OFFSETS): 501 """Load a YUV420 image file, and return as an RGB image. 502 503 Supported layouts include "planar" and "nv21". The "yuv" formatted captures 504 returned from the device via do_capture are in the "planar" layout; other 505 layouts may only be needed for loading files from other sources. 506 507 Args: 508 yuv_fname: The path of the YUV420 file. 509 w: The width of the image. 510 h: The height of the image. 511 layout: (Optional) the layout of the YUV data (as a string). 512 ccm_yuv_to_rgb: (Optional) the 3x3 CCM to convert from YUV to RGB. 513 yuv_off: (Optional) offsets to subtract from each of Y,U,V values. 514 515 Returns: 516 RGB float-3 image array, with pixel values in [0.0, 1.0]. 517 """ 518 with open(yuv_fname, "rb") as f: 519 if layout == "planar": 520 # Plane of Y, plane of V, plane of U. 521 y = numpy.fromfile(f, numpy.uint8, w*h, "") 522 v = numpy.fromfile(f, numpy.uint8, w*h/4, "") 523 u = numpy.fromfile(f, numpy.uint8, w*h/4, "") 524 elif layout == "nv21": 525 # Plane of Y, plane of interleaved VUVUVU... 526 y = numpy.fromfile(f, numpy.uint8, w*h, "") 527 vu = numpy.fromfile(f, numpy.uint8, w*h/2, "") 528 v = vu[0::2] 529 u = vu[1::2] 530 else: 531 raise its.error.Error('Unsupported image layout') 532 return convert_yuv420_planar_to_rgb_image( 533 y,u,v,w,h,ccm_yuv_to_rgb,yuv_off) 534 535 536def load_yuv420_planar_to_yuv_planes(yuv_fname, w, h): 537 """Load a YUV420 planar image file, and return Y, U, and V plane images. 538 539 Args: 540 yuv_fname: The path of the YUV420 file. 541 w: The width of the image. 542 h: The height of the image. 543 544 Returns: 545 Separate Y, U, and V images as float-1 Numpy arrays, pixels in [0,1]. 546 Note that pixel (0,0,0) is not black, since U,V pixels are centered at 547 0.5, and also that the Y and U,V plane images returned are different 548 sizes (due to chroma subsampling in the YUV420 format). 549 """ 550 with open(yuv_fname, "rb") as f: 551 y = numpy.fromfile(f, numpy.uint8, w*h, "") 552 v = numpy.fromfile(f, numpy.uint8, w*h/4, "") 553 u = numpy.fromfile(f, numpy.uint8, w*h/4, "") 554 return ((y.astype(numpy.float32) / 255.0).reshape(h, w, 1), 555 (u.astype(numpy.float32) / 255.0).reshape(h/2, w/2, 1), 556 (v.astype(numpy.float32) / 255.0).reshape(h/2, w/2, 1)) 557 558 559def decompress_jpeg_to_rgb_image(jpeg_buffer): 560 """Decompress a JPEG-compressed image, returning as an RGB image. 561 562 Args: 563 jpeg_buffer: The JPEG stream. 564 565 Returns: 566 A numpy array for the RGB image, with pixels in [0,1]. 567 """ 568 img = Image.open(cStringIO.StringIO(jpeg_buffer)) 569 w = img.size[0] 570 h = img.size[1] 571 return numpy.array(img).reshape(h,w,3) / 255.0 572 573 574def apply_lut_to_image(img, lut): 575 """Applies a LUT to every pixel in a float image array. 576 577 Internally converts to a 16b integer image, since the LUT can work with up 578 to 16b->16b mappings (i.e. values in the range [0,65535]). The lut can also 579 have fewer than 65536 entries, however it must be sized as a power of 2 580 (and for smaller luts, the scale must match the bitdepth). 581 582 For a 16b lut of 65536 entries, the operation performed is: 583 584 lut[r * 65535] / 65535 -> r' 585 lut[g * 65535] / 65535 -> g' 586 lut[b * 65535] / 65535 -> b' 587 588 For a 10b lut of 1024 entries, the operation becomes: 589 590 lut[r * 1023] / 1023 -> r' 591 lut[g * 1023] / 1023 -> g' 592 lut[b * 1023] / 1023 -> b' 593 594 Args: 595 img: Numpy float image array, with pixel values in [0,1]. 596 lut: Numpy table encoding a LUT, mapping 16b integer values. 597 598 Returns: 599 Float image array after applying LUT to each pixel. 600 """ 601 n = len(lut) 602 if n <= 0 or n > MAX_LUT_SIZE or (n & (n - 1)) != 0: 603 raise its.error.Error('Invalid arg LUT size: %d' % (n)) 604 m = float(n-1) 605 return (lut[(img * m).astype(numpy.uint16)] / m).astype(numpy.float32) 606 607 608def apply_matrix_to_image(img, mat): 609 """Multiplies a 3x3 matrix with each float-3 image pixel. 610 611 Each pixel is considered a column vector, and is left-multiplied by 612 the given matrix: 613 614 [ ] r r' 615 [ mat ] * g -> g' 616 [ ] b b' 617 618 Args: 619 img: Numpy float image array, with pixel values in [0,1]. 620 mat: Numpy 3x3 matrix. 621 622 Returns: 623 The numpy float-3 image array resulting from the matrix mult. 624 """ 625 h = img.shape[0] 626 w = img.shape[1] 627 img2 = numpy.empty([h, w, 3], dtype=numpy.float32) 628 img2.reshape(w*h*3)[:] = (numpy.dot(img.reshape(h*w, 3), mat.T) 629 ).reshape(w*h*3)[:] 630 return img2 631 632 633def get_image_patch(img, xnorm, ynorm, wnorm, hnorm): 634 """Get a patch (tile) of an image. 635 636 Args: 637 img: Numpy float image array, with pixel values in [0,1]. 638 xnorm,ynorm,wnorm,hnorm: Normalized (in [0,1]) coords for the tile. 639 640 Returns: 641 Float image array of the patch. 642 """ 643 hfull = img.shape[0] 644 wfull = img.shape[1] 645 xtile = int(math.ceil(xnorm * wfull)) 646 ytile = int(math.ceil(ynorm * hfull)) 647 wtile = int(math.floor(wnorm * wfull)) 648 htile = int(math.floor(hnorm * hfull)) 649 if len(img.shape)==2: 650 return img[ytile:ytile+htile,xtile:xtile+wtile].copy() 651 else: 652 return img[ytile:ytile+htile,xtile:xtile+wtile,:].copy() 653 654 655def compute_image_means(img): 656 """Calculate the mean of each color channel in the image. 657 658 Args: 659 img: Numpy float image array, with pixel values in [0,1]. 660 661 Returns: 662 A list of mean values, one per color channel in the image. 663 """ 664 means = [] 665 chans = img.shape[2] 666 for i in xrange(chans): 667 means.append(numpy.mean(img[:,:,i], dtype=numpy.float64)) 668 return means 669 670 671def compute_image_variances(img): 672 """Calculate the variance of each color channel in the image. 673 674 Args: 675 img: Numpy float image array, with pixel values in [0,1]. 676 677 Returns: 678 A list of mean values, one per color channel in the image. 679 """ 680 variances = [] 681 chans = img.shape[2] 682 for i in xrange(chans): 683 variances.append(numpy.var(img[:,:,i], dtype=numpy.float64)) 684 return variances 685 686 687def compute_image_snrs(img): 688 """Calculate the SNR (db) of each color channel in the image. 689 690 Args: 691 img: Numpy float image array, with pixel values in [0,1]. 692 693 Returns: 694 A list of SNR value, one per color channel in the image. 695 """ 696 means = compute_image_means(img) 697 variances = compute_image_variances(img) 698 std_devs = [math.sqrt(v) for v in variances] 699 snr = [20 * math.log10(m/s) for m,s in zip(means, std_devs)] 700 return snr 701 702 703def compute_image_max_gradients(img): 704 """Calculate the maximum gradient of each color channel in the image. 705 706 Args: 707 img: Numpy float image array, with pixel values in [0,1]. 708 709 Returns: 710 A list of gradient max values, one per color channel in the image. 711 """ 712 grads = [] 713 chans = img.shape[2] 714 for i in xrange(chans): 715 grads.append(numpy.amax(numpy.gradient(img[:, :, i]))) 716 return grads 717 718 719def write_image(img, fname, apply_gamma=False): 720 """Save a float-3 numpy array image to a file. 721 722 Supported formats: PNG, JPEG, and others; see PIL docs for more. 723 724 Image can be 3-channel, which is interpreted as RGB, or can be 1-channel, 725 which is greyscale. 726 727 Can optionally specify that the image should be gamma-encoded prior to 728 writing it out; this should be done if the image contains linear pixel 729 values, to make the image look "normal". 730 731 Args: 732 img: Numpy image array data. 733 fname: Path of file to save to; the extension specifies the format. 734 apply_gamma: (Optional) apply gamma to the image prior to writing it. 735 """ 736 if apply_gamma: 737 img = apply_lut_to_image(img, DEFAULT_GAMMA_LUT) 738 (h, w, chans) = img.shape 739 if chans == 3: 740 Image.fromarray((img * 255.0).astype(numpy.uint8), "RGB").save(fname) 741 elif chans == 1: 742 img3 = (img * 255.0).astype(numpy.uint8).repeat(3).reshape(h,w,3) 743 Image.fromarray(img3, "RGB").save(fname) 744 else: 745 raise its.error.Error('Unsupported image type') 746 747 748def downscale_image(img, f): 749 """Shrink an image by a given integer factor. 750 751 This function computes output pixel values by averaging over rectangular 752 regions of the input image; it doesn't skip or sample pixels, and all input 753 image pixels are evenly weighted. 754 755 If the downscaling factor doesn't cleanly divide the width and/or height, 756 then the remaining pixels on the right or bottom edge are discarded prior 757 to the downscaling. 758 759 Args: 760 img: The input image as an ndarray. 761 f: The downscaling factor, which should be an integer. 762 763 Returns: 764 The new (downscaled) image, as an ndarray. 765 """ 766 h,w,chans = img.shape 767 f = int(f) 768 assert(f >= 1) 769 h = (h/f)*f 770 w = (w/f)*f 771 img = img[0:h:,0:w:,::] 772 chs = [] 773 for i in xrange(chans): 774 ch = img.reshape(h*w*chans)[i::chans].reshape(h,w) 775 ch = ch.reshape(h,w/f,f).mean(2).reshape(h,w/f) 776 ch = ch.T.reshape(w/f,h/f,f).mean(2).T.reshape(h/f,w/f) 777 chs.append(ch.reshape(h*w/(f*f))) 778 img = numpy.vstack(chs).T.reshape(h/f,w/f,chans) 779 return img 780 781 782def compute_image_sharpness(img): 783 """Calculate the sharpness of input image. 784 785 Args: 786 img: Numpy float RGB/luma image array, with pixel values in [0,1]. 787 788 Returns: 789 A sharpness estimation value based on the average of gradient magnitude. 790 Larger value means the image is sharper. 791 """ 792 chans = img.shape[2] 793 assert(chans == 1 or chans == 3) 794 if (chans == 1): 795 luma = img[:, :, 0] 796 elif (chans == 3): 797 luma = 0.299 * img[:,:,0] + 0.587 * img[:,:,1] + 0.114 * img[:,:,2] 798 799 [gy, gx] = numpy.gradient(luma) 800 return numpy.average(numpy.sqrt(gy*gy + gx*gx)) 801 802 803def normalize_img(img): 804 """Normalize the image values to between 0 and 1. 805 806 Args: 807 img: 2-D numpy array of image values 808 Returns: 809 Normalized image 810 """ 811 return (img - numpy.amin(img))/(numpy.amax(img) - numpy.amin(img)) 812 813 814def chart_located_per_argv(): 815 """Determine if chart already located outside of test. 816 817 If chart info provided, return location and size. If not, return None. 818 819 Args: 820 None 821 Returns: 822 chart_loc: float converted xnorm,ynorm,wnorm,hnorm,scale from argv text. 823 argv is of form 'chart_loc=0.45,0.45,0.1,0.1,1.0' 824 """ 825 for s in sys.argv[1:]: 826 if s[:10] == "chart_loc=" and len(s) > 10: 827 chart_loc = s[10:].split(",") 828 return map(float, chart_loc) 829 return None, None, None, None, None 830 831 832def rotate_img_per_argv(img): 833 """Rotate an image 180 degrees if "rotate" is in argv 834 835 Args: 836 img: 2-D numpy array of image values 837 Returns: 838 Rotated image 839 """ 840 img_out = img 841 if "rotate180" in sys.argv: 842 img_out = numpy.fliplr(numpy.flipud(img_out)) 843 return img_out 844 845 846def stationary_lens_cap(cam, req, fmt): 847 """Take up to NUM_TRYS caps and save the 1st one with lens stationary. 848 849 Args: 850 cam: open device session 851 req: capture request 852 fmt: format for capture 853 854 Returns: 855 capture 856 """ 857 trys = 0 858 done = False 859 reqs = [req] * NUM_FRAMES 860 while not done: 861 print 'Waiting for lens to move to correct location...' 862 cap = cam.do_capture(reqs, fmt) 863 done = (cap[NUM_FRAMES-1]['metadata']['android.lens.state'] == 0) 864 print ' status: ', done 865 trys += 1 866 if trys == NUM_TRYS: 867 raise its.error.Error('Cannot settle lens after %d trys!' % trys) 868 return cap[NUM_FRAMES-1] 869 870 871class __UnitTest(unittest.TestCase): 872 """Run a suite of unit tests on this module. 873 """ 874 875 # TODO: Add more unit tests. 876 877 def test_apply_matrix_to_image(self): 878 """Unit test for apply_matrix_to_image. 879 880 Test by using a canned set of values on a 1x1 pixel image. 881 882 [ 1 2 3 ] [ 0.1 ] [ 1.4 ] 883 [ 4 5 6 ] * [ 0.2 ] = [ 3.2 ] 884 [ 7 8 9 ] [ 0.3 ] [ 5.0 ] 885 mat x y 886 """ 887 mat = numpy.array([[1,2,3], [4,5,6], [7,8,9]]) 888 x = numpy.array([0.1,0.2,0.3]).reshape(1,1,3) 889 y = apply_matrix_to_image(x, mat).reshape(3).tolist() 890 y_ref = [1.4,3.2,5.0] 891 passed = all([math.fabs(y[i] - y_ref[i]) < 0.001 for i in xrange(3)]) 892 self.assertTrue(passed) 893 894 def test_apply_lut_to_image(self): 895 """Unit test for apply_lut_to_image. 896 897 Test by using a canned set of values on a 1x1 pixel image. The LUT will 898 simply double the value of the index: 899 900 lut[x] = 2*x 901 """ 902 lut = numpy.array([2*i for i in xrange(65536)]) 903 x = numpy.array([0.1,0.2,0.3]).reshape(1,1,3) 904 y = apply_lut_to_image(x, lut).reshape(3).tolist() 905 y_ref = [0.2,0.4,0.6] 906 passed = all([math.fabs(y[i] - y_ref[i]) < 0.001 for i in xrange(3)]) 907 self.assertTrue(passed) 908 909 def test_unpack_raw10_image(self): 910 """Unit test for unpack_raw10_image. 911 912 RAW10 bit packing format 913 bit 7 bit 6 bit 5 bit 4 bit 3 bit 2 bit 1 bit 0 914 Byte 0: P0[9] P0[8] P0[7] P0[6] P0[5] P0[4] P0[3] P0[2] 915 Byte 1: P1[9] P1[8] P1[7] P1[6] P1[5] P1[4] P1[3] P1[2] 916 Byte 2: P2[9] P2[8] P2[7] P2[6] P2[5] P2[4] P2[3] P2[2] 917 Byte 3: P3[9] P3[8] P3[7] P3[6] P3[5] P3[4] P3[3] P3[2] 918 Byte 4: P3[1] P3[0] P2[1] P2[0] P1[1] P1[0] P0[1] P0[0] 919 """ 920 # test by using a random 4x4 10-bit image 921 H = 4 922 W = 4 923 check_list = random.sample(range(0, 1024), H*W) 924 img_check = numpy.array(check_list).reshape(H, W) 925 # pack bits 926 for row_start in range(0, len(check_list), W): 927 msbs = [] 928 lsbs = "" 929 for pixel in range(W): 930 val = numpy.binary_repr(check_list[row_start+pixel], 10) 931 msbs.append(int(val[:8], base=2)) 932 lsbs = val[8:] + lsbs 933 packed = msbs 934 packed.append(int(lsbs, base=2)) 935 chunk_raw10 = numpy.array(packed, dtype="uint8").reshape(1, 5) 936 if row_start == 0: 937 img_raw10 = chunk_raw10 938 else: 939 img_raw10 = numpy.vstack((img_raw10, chunk_raw10)) 940 # unpack and check against original 941 self.assertTrue(numpy.array_equal(unpack_raw10_image(img_raw10), 942 img_check)) 943 944if __name__ == "__main__": 945 unittest.main() 946