1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.camera.processing.imagebackend; 18 19 import android.graphics.Rect; 20 import com.android.camera.debug.Log; 21 import com.android.camera.one.v2.camera2proxy.ImageProxy; 22 import com.android.camera.session.CaptureSession; 23 import com.android.camera.util.Size; 24 25 import java.nio.ByteBuffer; 26 import java.util.List; 27 import java.util.concurrent.Executor; 28 29 /** 30 * Implements the conversion of a YUV_420_888 image to subsampled image targeted 31 * toward a given resolution. The task automatically calculates the largest 32 * integer sub-sample factor that is greater than the target resolution. There 33 * are four different thumbnail types: 34 * <ol> 35 * <li>DEBUG_SQUARE_ASPECT_CIRCULAR_INSET: a center-weighted circularly cropped 36 * gradient image</li> 37 * <li>SQUARE_ASPECT_CIRCULAR_INSET: a center-weighted circularly cropped 38 * sub-sampled image</li> 39 * <li>SQUARE_ASPECT_NO_INSET: a center-weighted square cropped sub-sampled 40 * image</li> 41 * <li>MAINTAIN_ASPECT_NO_INSET: a sub-sampled image without cropping (except to 42 * maintain even values of width and height for the image</li> 43 * </ol> 44 * This task does NOT implement rotation at the byte-level, since it is best 45 * implemented when displayed at the view level. 46 */ 47 public class TaskConvertImageToRGBPreview extends TaskImageContainer { 48 public enum ThumbnailShape { 49 DEBUG_SQUARE_ASPECT_CIRCULAR_INSET, 50 SQUARE_ASPECT_CIRCULAR_INSET, 51 SQUARE_ASPECT_NO_INSET, 52 MAINTAIN_ASPECT_NO_INSET, 53 } 54 55 // 24 bit-vector to be written for images that are out of bounds. 56 public final static int OUT_OF_BOUNDS_COLOR = 0x00000000; 57 58 /** 59 * Quick n' Dirty YUV to RGB conversion 60 * <ol> 61 * <li>R = Y + 1.402V'</li> 62 * <li>G = Y - 0.344U'- 0.714V'</li> 63 * <li>B = Y + 1.770U'</li> 64 * </ol> 65 * to be calculated at compile time. 66 */ 67 public final static int SHIFT_APPROXIMATION = 8; 68 public final static double SHIFTED_BITS_AS_VALUE = (double) (1 << SHIFT_APPROXIMATION); 69 public final static int V_FACTOR_FOR_R = (int) (1.402 * SHIFTED_BITS_AS_VALUE); 70 public final static int U_FACTOR_FOR_G = (int) (-0.344 * SHIFTED_BITS_AS_VALUE); 71 public final static int V_FACTOR_FOR_G = (int) (-0.714 * SHIFTED_BITS_AS_VALUE); 72 public final static int U_FACTOR_FOR_B = (int) (1.772 * SHIFTED_BITS_AS_VALUE); 73 74 protected final static Log.Tag TAG = new Log.Tag("TaskRGBPreview"); 75 76 protected final ThumbnailShape mThumbnailShape; 77 protected final Size mTargetSize; 78 79 /** 80 * Constructor 81 * 82 * @param image Image that the computation is dependent on 83 * @param executor Executor to fire off an events 84 * @param imageTaskManager Image task manager that allows reference counting 85 * and task spawning 86 * @param captureSession Capture session that bound to this image 87 * @param targetSize Approximate viewable pixel dimensions of the desired 88 * preview Image (Resultant image may NOT be of this width) 89 * @param thumbnailShape the desired thumbnail shape for resultant image 90 * artifact 91 */ TaskConvertImageToRGBPreview(ImageToProcess image, Executor executor, ImageTaskManager imageTaskManager, ProcessingPriority processingPriority, CaptureSession captureSession, Size targetSize, ThumbnailShape thumbnailShape)92 TaskConvertImageToRGBPreview(ImageToProcess image, Executor executor, 93 ImageTaskManager imageTaskManager, ProcessingPriority processingPriority, 94 CaptureSession captureSession, Size targetSize, ThumbnailShape thumbnailShape) { 95 super(image, executor, imageTaskManager, processingPriority, captureSession); 96 mTargetSize = targetSize; 97 mThumbnailShape = thumbnailShape; 98 } 99 logWrapper(String message)100 public void logWrapper(String message) { 101 Log.v(TAG, message); 102 } 103 104 /** 105 * Return the closest minimal value of the parameter that is evenly divisible by two. 106 */ quantizeBy2(int value)107 private static int quantizeBy2(int value) { 108 return (value / 2) * 2; 109 } 110 111 /** 112 * Way to calculate the resultant image sizes of inscribed circles: 113 * colorInscribedDataCircleFromYuvImage, 114 * dummyColorInscribedDataCircleFromYuvImage, colorDataCircleFromYuvImage 115 * 116 * @param height height of the input image 117 * @param width width of the input image 118 * @return height/width of the resultant square image TODO: Refactor 119 * functions in question to return the image size as a tuple for 120 * these functions, or re-use an general purpose holder object. 121 */ inscribedCircleRadius(int width, int height)122 protected int inscribedCircleRadius(int width, int height) { 123 return (Math.min(height, width) / 2) + 1; 124 } 125 126 /** 127 * Calculates the best integer subsample from a given height and width to a 128 * target width and height It is assumed that the exact scaling will be done 129 * with the Android Bitmap framework; this subsample value is to best 130 * convert raw images into the lowest resolution raw images in visually 131 * lossless manner without changing the aspect ratio or creating subsample 132 * artifacts. 133 * 134 * @param imageSize Dimensions of the original image 135 * @param targetSize Target dimensions of the resultant image 136 * @return inscribed image as ARGB_8888 137 */ calculateBestSubsampleFactor(Size imageSize, Size targetSize)138 protected int calculateBestSubsampleFactor(Size imageSize, Size targetSize) { 139 int maxSubsample = Math.min(imageSize.getWidth() / targetSize.getWidth(), 140 imageSize.getHeight() / targetSize.getHeight()); 141 if (maxSubsample < 1) { 142 return 1; 143 } 144 145 // Make sure the resultant image width/height is divisible by 2 to 146 // account 147 // for chroma subsampled images such as YUV 148 for (int i = maxSubsample; i >= 1; i--) { 149 if (((imageSize.getWidth() % (2 * i) == 0) 150 && (imageSize.getHeight() % (2 * i) == 0))) { 151 return i; 152 } 153 } 154 155 return 1; // If all fails, don't do the subsample. 156 } 157 158 /** 159 * Calculates the memory offset of a YUV 420 plane, given the parameters of 160 * the separate YUV color planes and the fact that UV components may be 161 * subsampled by a factor of 2. 162 * 163 * @param inscribedXMin X location that you want to start sampling on the 164 * input image in terms of input pixels 165 * @param inscribedYMin Y location that you want to start sampling on the 166 * input image in terms of input pixels 167 * @param subsample Subsample factor applied to the input image 168 * @param colorSubsample Color subsample due to the YUV color space (In YUV, 169 * it's 1 for Y, 2 for UV) 170 * @param rowStride Row Stride of the color plane in terms of bytes 171 * @param pixelStride Pixel Stride of the color plane in terms of bytes 172 * @param inputHorizontalOffset Horizontal Input Offset for sampling that 173 * you wish to add in terms of input pixels 174 * @param inputVerticalOffset Vertical Input Offset for sampling that you 175 * wish to add in terms of input pixels 176 * @return value of the corresponding memory offset. 177 */ calculateMemoryOffsetFromPixelOffsets(int inscribedXMin, int inscribedYMin, int subsample, int colorSubsample, int rowStride, int pixelStride, int inputHorizontalOffset, int inputVerticalOffset)178 protected static int calculateMemoryOffsetFromPixelOffsets(int inscribedXMin, 179 int inscribedYMin, int subsample, int colorSubsample, 180 int rowStride, int pixelStride, int inputHorizontalOffset, int inputVerticalOffset) { 181 return inputVerticalOffset * (rowStride / subsample) 182 + inputHorizontalOffset * (pixelStride / subsample) 183 + (inscribedYMin / colorSubsample) * rowStride 184 + (inscribedXMin / colorSubsample) * pixelStride; 185 } 186 187 /** 188 * Converts an Android Image to a inscribed circle bitmap of ARGB_8888 in a 189 * super-optimized loop unroll. Guarantees only one subsampled pass over the 190 * YUV data. This version of the function should be used in production and 191 * also feathers the edges with 50% alpha on its edges. <br> 192 * NOTE: To get the size of the resultant bitmap, you need to call 193 * inscribedCircleRadius(w, h) outside of this function. Runs in ~10-15ms 194 * for 4K image with a subsample of 13. <br> 195 * <p> 196 * <b>Crop Treatment: </b>Since this class does a lot of memory offset 197 * calculation, it is critical that it doesn't poke strange memory locations on 198 * strange crop values. Crop is always applied before any rotation. Out-of-bound 199 * crop boundaries are accepted, but treated mathematically as intersection with 200 * the Image rectangle. If this intersection is null, the result is minimal 2x2 201 * images. 202 * <p> 203 * <b>Known Image Artifacts</b> Since this class produces bitmaps that are 204 * transient on the screen, the implementation is geared toward efficiency 205 * rather than image quality. The image created is a straight, dumb integer 206 * subsample of the YUV space with an acceptable color conversion, but w/o any 207 * sort of re-sampling. So, expect the usual aliasing noise. Furthermore, when a 208 * subsample factor of n is chosen, the resultant UV pixels will have the same 209 * subsampling, even though the RGBA artifact produces could produce an 210 * effective resample of (n/2) in the U,V color space. For cases where subsample 211 * is odd-valued, there will be pixel-to-pixel color bleeding, which may be 212 * apparent in sharp color edges. But since our eyes are pretty bad at color 213 * edges anyway, it may be an acceptable trade-off for run-time efficiency on an 214 * image artifact that has a short lifetime on the screen. 215 * </p> 216 * TODO: Implement horizontal alpha feathering of the edge of the image. 217 * 218 * @param img YUV420_888 Image to convert 219 * @param subsample width/height subsample factor 220 * @return inscribed image as ARGB_8888 221 */ colorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample)222 protected int[] colorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample) { 223 Rect defaultCrop = new Rect(0, 0, img.getWidth(), img.getHeight()); 224 225 return colorInscribedDataCircleFromYuvImage(img, defaultCrop, subsample); 226 } 227 colorInscribedDataCircleFromYuvImage(ImageProxy img, Rect crop, int subsample)228 protected int[] colorInscribedDataCircleFromYuvImage(ImageProxy img, Rect crop, int subsample) { 229 crop = guaranteedSafeCrop(img, crop); 230 final List<ImageProxy.Plane> planeList = img.getPlanes(); 231 if (planeList.size() != 3) { 232 throw new IllegalArgumentException("Incorrect number planes (" + planeList.size() 233 + ") in YUV Image Object"); 234 } 235 236 int inputWidth = crop.width(); 237 int inputHeight = crop.height(); 238 int outputWidth = inputWidth / subsample; 239 int outputHeight = inputHeight / subsample; 240 int w = outputWidth; 241 int h = outputHeight; 242 int r = inscribedCircleRadius(w, h); 243 244 final int inscribedXMin; 245 final int inscribedXMax; 246 final int inscribedYMin; 247 final int inscribedYMax; 248 // To minimize color bleeding, always quantize the start coordinates by 2. 249 final int inputVerticalOffset = quantizeBy2(crop.top); 250 final int inputHorizontalOffset = quantizeBy2(crop.left); 251 252 // Set up input read boundaries. 253 if (w > h) { 254 inscribedYMin = 0; 255 inscribedYMax = h; 256 // since we're 2x2 blocks we need to quantize these values by 2 257 inscribedXMin = quantizeBy2(w / 2 - r); 258 inscribedXMax = quantizeBy2(w / 2 + r); 259 } else { 260 inscribedXMin = 0; 261 inscribedXMax = w; 262 // since we're 2x2 blocks we need to quantize these values by 2 263 inscribedYMin = quantizeBy2(h / 2 - r); 264 inscribedYMax = quantizeBy2(h / 2 + r); 265 } 266 267 ByteBuffer buf0 = planeList.get(0).getBuffer(); 268 ByteBuffer bufU = planeList.get(1).getBuffer(); // Downsampled by 2 269 ByteBuffer bufV = planeList.get(2).getBuffer(); // Downsampled by 2 270 int yByteStride = planeList.get(0).getRowStride() * subsample; 271 int uByteStride = planeList.get(1).getRowStride() * subsample; 272 int vByteStride = planeList.get(2).getRowStride() * subsample; 273 int yPixelStride = planeList.get(0).getPixelStride() * subsample; 274 int uPixelStride = planeList.get(1).getPixelStride() * subsample; 275 int vPixelStride = planeList.get(2).getPixelStride() * subsample; 276 int outputPixelStride = r * 2; 277 int centerY = h / 2; 278 int centerX = w / 2; 279 280 int len = r * r * 4; 281 int[] colors = new int[len]; 282 int alpha = 255 << 24; 283 284 logWrapper("TIMER_BEGIN Starting Native Java YUV420-to-RGB Circular Conversion"); 285 logWrapper("\t Y-Plane Size=" + w + "x" + h); 286 logWrapper("\t U-Plane Size=" + planeList.get(1).getRowStride() + " Pixel Stride=" 287 + planeList.get(1).getPixelStride()); 288 logWrapper("\t V-Plane Size=" + planeList.get(2).getRowStride() + " Pixel Stride=" 289 + planeList.get(2).getPixelStride()); 290 // Take in vertical lines by factor of two because of the u/v component 291 // subsample 292 for (int j = inscribedYMin; j < inscribedYMax; j += 2) { 293 int offsetColor = (j - inscribedYMin) * (outputPixelStride); 294 int offsetY = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 295 1 /* YComponent */, yByteStride, yPixelStride, inputHorizontalOffset, 296 inputVerticalOffset); 297 int offsetU = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 298 2 /* U Component downsampled by 2 */, uByteStride, uPixelStride, 299 inputHorizontalOffset / 2, inputVerticalOffset / 2); 300 int offsetV = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 301 2 /* v Component downsampled by 2 */, vByteStride, vPixelStride, 302 inputHorizontalOffset / 2, inputVerticalOffset / 2); 303 304 // Parametrize the circle boundaries w.r.t. the y component. 305 // Find the subsequence of pixels we need for each horizontal raster 306 // line. 307 int circleHalfWidth0 = 308 (int) (Math.sqrt((float) (r * r - (j - centerY) * (j - centerY))) + 0.5f); 309 int circleMin0 = centerX - (circleHalfWidth0); 310 int circleMax0 = centerX + circleHalfWidth0; 311 int circleHalfWidth1 = (int) (Math.sqrt((float) (r * r - (j + 1 - centerY) 312 * (j + 1 - centerY))) + 0.5f); 313 int circleMin1 = centerX - (circleHalfWidth1); 314 int circleMax1 = centerX + circleHalfWidth1; 315 316 // Take in horizontal lines by factor of two because of the u/v 317 // component subsample 318 // and everything as 2x2 blocks. 319 for (int i = inscribedXMin; i < inscribedXMax; i += 2, offsetY += 2 * yPixelStride, 320 offsetColor += 2, offsetU += uPixelStride, offsetV += vPixelStride) { 321 // Note i and j are in terms of pixels of the subsampled image 322 // offsetY, offsetU, and offsetV are in terms of bytes of the 323 // image 324 // offsetColor, output_pixel stride are in terms of the packed 325 // output image 326 if ((i > circleMax0 && i > circleMax1) || (i + 1 < circleMin0 && i < circleMin1)) { 327 colors[offsetColor] = OUT_OF_BOUNDS_COLOR; 328 colors[offsetColor + 1] = OUT_OF_BOUNDS_COLOR; 329 colors[offsetColor + outputPixelStride] = OUT_OF_BOUNDS_COLOR; 330 colors[offsetColor + outputPixelStride + 1] = OUT_OF_BOUNDS_COLOR; 331 continue; 332 } 333 334 // calculate the RGB component of the u/v channels and use it 335 // for all pixels in the 2x2 block 336 int u = (int) (bufU.get(offsetU) & 255) - 128; 337 int v = (int) (bufV.get(offsetV) & 255) - 128; 338 int redDiff = (v * V_FACTOR_FOR_R) >> SHIFT_APPROXIMATION; 339 int greenDiff = 340 ((u * U_FACTOR_FOR_G + v * V_FACTOR_FOR_G) >> SHIFT_APPROXIMATION); 341 int blueDiff = (u * U_FACTOR_FOR_B) >> SHIFT_APPROXIMATION; 342 343 if (i > circleMax0 || i < circleMin0) { 344 colors[offsetColor] = OUT_OF_BOUNDS_COLOR; 345 } else { 346 // Do a little alpha feathering on the edges 347 int alpha00 = (i == circleMax0 || i == circleMin0) ? (128 << 24) : (255 << 24); 348 349 int y00 = (int) (buf0.get(offsetY) & 255); 350 351 int green00 = y00 + greenDiff; 352 int blue00 = y00 + blueDiff; 353 int red00 = y00 + redDiff; 354 355 // Get the railing correct 356 if (green00 < 0) { 357 green00 = 0; 358 } 359 if (red00 < 0) { 360 red00 = 0; 361 } 362 if (blue00 < 0) { 363 blue00 = 0; 364 } 365 366 if (green00 > 255) { 367 green00 = 255; 368 } 369 if (red00 > 255) { 370 red00 = 255; 371 } 372 if (blue00 > 255) { 373 blue00 = 255; 374 } 375 376 colors[offsetColor] = (red00 & 255) << 16 | (green00 & 255) << 8 377 | (blue00 & 255) | alpha00; 378 } 379 380 if (i + 1 > circleMax0 || i + 1 < circleMin0) { 381 colors[offsetColor + 1] = OUT_OF_BOUNDS_COLOR; 382 } else { 383 int alpha01 = ((i + 1) == circleMax0 || (i + 1) == circleMin0) ? (128 << 24) 384 : (255 << 24); 385 int y01 = (int) (buf0.get(offsetY + yPixelStride) & 255); 386 int green01 = y01 + greenDiff; 387 int blue01 = y01 + blueDiff; 388 int red01 = y01 + redDiff; 389 390 // Get the railing correct 391 if (green01 < 0) { 392 green01 = 0; 393 } 394 if (red01 < 0) { 395 red01 = 0; 396 } 397 if (blue01 < 0) { 398 blue01 = 0; 399 } 400 401 if (green01 > 255) { 402 green01 = 255; 403 } 404 if (red01 > 255) { 405 red01 = 255; 406 } 407 if (blue01 > 255) { 408 blue01 = 255; 409 } 410 colors[offsetColor + 1] = (red01 & 255) << 16 | (green01 & 255) << 8 411 | (blue01 & 255) | alpha01; 412 } 413 414 if (i > circleMax1 || i < circleMin1) { 415 colors[offsetColor + outputPixelStride] = OUT_OF_BOUNDS_COLOR; 416 } else { 417 int alpha10 = (i == circleMax1 || i == circleMin1) ? (128 << 24) : (255 << 24); 418 int y10 = (int) (buf0.get(offsetY + yByteStride) & 255); 419 int green10 = y10 + greenDiff; 420 int blue10 = y10 + blueDiff; 421 int red10 = y10 + redDiff; 422 423 // Get the railing correct 424 if (green10 < 0) { 425 green10 = 0; 426 } 427 if (red10 < 0) { 428 red10 = 0; 429 } 430 if (blue10 < 0) { 431 blue10 = 0; 432 } 433 if (green10 > 255) { 434 green10 = 255; 435 } 436 if (red10 > 255) { 437 red10 = 255; 438 } 439 if (blue10 > 255) { 440 blue10 = 255; 441 } 442 443 colors[offsetColor + outputPixelStride] = (red10 & 255) << 16 444 | (green10 & 255) << 8 | (blue10 & 255) | alpha10; 445 } 446 447 if (i + 1 > circleMax1 || i + 1 < circleMin1) { 448 colors[offsetColor + outputPixelStride + 1] = OUT_OF_BOUNDS_COLOR; 449 } else { 450 int alpha11 = ((i + 1) == circleMax1 || (i + 1) == circleMin1) ? (128 << 24) 451 : (255 << 24); 452 int y11 = (int) (buf0.get(offsetY + yByteStride + yPixelStride) & 255); 453 int green11 = y11 + greenDiff; 454 int blue11 = y11 + blueDiff; 455 int red11 = y11 + redDiff; 456 457 // Get the railing correct 458 if (green11 < 0) { 459 green11 = 0; 460 } 461 if (red11 < 0) { 462 red11 = 0; 463 } 464 if (blue11 < 0) { 465 blue11 = 0; 466 } 467 468 if (green11 > 255) { 469 green11 = 255; 470 } 471 472 if (red11 > 255) { 473 red11 = 255; 474 } 475 if (blue11 > 255) { 476 blue11 = 255; 477 } 478 colors[offsetColor + outputPixelStride + 1] = (red11 & 255) << 16 479 | (green11 & 255) << 8 | (blue11 & 255) | alpha11; 480 } 481 482 } 483 } 484 logWrapper("TIMER_END Starting Native Java YUV420-to-RGB Circular Conversion"); 485 486 return colors; 487 } 488 489 /** 490 * Converts an Android Image to a subsampled image of ARGB_8888 data in a 491 * super-optimized loop unroll. Guarantees only one subsampled pass over the 492 * YUV data. No crop is applied. 493 * 494 * @param img YUV420_888 Image to convert 495 * @param subsample width/height subsample factor 496 * @param enableSquareInscribe true, output is an cropped square output; 497 * false, output maintains aspect ratio of input image 498 * @return inscribed image as ARGB_8888 499 */ colorSubSampleFromYuvImage(ImageProxy img, int subsample, boolean enableSquareInscribe)500 protected int[] colorSubSampleFromYuvImage(ImageProxy img, int subsample, 501 boolean enableSquareInscribe) { 502 Rect defaultCrop = new Rect(0, 0, img.getWidth(), img.getHeight()); 503 504 return colorSubSampleFromYuvImage(img, defaultCrop, subsample, enableSquareInscribe); 505 } 506 507 /** 508 * Converts an Android Image to a subsampled image of ARGB_8888 data in a 509 * super-optimized loop unroll. Guarantees only one subsampled pass over the 510 * YUV data. 511 * <p> 512 * <b>Crop Treatment: </b>Since this class does a lot of memory offset 513 * calculation, it is critical that it doesn't poke strange memory locations on 514 * strange crop values. Crop is always applied before any rotation. Out-of-bound 515 * crop boundaries are accepted, but treated mathematically as intersection with 516 * the Image rectangle. If this intersection is null, the result is minimal 2x2 517 * images. 518 * <p> 519 * <b>Known Image Artifacts</b> Since this class produces bitmaps that are 520 * transient on the screen, the implementation is geared toward efficiency 521 * rather than image quality. The image created is a straight, dumb integer 522 * subsample of the YUV space with an acceptable color conversion, but w/o any 523 * sort of re-sampling. So, expect the usual aliasing noise. Furthermore, when a 524 * subsample factor of n is chosen, the resultant UV pixels will have the same 525 * subsampling, even though the RGBA artifact produces could produce an 526 * effective resample of (n/2) in the U,V color space. For cases where subsample 527 * is odd-valued, there will be pixel-to-pixel color bleeding, which may be 528 * apparent in sharp color edges. But since our eyes are pretty bad at color 529 * edges anyway, it may be an acceptable trade-off for run-time efficiency on an 530 * image artifact that has a short lifetime on the screen. 531 * </p> 532 * 533 * @param img YUV420_888 Image to convert 534 * @param crop crop to be applied. 535 * @param subsample width/height subsample factor 536 * @param enableSquareInscribe true, output is an cropped square output; 537 * false, output maintains aspect ratio of input image 538 * @return inscribed image as ARGB_8888 539 */ colorSubSampleFromYuvImage(ImageProxy img, Rect crop, int subsample, boolean enableSquareInscribe)540 protected int[] colorSubSampleFromYuvImage(ImageProxy img, Rect crop, int subsample, 541 boolean enableSquareInscribe) { 542 crop = guaranteedSafeCrop(img, crop); 543 final List<ImageProxy.Plane> planeList = img.getPlanes(); 544 if (planeList.size() != 3) { 545 throw new IllegalArgumentException("Incorrect number planes (" + planeList.size() 546 + ") in YUV Image Object"); 547 } 548 549 int inputWidth = crop.width(); 550 int inputHeight = crop.height(); 551 int outputWidth = inputWidth / subsample; 552 int outputHeight = inputHeight / subsample; 553 554 // Set up input read boundaries. 555 556 ByteBuffer bufY = planeList.get(0).getBuffer(); 557 ByteBuffer bufU = planeList.get(1).getBuffer(); // Downsampled by 2 558 ByteBuffer bufV = planeList.get(2).getBuffer(); // Downsampled by 2 559 int yByteStride = planeList.get(0).getRowStride() * subsample; 560 int uByteStride = planeList.get(1).getRowStride() * subsample; 561 int vByteStride = planeList.get(2).getRowStride() * subsample; 562 int yPixelStride = planeList.get(0).getPixelStride() * subsample; 563 int uPixelStride = planeList.get(1).getPixelStride() * subsample; 564 int vPixelStride = planeList.get(2).getPixelStride() * subsample; 565 566 567 // Set up default input read boundaries. 568 final int outputPixelStride; 569 final int len; 570 final int inscribedXMin; 571 final int inscribedXMax; 572 final int inscribedYMin; 573 final int inscribedYMax; 574 final int inputVerticalOffset = quantizeBy2(crop.top); 575 final int inputHorizontalOffset = quantizeBy2(crop.left); 576 577 if (enableSquareInscribe) { 578 int r = inscribedCircleRadius(outputWidth, outputHeight); 579 len = r * r * 4; 580 outputPixelStride = r * 2; 581 582 if (outputWidth > outputHeight) { 583 // since we're 2x2 blocks we need to quantize these values by 2 584 inscribedXMin = quantizeBy2(outputWidth / 2 - r); 585 inscribedXMax = quantizeBy2(outputWidth / 2 + r); 586 inscribedYMin = 0; 587 inscribedYMax = outputHeight; 588 } else { 589 inscribedXMin = 0; 590 inscribedXMax = outputWidth; 591 // since we're 2x2 blocks we need to quantize these values by 2 592 inscribedYMin = quantizeBy2(outputHeight / 2 - r); 593 inscribedYMax = quantizeBy2(outputHeight / 2 + r); 594 } 595 } else { 596 outputPixelStride = outputWidth; 597 len = outputWidth * outputHeight; 598 inscribedXMin = 0; 599 inscribedXMax = quantizeBy2(outputWidth); 600 inscribedYMin = 0; 601 inscribedYMax = quantizeBy2(outputHeight); 602 } 603 604 int[] colors = new int[len]; 605 int alpha = 255 << 24; 606 607 logWrapper("TIMER_BEGIN Starting Native Java YUV420-to-RGB Rectangular Conversion"); 608 logWrapper("\t Y-Plane Size=" + outputWidth + "x" + outputHeight); 609 logWrapper("\t U-Plane Size=" + planeList.get(1).getRowStride() + " Pixel Stride=" 610 + planeList.get(1).getPixelStride()); 611 logWrapper("\t V-Plane Size=" + planeList.get(2).getRowStride() + " Pixel Stride=" 612 + planeList.get(2).getPixelStride()); 613 // Take in vertical lines by factor of two because of the u/v component 614 // subsample 615 for (int j = inscribedYMin; j < inscribedYMax; j += 2) { 616 int offsetColor = (j - inscribedYMin) * (outputPixelStride); 617 int offsetY = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 618 1 /* YComponent */, yByteStride, yPixelStride, inputHorizontalOffset, 619 inputVerticalOffset); 620 int offsetU = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 621 2 /* U Component downsampled by 2 */, uByteStride, uPixelStride, 622 inputHorizontalOffset / 2, inputVerticalOffset / 2); 623 int offsetV = calculateMemoryOffsetFromPixelOffsets(inscribedXMin, j, subsample, 624 2 /* v Component downsampled by 2 */, vByteStride, vPixelStride, 625 inputHorizontalOffset / 2, inputVerticalOffset / 2); 626 627 // Take in horizontal lines by factor of two because of the u/v 628 // component subsample 629 // and everything as 2x2 blocks. 630 for (int i = inscribedXMin; i < inscribedXMax; i += 2, offsetY += 2 * yPixelStride, 631 offsetColor += 2, offsetU += uPixelStride, offsetV += vPixelStride) { 632 // Note i and j are in terms of pixels of the subsampled image 633 // offsetY, offsetU, and offsetV are in terms of bytes of the 634 // image 635 // offsetColor, output_pixel stride are in terms of the packed 636 // output image 637 638 // calculate the RGB component of the u/v channels and use it 639 // for all pixels in the 2x2 block 640 int u = (int) (bufU.get(offsetU) & 255) - 128; 641 int v = (int) (bufV.get(offsetV) & 255) - 128; 642 int redDiff = (v * V_FACTOR_FOR_R) >> SHIFT_APPROXIMATION; 643 int greenDiff = ((u * U_FACTOR_FOR_G + v * V_FACTOR_FOR_G) >> SHIFT_APPROXIMATION); 644 int blueDiff = (u * U_FACTOR_FOR_B) >> SHIFT_APPROXIMATION; 645 646 // Do a little alpha feathering on the edges 647 int alpha00 = (255 << 24); 648 649 int y00 = (int) (bufY.get(offsetY) & 255); 650 651 int green00 = y00 + greenDiff; 652 int blue00 = y00 + blueDiff; 653 int red00 = y00 + redDiff; 654 655 // Get the railing correct 656 if (green00 < 0) { 657 green00 = 0; 658 } 659 if (red00 < 0) { 660 red00 = 0; 661 } 662 if (blue00 < 0) { 663 blue00 = 0; 664 } 665 666 if (green00 > 255) { 667 green00 = 255; 668 } 669 if (red00 > 255) { 670 red00 = 255; 671 } 672 if (blue00 > 255) { 673 blue00 = 255; 674 } 675 676 colors[offsetColor] = (red00 & 255) << 16 | (green00 & 255) << 8 677 | (blue00 & 255) | alpha00; 678 679 int alpha01 = (255 << 24); 680 int y01 = (int) (bufY.get(offsetY + yPixelStride) & 255); 681 int green01 = y01 + greenDiff; 682 int blue01 = y01 + blueDiff; 683 int red01 = y01 + redDiff; 684 685 // Get the railing correct 686 if (green01 < 0) { 687 green01 = 0; 688 } 689 if (red01 < 0) { 690 red01 = 0; 691 } 692 if (blue01 < 0) { 693 blue01 = 0; 694 } 695 696 if (green01 > 255) { 697 green01 = 255; 698 } 699 if (red01 > 255) { 700 red01 = 255; 701 } 702 if (blue01 > 255) { 703 blue01 = 255; 704 } 705 colors[offsetColor + 1] = (red01 & 255) << 16 | (green01 & 255) << 8 706 | (blue01 & 255) | alpha01; 707 708 int alpha10 = (255 << 24); 709 int y10 = (int) (bufY.get(offsetY + yByteStride) & 255); 710 int green10 = y10 + greenDiff; 711 int blue10 = y10 + blueDiff; 712 int red10 = y10 + redDiff; 713 714 // Get the railing correct 715 if (green10 < 0) { 716 green10 = 0; 717 } 718 if (red10 < 0) { 719 red10 = 0; 720 } 721 if (blue10 < 0) { 722 blue10 = 0; 723 } 724 if (green10 > 255) { 725 green10 = 255; 726 } 727 if (red10 > 255) { 728 red10 = 255; 729 } 730 if (blue10 > 255) { 731 blue10 = 255; 732 } 733 734 colors[offsetColor + outputPixelStride] = (red10 & 255) << 16 735 | (green10 & 255) << 8 | (blue10 & 255) | alpha10; 736 737 int alpha11 = (255 << 24); 738 int y11 = (int) (bufY.get(offsetY + yByteStride + yPixelStride) & 255); 739 int green11 = y11 + greenDiff; 740 int blue11 = y11 + blueDiff; 741 int red11 = y11 + redDiff; 742 743 // Get the railing correct 744 if (green11 < 0) { 745 green11 = 0; 746 } 747 if (red11 < 0) { 748 red11 = 0; 749 } 750 if (blue11 < 0) { 751 blue11 = 0; 752 } 753 754 if (green11 > 255) { 755 green11 = 255; 756 } 757 758 if (red11 > 255) { 759 red11 = 255; 760 } 761 if (blue11 > 255) { 762 blue11 = 255; 763 } 764 colors[offsetColor + outputPixelStride + 1] = (red11 & 255) << 16 765 | (green11 & 255) << 8 | (blue11 & 255) | alpha11; 766 } 767 } 768 logWrapper("TIMER_END Starting Native Java YUV420-to-RGB Rectangular Conversion"); 769 770 return colors; 771 } 772 773 /** 774 * DEBUG IMAGE FUNCTION Converts an Android Image to a inscribed circle 775 * bitmap, currently wired to the test pattern. Will subsample and optimize 776 * the image given a target resolution. 777 * 778 * @param img YUV420_888 Image to convert 779 * @param subsample width/height subsample factor 780 * @return inscribed image as ARGB_8888 781 */ dummyColorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample)782 protected int[] dummyColorInscribedDataCircleFromYuvImage(ImageProxy img, int subsample) { 783 logWrapper("RUNNING DUMMY dummyColorInscribedDataCircleFromYuvImage"); 784 int w = img.getWidth() / subsample; 785 int h = img.getHeight() / subsample; 786 int r = inscribedCircleRadius(w, h); 787 int len = r * r * 4; 788 int[] colors = new int[len]; 789 790 // Make a fun test pattern. 791 for (int i = 0; i < len; i++) { 792 int x = i % (2 * r); 793 int y = i / (2 * r); 794 colors[i] = (255 << 24) | ((x & 255) << 16) | ((y & 255) << 8); 795 } 796 797 return colors; 798 } 799 800 /** 801 * Calculates the input Task Image specification an ImageProxy 802 * 803 * @param img Specified ImageToProcess 804 * @return Calculated specification 805 */ calculateInputImage(ImageToProcess img, Rect cropApplied)806 protected TaskImage calculateInputImage(ImageToProcess img, Rect cropApplied) { 807 return new TaskImage(img.rotation, img.proxy.getWidth(), img.proxy.getHeight(), 808 img.proxy.getFormat(), cropApplied); 809 } 810 811 /** 812 * Calculates the resultant Task Image specification, given the shape 813 * selected at the time of task construction 814 * 815 * @param img Specified image to process 816 * @param subsample Amount of subsampling to be applied 817 * @return Calculated Specification 818 */ calculateResultImage(ImageToProcess img, int subsample)819 protected TaskImage calculateResultImage(ImageToProcess img, int subsample) { 820 final Rect safeCrop = guaranteedSafeCrop(img.proxy, img.crop); 821 int resultWidth, resultHeight; 822 823 if (mThumbnailShape == ThumbnailShape.MAINTAIN_ASPECT_NO_INSET) { 824 resultWidth = safeCrop.width() / subsample; 825 resultHeight = safeCrop.height() / subsample; 826 } else { 827 final int radius = inscribedCircleRadius(safeCrop.width() / subsample, safeCrop.height() 828 / subsample); 829 resultWidth = 2 * radius; 830 resultHeight = 2 * radius; 831 } 832 833 return new TaskImage(img.rotation, resultWidth, resultHeight, 834 TaskImage.EXTRA_USER_DEFINED_FORMAT_ARGB_8888, 835 null /* Crop already applied */); 836 837 } 838 839 /** 840 * Runs the correct image conversion routine, based upon the selected 841 * thumbnail shape. 842 * 843 * @param img Image to be converted 844 * @param subsample Amount of image subsampling 845 * @return an ARGB_888 packed array ready for Bitmap conversion 846 */ runSelectedConversion(ImageProxy img, Rect crop, int subsample)847 protected int[] runSelectedConversion(ImageProxy img, Rect crop, int subsample) { 848 switch (mThumbnailShape) { 849 case DEBUG_SQUARE_ASPECT_CIRCULAR_INSET: 850 return dummyColorInscribedDataCircleFromYuvImage(img, subsample); 851 case SQUARE_ASPECT_CIRCULAR_INSET: 852 return colorInscribedDataCircleFromYuvImage(img, crop, subsample); 853 case SQUARE_ASPECT_NO_INSET: 854 return colorSubSampleFromYuvImage(img, crop, subsample, true); 855 case MAINTAIN_ASPECT_NO_INSET: 856 return colorSubSampleFromYuvImage(img, crop, subsample, false); 857 default: 858 return null; 859 } 860 } 861 862 /** 863 * Runnable implementation 864 */ 865 @Override run()866 public void run() { 867 ImageToProcess img = mImage; 868 Rect safeCrop = guaranteedSafeCrop(img.proxy, img.crop); 869 870 final TaskImage inputImage = calculateInputImage(img, safeCrop); 871 final int subsample = calculateBestSubsampleFactor( 872 new Size(safeCrop.width(), safeCrop.height()), 873 mTargetSize); 874 final TaskImage resultImage = calculateResultImage(img, subsample); 875 final int[] convertedImage; 876 877 try { 878 onStart(mId, inputImage, resultImage, TaskInfo.Destination.FAST_THUMBNAIL); 879 880 logWrapper("TIMER_END Rendering preview YUV buffer available, w=" 881 + img.proxy.getWidth() 882 / subsample + " h=" + img.proxy.getHeight() / subsample + " of subsample " 883 + subsample); 884 885 convertedImage = runSelectedConversion(img.proxy, safeCrop, subsample); 886 } finally { 887 // Signal backend that reference has been released 888 mImageTaskManager.releaseSemaphoreReference(img, mExecutor); 889 } 890 onPreviewDone(resultImage, inputImage, convertedImage, TaskInfo.Destination.FAST_THUMBNAIL); 891 } 892 893 /** 894 * Wraps the onResultUncompressed listener function 895 * 896 * @param resultImage Image specification of result image 897 * @param inputImage Image specification of the input image 898 * @param colors Uncompressed data buffer 899 * @param destination Specifies the purpose of this image processing 900 * artifact 901 */ onPreviewDone(TaskImage resultImage, TaskImage inputImage, int[] colors, TaskInfo.Destination destination)902 public void onPreviewDone(TaskImage resultImage, TaskImage inputImage, int[] colors, 903 TaskInfo.Destination destination) { 904 TaskInfo job = new TaskInfo(mId, inputImage, resultImage, destination); 905 final ImageProcessorListener listener = mImageTaskManager.getProxyListener(); 906 907 listener.onResultUncompressed(job, new UncompressedPayload(colors)); 908 } 909 910 } 911