1 /* 2 * Copyright 2015 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.cts.rs; 18 19 import android.graphics.Bitmap; 20 import android.hardware.camera2.CameraCharacteristics; 21 import android.hardware.camera2.CameraMetadata; 22 import android.hardware.camera2.CaptureResult; 23 import android.hardware.camera2.params.ColorSpaceTransform; 24 import android.hardware.camera2.params.LensShadingMap; 25 import android.renderscript.Allocation; 26 import android.renderscript.Element; 27 import android.renderscript.Float3; 28 import android.renderscript.Float4; 29 import android.renderscript.Int4; 30 import android.renderscript.Matrix3f; 31 import android.renderscript.RenderScript; 32 import android.renderscript.Type; 33 34 import android.hardware.camera2.cts.ScriptC_raw_converter; 35 import android.util.Log; 36 import android.util.Rational; 37 import android.util.SparseIntArray; 38 39 import java.util.Arrays; 40 41 /** 42 * Utility class providing methods for rendering RAW16 images into other colorspaces. 43 */ 44 public class RawConverter { 45 private static final String TAG = "RawConverter"; 46 private static final boolean DEBUG = Log.isLoggable(TAG, Log.DEBUG); 47 48 /** 49 * Matrix to convert from CIE XYZ colorspace to sRGB, Bradford-adapted to D65. 50 */ 51 private static final float[] sXYZtoRGBBradford = new float[] { 52 3.1338561f, -1.6168667f, -0.4906146f, 53 -0.9787684f, 1.9161415f, 0.0334540f, 54 0.0719453f, -0.2289914f, 1.4052427f 55 }; 56 57 /** 58 * Matrix to convert from the ProPhoto RGB colorspace to CIE XYZ colorspace. 59 */ 60 private static final float[] sProPhotoToXYZ = new float[] { 61 0.797779f, 0.135213f, 0.031303f, 62 0.288000f, 0.711900f, 0.000100f, 63 0.000000f, 0.000000f, 0.825105f 64 }; 65 66 /** 67 * Matrix to convert from CIE XYZ colorspace to ProPhoto RGB colorspace. 68 */ 69 private static final float[] sXYZtoProPhoto = new float[] { 70 1.345753f, -0.255603f, -0.051025f, 71 -0.544426f, 1.508096f, 0.020472f, 72 0.000000f, 0.000000f, 1.211968f 73 }; 74 75 /** 76 * Coefficients for a 3rd order polynomial, ordered from highest to lowest power. This 77 * polynomial approximates the default tonemapping curve used for ACR3. 78 */ 79 private static final float[] DEFAULT_ACR3_TONEMAP_CURVE_COEFFS = new float[] { 80 -0.7836f, 0.8469f, 0.943f, 0.0209f 81 }; 82 83 /** 84 * The D50 whitepoint coordinates in CIE XYZ colorspace. 85 */ 86 private static final float[] D50_XYZ = new float[] { 0.9642f, 1, 0.8249f }; 87 88 /** 89 * An array containing the color temperatures for standard reference illuminants. 90 */ 91 private static final SparseIntArray sStandardIlluminants = new SparseIntArray(); 92 private static final int NO_ILLUMINANT = -1; 93 static { sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT, 6504)94 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT, 6504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D65, 6504)95 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D65, 6504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D50, 5003)96 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D50, 5003); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D55, 5503)97 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D55, 5503); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D75, 7504)98 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_D75, 7504); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_A, 2856)99 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_A, 2856); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_B, 4874)100 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_B, 4874); sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_C, 6774)101 sStandardIlluminants.append(CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_STANDARD_C, 6774); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT_FLUORESCENT, 6430)102 sStandardIlluminants.append( 103 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_DAYLIGHT_FLUORESCENT, 6430); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_COOL_WHITE_FLUORESCENT, 4230)104 sStandardIlluminants.append( 105 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_COOL_WHITE_FLUORESCENT, 4230); sStandardIlluminants.append( CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_WHITE_FLUORESCENT, 3450)106 sStandardIlluminants.append( 107 CameraMetadata.SENSOR_REFERENCE_ILLUMINANT1_WHITE_FLUORESCENT, 3450); 108 // TODO: Add the rest of the illuminants included in the LightSource EXIF tag. 109 } 110 111 /** 112 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 113 * 114 * <p> This function applies the operations roughly outlined in the Adobe DNG specification 115 * using the provided metadata about the image sensor. Sensor data for Android devices is 116 * assumed to be relatively linear, and no extra linearization step is applied here. The 117 * following operations are applied in the given order:</p> 118 * 119 * <ul> 120 * <li> 121 * Black level subtraction - the black levels given in the SENSOR_BLACK_LEVEL_PATTERN 122 * tag are subtracted from the corresponding raw pixels. 123 * </li> 124 * <li> 125 * Rescaling - each raw pixel is scaled by 1/(white level - black level). 126 * </li> 127 * <li> 128 * Lens shading correction - the interpolated gains from the gain map defined in the 129 * STATISTICS_LENS_SHADING_CORRECTION_MAP are applied to each raw pixel. 130 * </li> 131 * <li> 132 * Clipping - each raw pixel is clipped to a range of [0.0, 1.0]. 133 * </li> 134 * <li> 135 * Demosaic - the RGB channels for each pixel are retrieved from the Bayer mosaic 136 * of raw pixels using a simple bilinear-interpolation demosaicing algorithm. 137 * </li> 138 * <li> 139 * Colorspace transform to wide-gamut RGB - each pixel is mapped into a 140 * wide-gamut colorspace (in this case ProPhoto RGB is used) from the sensor 141 * colorspace. 142 * </li> 143 * <li> 144 * Tonemapping - A basic tonemapping curve using the default from ACR3 is applied 145 * (no further exposure compensation is applied here, though this could be improved). 146 * </li> 147 * <li> 148 * Colorspace transform to final RGB - each pixel is mapped into linear sRGB colorspace. 149 * </li> 150 * <li> 151 * Gamma correction - each pixel is gamma corrected using γ=2.2 to map into sRGB 152 * colorspace for viewing. 153 * </li> 154 * <li> 155 * Packing - each pixel is scaled so that each color channel has a range of [0, 255], 156 * and is packed into an Android bitmap. 157 * </li> 158 * </ul> 159 * 160 * <p> Arguments given here are assumed to come from the values for the corresponding 161 * {@link CameraCharacteristics.Key}s defined for the camera that produced this RAW16 buffer. 162 * </p> 163 * @param rs a {@link RenderScript} context to use. 164 * @param inputWidth width of the input RAW16 image in pixels. 165 * @param inputHeight height of the input RAW16 image in pixels. 166 * @param inputStride stride of the input RAW16 image in bytes. 167 * @param rawImageInput a byte array containing a RAW16 image. 168 * @param staticMetadata the {@link CameraCharacteristics} for this RAW capture. 169 * @param dynamicMetadata the {@link CaptureResult} for this RAW capture. 170 * @param outputOffsetX the offset width into the raw image of the left side of the output 171 * rectangle. 172 * @param outputOffsetY the offset height into the raw image of the top side of the output 173 * rectangle. 174 * @param argbOutput a {@link Bitmap} to output the rendered RAW image into. The height and 175 * width of this bitmap along with the output offsets are used to determine 176 * the dimensions and offset of the output rectangle contained in the RAW 177 * image to be rendered. 178 */ convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, int inputStride, byte[] rawImageInput, CameraCharacteristics staticMetadata, CaptureResult dynamicMetadata, int outputOffsetX, int outputOffsetY, Bitmap argbOutput)179 public static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 180 int inputStride, byte[] rawImageInput, CameraCharacteristics staticMetadata, 181 CaptureResult dynamicMetadata, int outputOffsetX, int outputOffsetY, 182 /*out*/Bitmap argbOutput) { 183 int cfa = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_COLOR_FILTER_ARRANGEMENT); 184 int[] blackLevelPattern = new int[4]; 185 staticMetadata.get(CameraCharacteristics.SENSOR_BLACK_LEVEL_PATTERN). 186 copyTo(blackLevelPattern, /*offset*/0); 187 int whiteLevel = staticMetadata.get(CameraCharacteristics.SENSOR_INFO_WHITE_LEVEL); 188 int ref1 = staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT1); 189 int ref2 = staticMetadata.get(CameraCharacteristics.SENSOR_REFERENCE_ILLUMINANT2); 190 float[] calib1 = new float[9]; 191 float[] calib2 = new float[9]; 192 convertColorspaceTransform( 193 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM1), calib1); 194 convertColorspaceTransform( 195 staticMetadata.get(CameraCharacteristics.SENSOR_CALIBRATION_TRANSFORM2), calib2); 196 float[] color1 = new float[9]; 197 float[] color2 = new float[9]; 198 convertColorspaceTransform( 199 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM1), color1); 200 convertColorspaceTransform( 201 staticMetadata.get(CameraCharacteristics.SENSOR_COLOR_TRANSFORM2), color2); 202 float[] forward1 = new float[9]; 203 float[] forward2 = new float[9]; 204 convertColorspaceTransform( 205 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX1), forward1); 206 convertColorspaceTransform( 207 staticMetadata.get(CameraCharacteristics.SENSOR_FORWARD_MATRIX2), forward2); 208 209 Rational[] neutral = dynamicMetadata.get(CaptureResult.SENSOR_NEUTRAL_COLOR_POINT); 210 211 LensShadingMap shadingMap = dynamicMetadata.get(CaptureResult.STATISTICS_LENS_SHADING_CORRECTION_MAP); 212 213 convertToSRGB(rs, inputWidth, inputHeight, inputStride, cfa, blackLevelPattern, whiteLevel, 214 rawImageInput, ref1, ref2, calib1, calib2, color1, color2, 215 forward1, forward2, neutral, shadingMap, outputOffsetX, outputOffsetY, argbOutput); 216 } 217 218 /** 219 * Convert a RAW16 buffer into an sRGB buffer, and write the result into a bitmap. 220 * 221 * @see #convertToSRGB 222 */ convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, int inputStride, int cfa, int[] blackLevelPattern, int whiteLevel, byte[] rawImageInput, int referenceIlluminant1, int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, float[] colorMatrix1, float[] colorMatrix2, float[] forwardTransform1, float[] forwardTransform2, Rational[ ] neutralColorPoint, LensShadingMap lensShadingMap, int outputOffsetX, int outputOffsetY, Bitmap argbOutput)223 private static void convertToSRGB(RenderScript rs, int inputWidth, int inputHeight, 224 int inputStride, int cfa, int[] blackLevelPattern, int whiteLevel, byte[] rawImageInput, 225 int referenceIlluminant1, int referenceIlluminant2, float[] calibrationTransform1, 226 float[] calibrationTransform2, float[] colorMatrix1, float[] colorMatrix2, 227 float[] forwardTransform1, float[] forwardTransform2, Rational[/*3*/] neutralColorPoint, 228 LensShadingMap lensShadingMap, int outputOffsetX, int outputOffsetY, 229 /*out*/Bitmap argbOutput) { 230 231 // Validate arguments 232 if (argbOutput == null || rs == null || rawImageInput == null) { 233 throw new IllegalArgumentException("Null argument to convertToSRGB"); 234 } 235 if (argbOutput.getConfig() != Bitmap.Config.ARGB_8888) { 236 throw new IllegalArgumentException( 237 "Output bitmap passed to convertToSRGB is not ARGB_8888 format"); 238 } 239 if (outputOffsetX < 0 || outputOffsetY < 0) { 240 throw new IllegalArgumentException("Negative offset passed to convertToSRGB"); 241 } 242 if ((inputStride / 2) < inputWidth) { 243 throw new IllegalArgumentException("Stride too small."); 244 } 245 if ((inputStride % 2) != 0) { 246 throw new IllegalArgumentException("Invalid stride for RAW16 format, see graphics.h."); 247 } 248 int outWidth = argbOutput.getWidth(); 249 int outHeight = argbOutput.getHeight(); 250 if (outWidth + outputOffsetX > inputWidth || outHeight + outputOffsetY > inputHeight) { 251 throw new IllegalArgumentException("Raw image with dimensions (w=" + inputWidth + 252 ", h=" + inputHeight + "), cannot converted into sRGB image with dimensions (w=" 253 + outWidth + ", h=" + outHeight + ")."); 254 } 255 if (cfa < 0 || cfa > 3) { 256 throw new IllegalArgumentException("Unsupported cfa pattern " + cfa + " used."); 257 } 258 if (DEBUG) { 259 Log.d(TAG, "Metadata Used:"); 260 Log.d(TAG, "Input width,height: " + inputWidth + "," + inputHeight); 261 Log.d(TAG, "Output offset x,y: " + outputOffsetX + "," + outputOffsetY); 262 Log.d(TAG, "Output width,height: " + outWidth + "," + outHeight); 263 Log.d(TAG, "CFA: " + cfa); 264 Log.d(TAG, "BlackLevelPattern: " + Arrays.toString(blackLevelPattern)); 265 Log.d(TAG, "WhiteLevel: " + whiteLevel); 266 Log.d(TAG, "ReferenceIlluminant1: " + referenceIlluminant1); 267 Log.d(TAG, "ReferenceIlluminant2: " + referenceIlluminant2); 268 Log.d(TAG, "CalibrationTransform1: " + Arrays.toString(calibrationTransform1)); 269 Log.d(TAG, "CalibrationTransform2: " + Arrays.toString(calibrationTransform2)); 270 Log.d(TAG, "ColorMatrix1: " + Arrays.toString(colorMatrix1)); 271 Log.d(TAG, "ColorMatrix2: " + Arrays.toString(colorMatrix2)); 272 Log.d(TAG, "ForwardTransform1: " + Arrays.toString(forwardTransform1)); 273 Log.d(TAG, "ForwardTransform2: " + Arrays.toString(forwardTransform2)); 274 Log.d(TAG, "NeutralColorPoint: " + Arrays.toString(neutralColorPoint)); 275 } 276 277 Allocation gainMap = null; 278 if (lensShadingMap != null) { 279 float[] lsm = new float[lensShadingMap.getGainFactorCount()]; 280 lensShadingMap.copyGainFactors(/*inout*/lsm, /*offset*/0); 281 gainMap = createFloat4Allocation(rs, lsm, lensShadingMap.getColumnCount(), 282 lensShadingMap.getRowCount()); 283 } 284 285 float[] normalizedForwardTransform1 = Arrays.copyOf(forwardTransform1, 286 forwardTransform1.length); 287 normalizeFM(normalizedForwardTransform1); 288 float[] normalizedForwardTransform2 = Arrays.copyOf(forwardTransform2, 289 forwardTransform2.length); 290 normalizeFM(normalizedForwardTransform2); 291 292 float[] normalizedColorMatrix1 = Arrays.copyOf(colorMatrix1, colorMatrix1.length); 293 normalizeCM(normalizedColorMatrix1); 294 float[] normalizedColorMatrix2 = Arrays.copyOf(colorMatrix2, colorMatrix2.length); 295 normalizeCM(normalizedColorMatrix2); 296 297 if (DEBUG) { 298 Log.d(TAG, "Normalized ForwardTransform1: " + Arrays.toString(normalizedForwardTransform1)); 299 Log.d(TAG, "Normalized ForwardTransform2: " + Arrays.toString(normalizedForwardTransform2)); 300 Log.d(TAG, "Normalized ColorMatrix1: " + Arrays.toString(normalizedColorMatrix1)); 301 Log.d(TAG, "Normalized ColorMatrix2: " + Arrays.toString(normalizedColorMatrix2)); 302 } 303 304 // Calculate full sensor colorspace to sRGB colorspace transform. 305 double interpolationFactor = findDngInterpolationFactor(referenceIlluminant1, 306 referenceIlluminant2, calibrationTransform1, calibrationTransform2, 307 normalizedColorMatrix1, normalizedColorMatrix2, neutralColorPoint); 308 if (DEBUG) Log.d(TAG, "Interpolation factor used: " + interpolationFactor); 309 float[] sensorToXYZ = new float[9]; 310 calculateCameraToXYZD50Transform(normalizedForwardTransform1, normalizedForwardTransform2, 311 calibrationTransform1, calibrationTransform2, neutralColorPoint, 312 interpolationFactor, /*out*/sensorToXYZ); 313 if (DEBUG) Log.d(TAG, "CameraToXYZ xform used: " + Arrays.toString(sensorToXYZ)); 314 float[] sensorToProPhoto = new float[9]; 315 multiply(sXYZtoProPhoto, sensorToXYZ, /*out*/sensorToProPhoto); 316 if (DEBUG) Log.d(TAG, "CameraToIntemediate xform used: " + Arrays.toString(sensorToProPhoto)); 317 Allocation output = Allocation.createFromBitmap(rs, argbOutput); 318 319 float[] proPhotoToSRGB = new float[9]; 320 multiply(sXYZtoRGBBradford, sProPhotoToXYZ, /*out*/proPhotoToSRGB); 321 322 // Setup input allocation (16-bit raw pixels) 323 Type.Builder typeBuilder = new Type.Builder(rs, Element.U16(rs)); 324 typeBuilder.setX((inputStride / 2)); 325 typeBuilder.setY(inputHeight); 326 Type inputType = typeBuilder.create(); 327 Allocation input = Allocation.createTyped(rs, inputType); 328 input.copyFromUnchecked(rawImageInput); 329 330 // Setup RS kernel globals 331 ScriptC_raw_converter converterKernel = new ScriptC_raw_converter(rs); 332 converterKernel.set_inputRawBuffer(input); 333 converterKernel.set_whiteLevel(whiteLevel); 334 converterKernel.set_sensorToIntermediate(new Matrix3f(transpose(sensorToProPhoto))); 335 converterKernel.set_intermediateToSRGB(new Matrix3f(transpose(proPhotoToSRGB))); 336 converterKernel.set_offsetX(outputOffsetX); 337 converterKernel.set_offsetY(outputOffsetY); 338 converterKernel.set_rawHeight(inputHeight); 339 converterKernel.set_rawWidth(inputWidth); 340 converterKernel.set_neutralPoint(new Float3(neutralColorPoint[0].floatValue(), 341 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue())); 342 converterKernel.set_toneMapCoeffs(new Float4(DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[0], 343 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[1], DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[2], 344 DEFAULT_ACR3_TONEMAP_CURVE_COEFFS[3])); 345 converterKernel.set_hasGainMap(gainMap != null); 346 if (gainMap != null) { 347 converterKernel.set_gainMap(gainMap); 348 converterKernel.set_gainMapWidth(lensShadingMap.getColumnCount()); 349 converterKernel.set_gainMapHeight(lensShadingMap.getRowCount()); 350 } 351 352 converterKernel.set_cfaPattern(cfa); 353 converterKernel.set_blackLevelPattern(new Int4(blackLevelPattern[0], 354 blackLevelPattern[1], blackLevelPattern[2], blackLevelPattern[3])); 355 converterKernel.forEach_convert_RAW_To_ARGB(output); 356 output.copyTo(argbOutput); // Force RS sync with bitmap (does not do an extra copy). 357 } 358 359 /** 360 * Create a float-backed renderscript {@link Allocation} with the given dimensions, containing 361 * the contents of the given float array. 362 * 363 * @param rs a {@link RenderScript} context to use. 364 * @param fArray the float array to copy into the {@link Allocation}. 365 * @param width the width of the {@link Allocation}. 366 * @param height the height of the {@link Allocation}. 367 * @return an {@link Allocation} containing the given floats. 368 */ createFloat4Allocation(RenderScript rs, float[] fArray, int width, int height)369 private static Allocation createFloat4Allocation(RenderScript rs, float[] fArray, 370 int width, int height) { 371 if (fArray.length != width * height * 4) { 372 throw new IllegalArgumentException("Invalid float array of length " + fArray.length + 373 ", must be correct size for Allocation of dimensions " + width + "x" + height); 374 } 375 Type.Builder builder = new Type.Builder(rs, Element.F32_4(rs)); 376 builder.setX(width); 377 builder.setY(height); 378 Allocation fAlloc = Allocation.createTyped(rs, builder.create()); 379 fAlloc.copyFrom(fArray); 380 return fAlloc; 381 } 382 383 /** 384 * Calculate the correlated color temperature (CCT) for a given x,y chromaticity in CIE 1931 x,y 385 * chromaticity space using McCamy's cubic approximation algorithm given in: 386 * 387 * McCamy, Calvin S. (April 1992). 388 * "Correlated color temperature as an explicit function of chromaticity coordinates". 389 * Color Research & Application 17 (2): 142–144 390 * 391 * @param x x chromaticity component. 392 * @param y y chromaticity component. 393 * 394 * @return the CCT associated with this chromaticity coordinate. 395 */ calculateColorTemperature(double x, double y)396 private static double calculateColorTemperature(double x, double y) { 397 double n = (x - 0.332) / (y - 0.1858); 398 return -449 * Math.pow(n, 3) + 3525 * Math.pow(n, 2) - 6823.3 * n + 5520.33; 399 } 400 401 /** 402 * Calculate the x,y chromaticity coordinates in CIE 1931 x,y chromaticity space from the given 403 * CIE XYZ coordinates. 404 * 405 * @param X the CIE XYZ X coordinate. 406 * @param Y the CIE XYZ Y coordinate. 407 * @param Z the CIE XYZ Z coordinate. 408 * 409 * @return the [x, y] chromaticity coordinates as doubles. 410 */ calculateCIExyCoordinates(double X, double Y, double Z)411 private static double[] calculateCIExyCoordinates(double X, double Y, double Z) { 412 double[] ret = new double[] { 0, 0 }; 413 ret[0] = X / (X + Y + Z); 414 ret[1] = Y / (X + Y + Z); 415 return ret; 416 } 417 418 /** 419 * Linearly interpolate between a and b given fraction f. 420 * 421 * @param a first term to interpolate between, a will be returned when f == 0. 422 * @param b second term to interpolate between, b will be returned when f == 1. 423 * @param f the fraction to interpolate by. 424 * 425 * @return interpolated result as double. 426 */ lerp(double a, double b, double f)427 private static double lerp(double a, double b, double f) { 428 return (a * (1.0f - f)) + (b * f); 429 } 430 431 /** 432 * Linearly interpolate between 3x3 matrices a and b given fraction f. 433 * 434 * @param a first 3x3 matrix to interpolate between, a will be returned when f == 0. 435 * @param b second 3x3 matrix to interpolate between, b will be returned when f == 1. 436 * @param f the fraction to interpolate by. 437 * @param result will be set to contain the interpolated matrix. 438 */ lerp(float[] a, float[] b, double f, float[] result)439 private static void lerp(float[] a, float[] b, double f, /*out*/float[] result) { 440 for (int i = 0; i < 9; i++) { 441 result[i] = (float) lerp(a[i], b[i], f); 442 } 443 } 444 445 /** 446 * Convert a 9x9 {@link ColorSpaceTransform} to a matrix and write the matrix into the 447 * output. 448 * 449 * @param xform a {@link ColorSpaceTransform} to transform. 450 * @param output the 3x3 matrix to overwrite. 451 */ convertColorspaceTransform(ColorSpaceTransform xform, float[] output)452 private static void convertColorspaceTransform(ColorSpaceTransform xform, /*out*/float[] output) { 453 for (int i = 0; i < 3; i++) { 454 for (int j = 0; j < 3; j++) { 455 output[i * 3 + j] = xform.getElement(j, i).floatValue(); 456 } 457 } 458 } 459 460 /** 461 * Find the interpolation factor to use with the RAW matrices given a neutral color point. 462 * 463 * @param referenceIlluminant1 first reference illuminant. 464 * @param referenceIlluminant2 second reference illuminant. 465 * @param calibrationTransform1 calibration matrix corresponding to the first reference 466 * illuminant. 467 * @param calibrationTransform2 calibration matrix corresponding to the second reference 468 * illuminant. 469 * @param colorMatrix1 color matrix corresponding to the first reference illuminant. 470 * @param colorMatrix2 color matrix corresponding to the second reference illuminant. 471 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 472 * 473 * @return the interpolation factor corresponding to the given neutral color point. 474 */ findDngInterpolationFactor(int referenceIlluminant1, int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, float[] colorMatrix1, float[] colorMatrix2, Rational[ ] neutralColorPoint)475 private static double findDngInterpolationFactor(int referenceIlluminant1, 476 int referenceIlluminant2, float[] calibrationTransform1, float[] calibrationTransform2, 477 float[] colorMatrix1, float[] colorMatrix2, Rational[/*3*/] neutralColorPoint) { 478 479 int colorTemperature1 = sStandardIlluminants.get(referenceIlluminant1, NO_ILLUMINANT); 480 if (colorTemperature1 == NO_ILLUMINANT) { 481 throw new IllegalArgumentException("No such illuminant for reference illuminant 1: " + 482 referenceIlluminant1); 483 } 484 485 int colorTemperature2 = sStandardIlluminants.get(referenceIlluminant2, NO_ILLUMINANT); 486 if (colorTemperature2 == NO_ILLUMINANT) { 487 throw new IllegalArgumentException("No such illuminant for reference illuminant 2: " + 488 referenceIlluminant2); 489 } 490 491 if (DEBUG) Log.d(TAG, "ColorTemperature1: " + colorTemperature1); 492 if (DEBUG) Log.d(TAG, "ColorTemperature2: " + colorTemperature2); 493 494 double interpFactor = 0.5; // Initial guess for interpolation factor 495 double oldInterpFactor = interpFactor; 496 497 double lastDiff = Double.MAX_VALUE; 498 double tolerance = 0.0001; 499 float[] XYZToCamera1 = new float[9]; 500 float[] XYZToCamera2 = new float[9]; 501 multiply(calibrationTransform1, colorMatrix1, /*out*/XYZToCamera1); 502 multiply(calibrationTransform2, colorMatrix2, /*out*/XYZToCamera2); 503 504 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 505 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 506 507 float[] neutralGuess = new float[3]; 508 float[] interpXYZToCamera = new float[9]; 509 float[] interpXYZToCameraInverse = new float[9]; 510 511 512 double lower = Math.min(colorTemperature1, colorTemperature2); 513 double upper = Math.max(colorTemperature1, colorTemperature2); 514 515 if(DEBUG) { 516 Log.d(TAG, "XYZtoCamera1: " + Arrays.toString(XYZToCamera1)); 517 Log.d(TAG, "XYZtoCamera2: " + Arrays.toString(XYZToCamera2)); 518 Log.d(TAG, "Finding interpolation factor, initial guess 0.5..."); 519 } 520 // Iteratively guess xy value, find new CCT, and update interpolation factor. 521 int loopLimit = 30; 522 int count = 0; 523 while (lastDiff > tolerance && loopLimit > 0) { 524 if (DEBUG) Log.d(TAG, "Loop count " + count); 525 lerp(XYZToCamera1, XYZToCamera2, interpFactor, interpXYZToCamera); 526 if (!invert(interpXYZToCamera, /*out*/interpXYZToCameraInverse)) { 527 throw new IllegalArgumentException( 528 "Cannot invert XYZ to Camera matrix, input matrices are invalid."); 529 } 530 531 map(interpXYZToCameraInverse, cameraNeutral, /*out*/neutralGuess); 532 double[] xy = calculateCIExyCoordinates(neutralGuess[0], neutralGuess[1], 533 neutralGuess[2]); 534 535 double colorTemperature = calculateColorTemperature(xy[0], xy[1]); 536 537 if (colorTemperature <= lower) { 538 interpFactor = 1; 539 } else if (colorTemperature >= upper) { 540 interpFactor = 0; 541 } else { 542 double invCT = 1.0 / colorTemperature; 543 interpFactor = (invCT - 1.0 / upper) / ( 1.0 / lower - 1.0 / upper); 544 } 545 546 if (lower == colorTemperature1) { 547 interpFactor = 1.0 - interpFactor; 548 } 549 550 interpFactor = (interpFactor + oldInterpFactor) / 2; 551 lastDiff = Math.abs(oldInterpFactor - interpFactor); 552 oldInterpFactor = interpFactor; 553 loopLimit--; 554 count++; 555 556 if (DEBUG) { 557 Log.d(TAG, "CameraToXYZ chosen: " + Arrays.toString(interpXYZToCameraInverse)); 558 Log.d(TAG, "XYZ neutral color guess: " + Arrays.toString(neutralGuess)); 559 Log.d(TAG, "xy coordinate: " + Arrays.toString(xy)); 560 Log.d(TAG, "xy color temperature: " + colorTemperature); 561 Log.d(TAG, "New interpolation factor: " + interpFactor); 562 } 563 } 564 565 if (loopLimit == 0) { 566 Log.w(TAG, "Could not converge on interpolation factor, using factor " + interpFactor + 567 " with remaining error factor of " + lastDiff); 568 } 569 return interpFactor; 570 } 571 572 /** 573 * Calculate the transform from the raw camera sensor colorspace to CIE XYZ colorspace with a 574 * D50 whitepoint. 575 * 576 * @param forwardTransform1 forward transform matrix corresponding to the first reference 577 * illuminant. 578 * @param forwardTransform2 forward transform matrix corresponding to the second reference 579 * illuminant. 580 * @param calibrationTransform1 calibration transform matrix corresponding to the first 581 * reference illuminant. 582 * @param calibrationTransform2 calibration transform matrix corresponding to the second 583 * reference illuminant. 584 * @param neutralColorPoint the neutral color point used to calculate the interpolation factor. 585 * @param interpolationFactor the interpolation factor to use for the forward and 586 * calibration transforms. 587 * @param outputTransform set to the full sensor to XYZ colorspace transform. 588 */ calculateCameraToXYZD50Transform(float[] forwardTransform1, float[] forwardTransform2, float[] calibrationTransform1, float[] calibrationTransform2, Rational[ ] neutralColorPoint, double interpolationFactor, float[] outputTransform)589 private static void calculateCameraToXYZD50Transform(float[] forwardTransform1, 590 float[] forwardTransform2, float[] calibrationTransform1, float[] calibrationTransform2, 591 Rational[/*3*/] neutralColorPoint, double interpolationFactor, 592 /*out*/float[] outputTransform) { 593 float[] cameraNeutral = new float[] { neutralColorPoint[0].floatValue(), 594 neutralColorPoint[1].floatValue(), neutralColorPoint[2].floatValue()}; 595 if (DEBUG) Log.d(TAG, "Camera neutral: " + Arrays.toString(cameraNeutral)); 596 597 float[] interpolatedCC = new float[9]; 598 lerp(calibrationTransform1, calibrationTransform2, interpolationFactor, 599 interpolatedCC); 600 float[] inverseInterpolatedCC = new float[9]; 601 if (!invert(interpolatedCC, /*out*/inverseInterpolatedCC)) { 602 throw new IllegalArgumentException( "Cannot invert interpolated calibration transform" + 603 ", input matrices are invalid."); 604 } 605 if (DEBUG) Log.d(TAG, "Inverted interpolated CalibrationTransform: " + 606 Arrays.toString(inverseInterpolatedCC)); 607 608 float[] referenceNeutral = new float[3]; 609 map(inverseInterpolatedCC, cameraNeutral, /*out*/referenceNeutral); 610 if (DEBUG) Log.d(TAG, "Reference neutral: " + Arrays.toString(referenceNeutral)); 611 float maxNeutral = Math.max(Math.max(referenceNeutral[0], referenceNeutral[1]), 612 referenceNeutral[2]); 613 float[] D = new float[] { maxNeutral/referenceNeutral[0], 0, 0, 614 0, maxNeutral/referenceNeutral[1], 0, 615 0, 0, maxNeutral/referenceNeutral[2] }; 616 if (DEBUG) Log.d(TAG, "Reference Neutral Diagonal: " + Arrays.toString(D)); 617 618 float[] intermediate = new float[9]; 619 float[] intermediate2 = new float[9]; 620 621 lerp(forwardTransform1, forwardTransform2, interpolationFactor, /*out*/intermediate); 622 if (DEBUG) Log.d(TAG, "Interpolated ForwardTransform: " + Arrays.toString(intermediate)); 623 624 multiply(D, inverseInterpolatedCC, /*out*/intermediate2); 625 multiply(intermediate, intermediate2, /*out*/outputTransform); 626 } 627 628 /** 629 * Map a 3d column vector using the given matrix. 630 * 631 * @param matrix float array containing 3x3 matrix to map vector by. 632 * @param input 3 dimensional vector to map. 633 * @param output 3 dimensional vector result. 634 */ map(float[] matrix, float[] input, float[] output)635 private static void map(float[] matrix, float[] input, /*out*/float[] output) { 636 output[0] = input[0] * matrix[0] + input[1] * matrix[1] + input[2] * matrix[2]; 637 output[1] = input[0] * matrix[3] + input[1] * matrix[4] + input[2] * matrix[5]; 638 output[2] = input[0] * matrix[6] + input[1] * matrix[7] + input[2] * matrix[8]; 639 } 640 641 /** 642 * Multiply two 3x3 matrices together: A * B 643 * 644 * @param a left matrix. 645 * @param b right matrix. 646 */ multiply(float[] a, float[] b, float[] output)647 private static void multiply(float[] a, float[] b, /*out*/float[] output) { 648 output[0] = a[0] * b[0] + a[1] * b[3] + a[2] * b[6]; 649 output[3] = a[3] * b[0] + a[4] * b[3] + a[5] * b[6]; 650 output[6] = a[6] * b[0] + a[7] * b[3] + a[8] * b[6]; 651 output[1] = a[0] * b[1] + a[1] * b[4] + a[2] * b[7]; 652 output[4] = a[3] * b[1] + a[4] * b[4] + a[5] * b[7]; 653 output[7] = a[6] * b[1] + a[7] * b[4] + a[8] * b[7]; 654 output[2] = a[0] * b[2] + a[1] * b[5] + a[2] * b[8]; 655 output[5] = a[3] * b[2] + a[4] * b[5] + a[5] * b[8]; 656 output[8] = a[6] * b[2] + a[7] * b[5] + a[8] * b[8]; 657 } 658 659 /** 660 * Transpose a 3x3 matrix in-place. 661 * 662 * @param m the matrix to transpose. 663 * @return the transposed matrix. 664 */ transpose( float[ ] m)665 private static float[] transpose(/*inout*/float[/*9*/] m) { 666 float t = m[1]; 667 m[1] = m[3]; 668 m[3] = t; 669 t = m[2]; 670 m[2] = m[6]; 671 m[6] = t; 672 t = m[5]; 673 m[5] = m[7]; 674 m[7] = t; 675 return m; 676 } 677 678 /** 679 * Invert a 3x3 matrix, or return false if the matrix is singular. 680 * 681 * @param m matrix to invert. 682 * @param output set the output to be the inverse of m. 683 */ invert(float[] m, float[] output)684 private static boolean invert(float[] m, /*out*/float[] output) { 685 double a00 = m[0]; 686 double a01 = m[1]; 687 double a02 = m[2]; 688 double a10 = m[3]; 689 double a11 = m[4]; 690 double a12 = m[5]; 691 double a20 = m[6]; 692 double a21 = m[7]; 693 double a22 = m[8]; 694 695 double t00 = a11 * a22 - a21 * a12; 696 double t01 = a21 * a02 - a01 * a22; 697 double t02 = a01 * a12 - a11 * a02; 698 double t10 = a20 * a12 - a10 * a22; 699 double t11 = a00 * a22 - a20 * a02; 700 double t12 = a10 * a02 - a00 * a12; 701 double t20 = a10 * a21 - a20 * a11; 702 double t21 = a20 * a01 - a00 * a21; 703 double t22 = a00 * a11 - a10 * a01; 704 705 double det = a00 * t00 + a01 * t10 + a02 * t20; 706 if (Math.abs(det) < 1e-9) { 707 return false; // Inverse too close to zero, not invertible. 708 } 709 710 output[0] = (float) (t00 / det); 711 output[1] = (float) (t01 / det); 712 output[2] = (float) (t02 / det); 713 output[3] = (float) (t10 / det); 714 output[4] = (float) (t11 / det); 715 output[5] = (float) (t12 / det); 716 output[6] = (float) (t20 / det); 717 output[7] = (float) (t21 / det); 718 output[8] = (float) (t22 / det); 719 return true; 720 } 721 722 /** 723 * Scale each element in a matrix by the given scaling factor. 724 * 725 * @param factor factor to scale by. 726 * @param matrix the float array containing a 3x3 matrix to scale. 727 */ scale(float factor, float[] matrix)728 private static void scale(float factor, /*inout*/float[] matrix) { 729 for (int i = 0; i < 9; i++) { 730 matrix[i] *= factor; 731 } 732 } 733 734 /** 735 * Clamp a value to a given range. 736 * 737 * @param low lower bound to clamp to. 738 * @param high higher bound to clamp to. 739 * @param value the value to clamp. 740 * @return the clamped value. 741 */ clamp(double low, double high, double value)742 private static double clamp(double low, double high, double value) { 743 return Math.max(low, Math.min(high, value)); 744 } 745 746 /** 747 * Return the max float in the array. 748 * 749 * @param array array of floats to search. 750 * @return max float in the array. 751 */ max(float[] array)752 private static float max(float[] array) { 753 float val = array[0]; 754 for (float f : array) { 755 val = (f > val) ? f : val; 756 } 757 return val; 758 } 759 760 /** 761 * Normalize ColorMatrix to eliminate headroom for input space scaled to [0, 1] using 762 * the D50 whitepoint. This maps the D50 whitepoint into the colorspace used by the 763 * ColorMatrix, then uses the resulting whitepoint to renormalize the ColorMatrix so 764 * that the channel values in the resulting whitepoint for this operation are clamped 765 * to the range [0, 1]. 766 * 767 * @param colorMatrix a 3x3 matrix containing a DNG ColorMatrix to be normalized. 768 */ normalizeCM( float[] colorMatrix)769 private static void normalizeCM(/*inout*/float[] colorMatrix) { 770 float[] tmp = new float[3]; 771 map(colorMatrix, D50_XYZ, /*out*/tmp); 772 float maxVal = max(tmp); 773 if (maxVal > 0) { 774 scale(1.0f / maxVal, colorMatrix); 775 } 776 } 777 778 /** 779 * Normalize ForwardMatrix to ensure that sensor whitepoint [1, 1, 1] maps to D50 in CIE XYZ 780 * colorspace. 781 * 782 * @param forwardMatrix a 3x3 matrix containing a DNG ForwardTransform to be normalized. 783 */ normalizeFM( float[] forwardMatrix)784 private static void normalizeFM(/*inout*/float[] forwardMatrix) { 785 float[] tmp = new float[] {1, 1, 1}; 786 float[] xyz = new float[3]; 787 map(forwardMatrix, tmp, /*out*/xyz); 788 789 float[] intermediate = new float[9]; 790 float[] m = new float[] {1.0f / xyz[0], 0, 0, 0, 1.0f / xyz[1], 0, 0, 0, 1.0f / xyz[2]}; 791 792 multiply(m, forwardMatrix, /*out*/ intermediate); 793 float[] m2 = new float[] {D50_XYZ[0], 0, 0, 0, D50_XYZ[1], 0, 0, 0, D50_XYZ[2]}; 794 multiply(m2, intermediate, /*out*/forwardMatrix); 795 } 796 } 797