1 /* 2 * Copyright (C) 2021 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.impl; 18 19 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_QUALITY; 20 import static android.hardware.camera2.impl.CameraExtensionUtils.JPEG_DEFAULT_ROTATION; 21 22 import android.annotation.NonNull; 23 import android.graphics.ImageFormat; 24 import android.hardware.camera2.CaptureResult; 25 import android.hardware.camera2.extension.CaptureBundle; 26 import android.hardware.camera2.extension.ICaptureProcessorImpl; 27 import android.hardware.camera2.extension.IProcessResultImpl; 28 import android.media.Image; 29 import android.media.Image.Plane; 30 import android.media.ImageReader; 31 import android.media.ImageWriter; 32 import android.os.Handler; 33 import android.os.HandlerThread; 34 import android.os.IBinder; 35 import android.os.RemoteException; 36 import android.util.Log; 37 import android.view.Surface; 38 39 import com.android.internal.camera.flags.Flags; 40 41 import java.nio.ByteBuffer; 42 import java.util.HashSet; 43 import java.util.Iterator; 44 import java.util.List; 45 import java.util.concurrent.ConcurrentLinkedQueue; 46 47 // Jpeg compress input YUV and queue back in the client target surface. 48 public class CameraExtensionJpegProcessor implements ICaptureProcessorImpl { 49 public final static String TAG = "CameraExtensionJpeg"; 50 private final static int JPEG_QUEUE_SIZE = 1; 51 private final static int JPEG_APP_SEGMENT_SIZE = 64 * 1024; 52 53 private final Handler mHandler; 54 private final HandlerThread mHandlerThread; 55 private final ICaptureProcessorImpl mProcessor; 56 57 private ImageReader mYuvReader = null; 58 private ImageReader mPostviewYuvReader = null; 59 private android.hardware.camera2.extension.Size mResolution = null; 60 private android.hardware.camera2.extension.Size mPostviewResolution = null; 61 private int mFormat = -1; 62 private int mPostviewFormat = -1; 63 private int mCaptureFormat = -1; 64 private Surface mOutputSurface = null; 65 private ImageWriter mOutputWriter = null; 66 private Surface mPostviewOutputSurface = null; 67 private ImageWriter mPostviewOutputWriter = null; 68 69 private static final class JpegParameters { 70 public HashSet<Long> mTimeStamps = new HashSet<>(); 71 public int mRotation = JPEG_DEFAULT_ROTATION; // CW multiple of 90 degrees 72 public int mQuality = JPEG_DEFAULT_QUALITY; // [0..100] 73 } 74 75 private ConcurrentLinkedQueue<JpegParameters> mJpegParameters = new ConcurrentLinkedQueue<>(); 76 CameraExtensionJpegProcessor(@onNull ICaptureProcessorImpl processor)77 public CameraExtensionJpegProcessor(@NonNull ICaptureProcessorImpl processor) { 78 mProcessor = processor; 79 mHandlerThread = new HandlerThread(TAG); 80 mHandlerThread.start(); 81 mHandler = new Handler(mHandlerThread.getLooper()); 82 } 83 close()84 public void close() { 85 mHandlerThread.quitSafely(); 86 87 if (mOutputWriter != null) { 88 mOutputWriter.close(); 89 mOutputWriter = null; 90 } 91 92 if (mYuvReader != null) { 93 mYuvReader.close(); 94 mYuvReader = null; 95 } 96 } 97 getJpegParameters(List<CaptureBundle> captureBundles)98 private static JpegParameters getJpegParameters(List<CaptureBundle> captureBundles) { 99 JpegParameters ret = new JpegParameters(); 100 if (!captureBundles.isEmpty()) { 101 // The quality and orientation settings must be equal for requests in a burst 102 103 Byte jpegQuality = captureBundles.get(0).captureResult.get(CaptureResult.JPEG_QUALITY); 104 if (jpegQuality != null) { 105 ret.mQuality = jpegQuality; 106 } else { 107 Log.w(TAG, "No jpeg quality set, using default: " + JPEG_DEFAULT_QUALITY); 108 } 109 110 Integer orientation = captureBundles.get(0).captureResult.get( 111 CaptureResult.JPEG_ORIENTATION); 112 if (orientation != null) { 113 // The jpeg encoder expects CCW rotation, convert from CW 114 ret.mRotation = (360 - (orientation % 360)) / 90; 115 } else { 116 Log.w(TAG, "No jpeg rotation set, using default: " + JPEG_DEFAULT_ROTATION); 117 } 118 119 for (CaptureBundle bundle : captureBundles) { 120 Long timeStamp = bundle.captureResult.get(CaptureResult.SENSOR_TIMESTAMP); 121 if (timeStamp != null) { 122 ret.mTimeStamps.add(timeStamp); 123 } else { 124 Log.e(TAG, "Capture bundle without valid sensor timestamp!"); 125 } 126 } 127 } 128 129 return ret; 130 } 131 132 /** 133 * Compresses a YCbCr image to jpeg, applying a crop and rotation. 134 * <p> 135 * The input is defined as a set of 3 planes of 8-bit samples, one plane for 136 * each channel of Y, Cb, Cr.<br> 137 * The Y plane is assumed to have the same width and height of the entire 138 * image.<br> 139 * The Cb and Cr planes are assumed to be downsampled by a factor of 2, to 140 * have dimensions (floor(width / 2), floor(height / 2)).<br> 141 * Each plane is specified by a direct java.nio.ByteBuffer, a pixel-stride, 142 * and a row-stride. So, the sample at coordinate (x, y) can be retrieved 143 * from byteBuffer[x * pixel_stride + y * row_stride]. 144 * <p> 145 * The pre-compression transformation is applied as follows: 146 * <ol> 147 * <li>The image is cropped to the rectangle from (cropLeft, cropTop) to 148 * (cropRight - 1, cropBottom - 1). So, a cropping-rectangle of (0, 0) - 149 * (width, height) is a no-op.</li> 150 * <li>The rotation is applied counter-clockwise relative to the coordinate 151 * space of the image, so a CCW rotation will appear CW when the image is 152 * rendered in scanline order. Only rotations which are multiples of 153 * 90-degrees are suppored, so the parameter 'rot90' specifies which 154 * multiple of 90 to rotate the image.</li> 155 * </ol> 156 * 157 * @param width the width of the image to compress 158 * @param height the height of the image to compress 159 * @param yBuf the buffer containing the Y component of the image 160 * @param yPStride the stride between adjacent pixels in the same row in 161 * yBuf 162 * @param yRStride the stride between adjacent rows in yBuf 163 * @param cbBuf the buffer containing the Cb component of the image 164 * @param cbPStride the stride between adjacent pixels in the same row in 165 * cbBuf 166 * @param cbRStride the stride between adjacent rows in cbBuf 167 * @param crBuf the buffer containing the Cr component of the image 168 * @param crPStride the stride between adjacent pixels in the same row in 169 * crBuf 170 * @param crRStride the stride between adjacent rows in crBuf 171 * @param outBuf a direct java.nio.ByteBuffer to hold the compressed jpeg. 172 * This must have enough capacity to store the result, or an 173 * error code will be returned. 174 * @param outBufCapacity the capacity of outBuf 175 * @param quality the jpeg-quality (1-100) to use 176 * @param cropLeft left-edge of the bounds of the image to crop to before 177 * rotation 178 * @param cropTop top-edge of the bounds of the image to crop to before 179 * rotation 180 * @param cropRight right-edge of the bounds of the image to crop to before 181 * rotation 182 * @param cropBottom bottom-edge of the bounds of the image to crop to 183 * before rotation 184 * @param rot90 the multiple of 90 to rotate the image CCW (after cropping) 185 */ compressJpegFromYUV420pNative( int width, int height, ByteBuffer yBuf, int yPStride, int yRStride, ByteBuffer cbBuf, int cbPStride, int cbRStride, ByteBuffer crBuf, int crPStride, int crRStride, ByteBuffer outBuf, int outBufCapacity, int quality, int cropLeft, int cropTop, int cropRight, int cropBottom, int rot90)186 private static native int compressJpegFromYUV420pNative( 187 int width, int height, 188 ByteBuffer yBuf, int yPStride, int yRStride, 189 ByteBuffer cbBuf, int cbPStride, int cbRStride, 190 ByteBuffer crBuf, int crPStride, int crRStride, 191 ByteBuffer outBuf, int outBufCapacity, 192 int quality, 193 int cropLeft, int cropTop, int cropRight, int cropBottom, 194 int rot90); 195 196 @Override process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, boolean isPostviewRequested)197 public void process(List<CaptureBundle> captureBundle, IProcessResultImpl captureCallback, 198 boolean isPostviewRequested) 199 throws RemoteException { 200 JpegParameters jpegParams = getJpegParameters(captureBundle); 201 try { 202 mJpegParameters.add(jpegParams); 203 mProcessor.process(captureBundle, captureCallback, isPostviewRequested); 204 } catch (Exception e) { 205 mJpegParameters.remove(jpegParams); 206 throw e; 207 } 208 } 209 onOutputSurface(Surface surface, int format)210 public void onOutputSurface(Surface surface, int format) throws RemoteException { 211 if (!Flags.extension10Bit() && format != ImageFormat.JPEG) { 212 Log.e(TAG, "Unsupported output format: " + format); 213 return; 214 } 215 CameraExtensionUtils.SurfaceInfo surfaceInfo = CameraExtensionUtils.querySurface(surface); 216 mCaptureFormat = surfaceInfo.mFormat; 217 mOutputSurface = surface; 218 initializePipeline(); 219 } 220 onPostviewOutputSurface(Surface surface)221 public void onPostviewOutputSurface(Surface surface) throws RemoteException { 222 CameraExtensionUtils.SurfaceInfo postviewSurfaceInfo = 223 CameraExtensionUtils.querySurface(surface); 224 if (!Flags.extension10Bit() && postviewSurfaceInfo.mFormat != ImageFormat.JPEG) { 225 Log.e(TAG, "Unsupported output format: " + postviewSurfaceInfo.mFormat); 226 return; 227 } 228 mPostviewFormat = postviewSurfaceInfo.mFormat; 229 mPostviewOutputSurface = surface; 230 initializePostviewPipeline(); 231 } 232 233 @Override onResolutionUpdate(android.hardware.camera2.extension.Size size, android.hardware.camera2.extension.Size postviewSize)234 public void onResolutionUpdate(android.hardware.camera2.extension.Size size, 235 android.hardware.camera2.extension.Size postviewSize) 236 throws RemoteException { 237 mResolution = size; 238 mPostviewResolution = postviewSize; 239 initializePipeline(); 240 } 241 onImageFormatUpdate(int format)242 public void onImageFormatUpdate(int format) throws RemoteException { 243 if (!Flags.extension10Bit() && format != ImageFormat.YUV_420_888) { 244 Log.e(TAG, "Unsupported input format: " + format); 245 return; 246 } 247 mFormat = format; 248 initializePipeline(); 249 } 250 initializePipeline()251 private void initializePipeline() throws RemoteException { 252 if ((mFormat != -1) && (mOutputSurface != null) && (mResolution != null) && 253 (mYuvReader == null)) { 254 if (Flags.extension10Bit() && mCaptureFormat == ImageFormat.YUV_420_888) { 255 // For the case when postview is JPEG and capture is YUV 256 mProcessor.onOutputSurface(mOutputSurface, mCaptureFormat); 257 } else { 258 // Jpeg/blobs are expected to be configured with (w*h)x1.5 + 64k Jpeg APP1 segment 259 mOutputWriter = ImageWriter.newInstance(mOutputSurface, 1 /*maxImages*/, 260 ImageFormat.JPEG, 261 (mResolution.width * mResolution.height * 3) / 2 262 + JPEG_APP_SEGMENT_SIZE, 1); 263 mYuvReader = ImageReader.newInstance(mResolution.width, mResolution.height, 264 mFormat, JPEG_QUEUE_SIZE); 265 mYuvReader.setOnImageAvailableListener( 266 new YuvCallback(mYuvReader, mOutputWriter), mHandler); 267 mProcessor.onOutputSurface(mYuvReader.getSurface(), mFormat); 268 } 269 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 270 mProcessor.onImageFormatUpdate(ImageFormat.YUV_420_888); 271 } 272 } 273 initializePostviewPipeline()274 private void initializePostviewPipeline() throws RemoteException { 275 if ((mFormat != -1) && (mPostviewOutputSurface != null) && (mPostviewResolution != null) 276 && (mPostviewYuvReader == null)) { 277 if (Flags.extension10Bit() && mPostviewFormat == ImageFormat.YUV_420_888) { 278 // For the case when postview is YUV and capture is JPEG 279 mProcessor.onPostviewOutputSurface(mPostviewOutputSurface); 280 } else { 281 // Jpeg/blobs are expected to be configured with (w*h)x1 282 mPostviewOutputWriter = ImageWriter.newInstance(mPostviewOutputSurface, 283 1/*maxImages*/, ImageFormat.JPEG, 284 mPostviewResolution.width * mPostviewResolution.height, 1); 285 mPostviewYuvReader = ImageReader.newInstance(mPostviewResolution.width, 286 mPostviewResolution.height, mFormat, JPEG_QUEUE_SIZE); 287 mPostviewYuvReader.setOnImageAvailableListener( 288 new YuvCallback(mPostviewYuvReader, mPostviewOutputWriter), mHandler); 289 mProcessor.onPostviewOutputSurface(mPostviewYuvReader.getSurface()); 290 } 291 mProcessor.onResolutionUpdate(mResolution, mPostviewResolution); 292 mProcessor.onImageFormatUpdate(ImageFormat.YUV_420_888); 293 } 294 } 295 296 @Override asBinder()297 public IBinder asBinder() { 298 throw new UnsupportedOperationException("Binder IPC not supported!"); 299 } 300 301 private class YuvCallback implements ImageReader.OnImageAvailableListener { 302 private ImageReader mImageReader; 303 private ImageWriter mImageWriter; 304 YuvCallback(ImageReader imageReader, ImageWriter imageWriter)305 public YuvCallback(ImageReader imageReader, ImageWriter imageWriter) { 306 mImageReader = imageReader; 307 mImageWriter = imageWriter; 308 } 309 310 @Override onImageAvailable(ImageReader reader)311 public void onImageAvailable(ImageReader reader) { 312 Image yuvImage = null; 313 Image jpegImage = null; 314 try { 315 yuvImage = mImageReader.acquireNextImage(); 316 jpegImage = mImageWriter.dequeueInputImage(); 317 } catch (IllegalStateException e) { 318 if (yuvImage != null) { 319 yuvImage.close(); 320 } 321 if (jpegImage != null) { 322 jpegImage.close(); 323 } 324 Log.e(TAG, "Failed to acquire processed yuv image or jpeg image!"); 325 return; 326 } 327 328 ByteBuffer jpegBuffer = jpegImage.getPlanes()[0].getBuffer(); 329 jpegBuffer.clear(); 330 // Jpeg/blobs are expected to be configured with (w*h)x1 331 int jpegCapacity = jpegImage.getWidth(); 332 333 Plane lumaPlane = yuvImage.getPlanes()[0]; 334 Plane crPlane = yuvImage.getPlanes()[1]; 335 Plane cbPlane = yuvImage.getPlanes()[2]; 336 337 ConcurrentLinkedQueue<JpegParameters> jpegParameters = 338 new ConcurrentLinkedQueue(mJpegParameters); 339 Iterator<JpegParameters> jpegIter = jpegParameters.iterator(); 340 JpegParameters jpegParams = null; 341 while(jpegIter.hasNext()) { 342 JpegParameters currentParams = jpegIter.next(); 343 if (currentParams.mTimeStamps.contains(yuvImage.getTimestamp())) { 344 jpegParams = currentParams; 345 jpegIter.remove(); 346 break; 347 } 348 } 349 if (jpegParams == null) { 350 if (jpegParameters.isEmpty()) { 351 Log.w(TAG, "Empty jpeg settings queue! Using default jpeg orientation" 352 + " and quality!"); 353 jpegParams = new JpegParameters(); 354 jpegParams.mRotation = JPEG_DEFAULT_ROTATION; 355 jpegParams.mQuality = JPEG_DEFAULT_QUALITY; 356 } else { 357 Log.w(TAG, "No jpeg settings found with matching timestamp for current" 358 + " processed input!"); 359 Log.w(TAG, "Using values from the top of the queue!"); 360 jpegParams = jpegParameters.poll(); 361 } 362 } 363 364 compressJpegFromYUV420pNative( 365 yuvImage.getWidth(), yuvImage.getHeight(), 366 lumaPlane.getBuffer(), lumaPlane.getPixelStride(), lumaPlane.getRowStride(), 367 crPlane.getBuffer(), crPlane.getPixelStride(), crPlane.getRowStride(), 368 cbPlane.getBuffer(), cbPlane.getPixelStride(), cbPlane.getRowStride(), 369 jpegBuffer, jpegCapacity, jpegParams.mQuality, 370 0, 0, yuvImage.getWidth(), yuvImage.getHeight(), 371 jpegParams.mRotation); 372 jpegImage.setTimestamp(yuvImage.getTimestamp()); 373 yuvImage.close(); 374 375 try { 376 mImageWriter.queueInputImage(jpegImage); 377 } catch (IllegalStateException e) { 378 Log.e(TAG, "Failed to queue encoded result!"); 379 } finally { 380 jpegImage.close(); 381 } 382 } 383 } 384 } 385