1 /* 2 * Copyright (C) 2014 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.hardware.camera2.legacy; 18 19 import android.graphics.ImageFormat; 20 import android.graphics.SurfaceTexture; 21 import android.hardware.Camera; 22 import android.hardware.camera2.CameraCharacteristics; 23 import android.hardware.camera2.CaptureRequest; 24 import android.hardware.camera2.impl.CameraDeviceImpl; 25 import android.hardware.camera2.params.StreamConfigurationMap; 26 import android.hardware.camera2.utils.LongParcelable; 27 import android.hardware.camera2.utils.SizeAreaComparator; 28 import android.hardware.camera2.impl.CameraMetadataNative; 29 import android.os.ConditionVariable; 30 import android.os.Handler; 31 import android.os.Message; 32 import android.os.SystemClock; 33 import android.util.Log; 34 import android.util.MutableLong; 35 import android.util.Pair; 36 import android.util.Size; 37 import android.view.Surface; 38 39 import java.io.IOException; 40 import java.util.ArrayList; 41 import java.util.Arrays; 42 import java.util.Collection; 43 import java.util.Collections; 44 import java.util.Iterator; 45 import java.util.List; 46 import java.util.concurrent.TimeUnit; 47 import java.util.concurrent.atomic.AtomicBoolean; 48 49 import static com.android.internal.util.Preconditions.*; 50 51 /** 52 * This class executes requests to the {@link Camera}. 53 * 54 * <p> 55 * The main components of this class are: 56 * - A message queue of requests to the {@link Camera}. 57 * - A thread that consumes requests to the {@link Camera} and executes them. 58 * - A {@link GLThreadManager} that draws to the configured output {@link Surface}s. 59 * - An {@link CameraDeviceState} state machine that manages the callbacks for various operations. 60 * </p> 61 */ 62 @SuppressWarnings("deprecation") 63 public class RequestThreadManager { 64 private final String TAG; 65 private final int mCameraId; 66 private final RequestHandlerThread mRequestThread; 67 68 private static final boolean DEBUG = Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.DEBUG); 69 // For slightly more spammy messages that will get repeated every frame 70 private static final boolean VERBOSE = 71 Log.isLoggable(LegacyCameraDevice.DEBUG_PROP, Log.VERBOSE); 72 private Camera mCamera; 73 private final CameraCharacteristics mCharacteristics; 74 75 private final CameraDeviceState mDeviceState; 76 private final CaptureCollector mCaptureCollector; 77 private final LegacyFocusStateMapper mFocusStateMapper; 78 private final LegacyFaceDetectMapper mFaceDetectMapper; 79 80 private static final int MSG_CONFIGURE_OUTPUTS = 1; 81 private static final int MSG_SUBMIT_CAPTURE_REQUEST = 2; 82 private static final int MSG_CLEANUP = 3; 83 84 private static final int MAX_IN_FLIGHT_REQUESTS = 2; 85 86 private static final int PREVIEW_FRAME_TIMEOUT = 1000; // ms 87 private static final int JPEG_FRAME_TIMEOUT = 4000; // ms (same as CTS for API2) 88 private static final int REQUEST_COMPLETE_TIMEOUT = JPEG_FRAME_TIMEOUT; // ms (same as JPEG timeout) 89 90 private static final float ASPECT_RATIO_TOLERANCE = 0.01f; 91 private boolean mPreviewRunning = false; 92 93 private final List<Surface> mPreviewOutputs = new ArrayList<>(); 94 private final List<Surface> mCallbackOutputs = new ArrayList<>(); 95 private GLThreadManager mGLThreadManager; 96 private SurfaceTexture mPreviewTexture; 97 private Camera.Parameters mParams; 98 99 private final List<Long> mJpegSurfaceIds = new ArrayList<>(); 100 101 private Size mIntermediateBufferSize; 102 103 private final RequestQueue mRequestQueue = new RequestQueue(mJpegSurfaceIds); 104 private LegacyRequest mLastRequest = null; 105 private SurfaceTexture mDummyTexture; 106 private Surface mDummySurface; 107 108 private final Object mIdleLock = new Object(); 109 private final FpsCounter mPrevCounter = new FpsCounter("Incoming Preview"); 110 private final FpsCounter mRequestCounter = new FpsCounter("Incoming Requests"); 111 112 private final AtomicBoolean mQuit = new AtomicBoolean(false); 113 114 // Stuff JPEGs into HAL_PIXEL_FORMAT_RGBA_8888 gralloc buffers to get around SW write 115 // limitations for (b/17379185). 116 private static final boolean USE_BLOB_FORMAT_OVERRIDE = true; 117 118 /** 119 * Container object for Configure messages. 120 */ 121 private static class ConfigureHolder { 122 public final ConditionVariable condition; 123 public final Collection<Pair<Surface, Size>> surfaces; 124 ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, Size>> surfaces)125 public ConfigureHolder(ConditionVariable condition, Collection<Pair<Surface, 126 Size>> surfaces) { 127 this.condition = condition; 128 this.surfaces = surfaces; 129 } 130 } 131 132 /** 133 * Counter class used to calculate and log the current FPS of frame production. 134 */ 135 public static class FpsCounter { 136 //TODO: Hook this up to SystTrace? 137 private static final String TAG = "FpsCounter"; 138 private int mFrameCount = 0; 139 private long mLastTime = 0; 140 private long mLastPrintTime = 0; 141 private double mLastFps = 0; 142 private final String mStreamType; 143 private static final long NANO_PER_SECOND = 1000000000; //ns 144 FpsCounter(String streamType)145 public FpsCounter(String streamType) { 146 mStreamType = streamType; 147 } 148 countFrame()149 public synchronized void countFrame() { 150 mFrameCount++; 151 long nextTime = SystemClock.elapsedRealtimeNanos(); 152 if (mLastTime == 0) { 153 mLastTime = nextTime; 154 } 155 if (nextTime > mLastTime + NANO_PER_SECOND) { 156 long elapsed = nextTime - mLastTime; 157 mLastFps = mFrameCount * (NANO_PER_SECOND / (double) elapsed); 158 mFrameCount = 0; 159 mLastTime = nextTime; 160 } 161 } 162 checkFps()163 public synchronized double checkFps() { 164 return mLastFps; 165 } 166 staggeredLog()167 public synchronized void staggeredLog() { 168 if (mLastTime > mLastPrintTime + 5 * NANO_PER_SECOND) { 169 mLastPrintTime = mLastTime; 170 Log.d(TAG, "FPS for " + mStreamType + " stream: " + mLastFps ); 171 } 172 } 173 countAndLog()174 public synchronized void countAndLog() { 175 countFrame(); 176 staggeredLog(); 177 } 178 } 179 /** 180 * Fake preview for jpeg captures when there is no active preview 181 */ createDummySurface()182 private void createDummySurface() { 183 if (mDummyTexture == null || mDummySurface == null) { 184 mDummyTexture = new SurfaceTexture(/*ignored*/0); 185 // TODO: use smallest default sizes 186 mDummyTexture.setDefaultBufferSize(640, 480); 187 mDummySurface = new Surface(mDummyTexture); 188 } 189 } 190 191 private final Camera.ErrorCallback mErrorCallback = new Camera.ErrorCallback() { 192 @Override 193 public void onError(int i, Camera camera) { 194 Log.e(TAG, "Received error " + i + " from the Camera1 ErrorCallback"); 195 mDeviceState.setError(CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 196 } 197 }; 198 199 private final ConditionVariable mReceivedJpeg = new ConditionVariable(false); 200 201 private final Camera.PictureCallback mJpegCallback = new Camera.PictureCallback() { 202 @Override 203 public void onPictureTaken(byte[] data, Camera camera) { 204 Log.i(TAG, "Received jpeg."); 205 Pair<RequestHolder, Long> captureInfo = mCaptureCollector.jpegProduced(); 206 if (captureInfo == null || captureInfo.first == null) { 207 Log.e(TAG, "Dropping jpeg frame."); 208 return; 209 } 210 RequestHolder holder = captureInfo.first; 211 long timestamp = captureInfo.second; 212 for (Surface s : holder.getHolderTargets()) { 213 try { 214 if (LegacyCameraDevice.containsSurfaceId(s, mJpegSurfaceIds)) { 215 Log.i(TAG, "Producing jpeg buffer..."); 216 217 int totalSize = data.length + LegacyCameraDevice.nativeGetJpegFooterSize(); 218 totalSize = (totalSize + 3) & ~0x3; // round up to nearest octonibble 219 LegacyCameraDevice.setNextTimestamp(s, timestamp); 220 221 if (USE_BLOB_FORMAT_OVERRIDE) { 222 // Override to RGBA_8888 format. 223 LegacyCameraDevice.setSurfaceFormat(s, 224 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 225 226 int dimen = (int) Math.ceil(Math.sqrt(totalSize)); 227 dimen = (dimen + 0xf) & ~0xf; // round up to nearest multiple of 16 228 LegacyCameraDevice.setSurfaceDimens(s, dimen, dimen); 229 LegacyCameraDevice.produceFrame(s, data, dimen, dimen, 230 CameraMetadataNative.NATIVE_JPEG_FORMAT); 231 } else { 232 LegacyCameraDevice.setSurfaceDimens(s, totalSize, /*height*/1); 233 LegacyCameraDevice.produceFrame(s, data, totalSize, /*height*/1, 234 CameraMetadataNative.NATIVE_JPEG_FORMAT); 235 } 236 } 237 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 238 Log.w(TAG, "Surface abandoned, dropping frame. ", e); 239 } 240 } 241 242 mReceivedJpeg.open(); 243 } 244 }; 245 246 private final Camera.ShutterCallback mJpegShutterCallback = new Camera.ShutterCallback() { 247 @Override 248 public void onShutter() { 249 mCaptureCollector.jpegCaptured(SystemClock.elapsedRealtimeNanos()); 250 } 251 }; 252 253 private final SurfaceTexture.OnFrameAvailableListener mPreviewCallback = 254 new SurfaceTexture.OnFrameAvailableListener() { 255 @Override 256 public void onFrameAvailable(SurfaceTexture surfaceTexture) { 257 if (DEBUG) { 258 mPrevCounter.countAndLog(); 259 } 260 mGLThreadManager.queueNewFrame(); 261 } 262 }; 263 stopPreview()264 private void stopPreview() { 265 if (VERBOSE) { 266 Log.v(TAG, "stopPreview - preview running? " + mPreviewRunning); 267 } 268 if (mPreviewRunning) { 269 mCamera.stopPreview(); 270 mPreviewRunning = false; 271 } 272 } 273 startPreview()274 private void startPreview() { 275 if (VERBOSE) { 276 Log.v(TAG, "startPreview - preview running? " + mPreviewRunning); 277 } 278 if (!mPreviewRunning) { 279 // XX: CameraClient:;startPreview is not getting called after a stop 280 mCamera.startPreview(); 281 mPreviewRunning = true; 282 } 283 } 284 doJpegCapturePrepare(RequestHolder request)285 private void doJpegCapturePrepare(RequestHolder request) throws IOException { 286 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare - preview running? " + mPreviewRunning); 287 288 if (!mPreviewRunning) { 289 if (DEBUG) Log.d(TAG, "doJpegCapture - create fake surface"); 290 291 createDummySurface(); 292 mCamera.setPreviewTexture(mDummyTexture); 293 startPreview(); 294 } 295 } 296 doJpegCapture(RequestHolder request)297 private void doJpegCapture(RequestHolder request) { 298 if (DEBUG) Log.d(TAG, "doJpegCapturePrepare"); 299 300 mCamera.takePicture(mJpegShutterCallback, /*raw*/null, mJpegCallback); 301 mPreviewRunning = false; 302 } 303 doPreviewCapture(RequestHolder request)304 private void doPreviewCapture(RequestHolder request) throws IOException { 305 if (VERBOSE) { 306 Log.v(TAG, "doPreviewCapture - preview running? " + mPreviewRunning); 307 } 308 309 if (mPreviewRunning) { 310 return; // Already running 311 } 312 313 if (mPreviewTexture == null) { 314 throw new IllegalStateException( 315 "Preview capture called with no preview surfaces configured."); 316 } 317 318 mPreviewTexture.setDefaultBufferSize(mIntermediateBufferSize.getWidth(), 319 mIntermediateBufferSize.getHeight()); 320 mCamera.setPreviewTexture(mPreviewTexture); 321 322 startPreview(); 323 } 324 configureOutputs(Collection<Pair<Surface, Size>> outputs)325 private void configureOutputs(Collection<Pair<Surface, Size>> outputs) { 326 if (DEBUG) { 327 String outputsStr = outputs == null ? "null" : (outputs.size() + " surfaces"); 328 Log.d(TAG, "configureOutputs with " + outputsStr); 329 } 330 331 try { 332 stopPreview(); 333 } catch (RuntimeException e) { 334 Log.e(TAG, "Received device exception in configure call: ", e); 335 mDeviceState.setError( 336 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 337 return; 338 } 339 340 /* 341 * Try to release the previous preview's surface texture earlier if we end up 342 * using a different one; this also reduces the likelihood of getting into a deadlock 343 * when disconnecting from the old previous texture at a later time. 344 */ 345 try { 346 mCamera.setPreviewTexture(/*surfaceTexture*/null); 347 } catch (IOException e) { 348 Log.w(TAG, "Failed to clear prior SurfaceTexture, may cause GL deadlock: ", e); 349 } catch (RuntimeException e) { 350 Log.e(TAG, "Received device exception in configure call: ", e); 351 mDeviceState.setError( 352 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 353 return; 354 } 355 356 if (mGLThreadManager != null) { 357 mGLThreadManager.waitUntilStarted(); 358 mGLThreadManager.ignoreNewFrames(); 359 mGLThreadManager.waitUntilIdle(); 360 } 361 resetJpegSurfaceFormats(mCallbackOutputs); 362 mPreviewOutputs.clear(); 363 mCallbackOutputs.clear(); 364 mJpegSurfaceIds.clear(); 365 mPreviewTexture = null; 366 367 List<Size> previewOutputSizes = new ArrayList<>(); 368 List<Size> callbackOutputSizes = new ArrayList<>(); 369 370 int facing = mCharacteristics.get(CameraCharacteristics.LENS_FACING); 371 int orientation = mCharacteristics.get(CameraCharacteristics.SENSOR_ORIENTATION); 372 if (outputs != null) { 373 for (Pair<Surface, Size> outPair : outputs) { 374 Surface s = outPair.first; 375 Size outSize = outPair.second; 376 try { 377 int format = LegacyCameraDevice.detectSurfaceType(s); 378 LegacyCameraDevice.setSurfaceOrientation(s, facing, orientation); 379 switch (format) { 380 case CameraMetadataNative.NATIVE_JPEG_FORMAT: 381 if (USE_BLOB_FORMAT_OVERRIDE) { 382 // Override to RGBA_8888 format. 383 LegacyCameraDevice.setSurfaceFormat(s, 384 LegacyMetadataMapper.HAL_PIXEL_FORMAT_RGBA_8888); 385 } 386 mJpegSurfaceIds.add(LegacyCameraDevice.getSurfaceId(s)); 387 mCallbackOutputs.add(s); 388 callbackOutputSizes.add(outSize); 389 break; 390 default: 391 mPreviewOutputs.add(s); 392 previewOutputSizes.add(outSize); 393 break; 394 } 395 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 396 Log.w(TAG, "Surface abandoned, skipping...", e); 397 } 398 } 399 } 400 try { 401 mParams = mCamera.getParameters(); 402 } catch (RuntimeException e) { 403 Log.e(TAG, "Received device exception: ", e); 404 mDeviceState.setError( 405 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 406 return; 407 } 408 409 List<int[]> supportedFpsRanges = mParams.getSupportedPreviewFpsRange(); 410 int[] bestRange = getPhotoPreviewFpsRange(supportedFpsRanges); 411 if (DEBUG) { 412 Log.d(TAG, "doPreviewCapture - Selected range [" + 413 bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX] + "," + 414 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX] + "]"); 415 } 416 mParams.setPreviewFpsRange(bestRange[Camera.Parameters.PREVIEW_FPS_MIN_INDEX], 417 bestRange[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]); 418 419 if (previewOutputSizes.size() > 0) { 420 421 Size largestOutput = SizeAreaComparator.findLargestByArea(previewOutputSizes); 422 423 // Find largest jpeg dimension - assume to have the same aspect ratio as sensor. 424 Size largestJpegDimen = ParameterUtils.getLargestSupportedJpegSizeByArea(mParams); 425 426 List<Size> supportedPreviewSizes = ParameterUtils.convertSizeList( 427 mParams.getSupportedPreviewSizes()); 428 429 // Use smallest preview dimension with same aspect ratio as sensor that is >= than all 430 // of the configured output dimensions. If none exists, fall back to using the largest 431 // supported preview size. 432 long largestOutputArea = largestOutput.getHeight() * (long) largestOutput.getWidth(); 433 Size bestPreviewDimen = SizeAreaComparator.findLargestByArea(supportedPreviewSizes); 434 for (Size s : supportedPreviewSizes) { 435 long currArea = s.getWidth() * s.getHeight(); 436 long bestArea = bestPreviewDimen.getWidth() * bestPreviewDimen.getHeight(); 437 if (checkAspectRatiosMatch(largestJpegDimen, s) && (currArea < bestArea && 438 currArea >= largestOutputArea)) { 439 bestPreviewDimen = s; 440 } 441 } 442 443 mIntermediateBufferSize = bestPreviewDimen; 444 mParams.setPreviewSize(mIntermediateBufferSize.getWidth(), 445 mIntermediateBufferSize.getHeight()); 446 447 if (DEBUG) { 448 Log.d(TAG, "Intermediate buffer selected with dimens: " + 449 bestPreviewDimen.toString()); 450 } 451 } else { 452 mIntermediateBufferSize = null; 453 if (DEBUG) { 454 Log.d(TAG, "No Intermediate buffer selected, no preview outputs were configured"); 455 } 456 } 457 458 Size smallestSupportedJpegSize = calculatePictureSize(mCallbackOutputs, 459 callbackOutputSizes, mParams); 460 if (smallestSupportedJpegSize != null) { 461 /* 462 * Set takePicture size to the smallest supported JPEG size large enough 463 * to scale/crop out of for the bounding rectangle of the configured JPEG sizes. 464 */ 465 466 Log.i(TAG, "configureOutputs - set take picture size to " + smallestSupportedJpegSize); 467 mParams.setPictureSize( 468 smallestSupportedJpegSize.getWidth(), smallestSupportedJpegSize.getHeight()); 469 } 470 471 // TODO: Detect and optimize single-output paths here to skip stream teeing. 472 if (mGLThreadManager == null) { 473 mGLThreadManager = new GLThreadManager(mCameraId, facing, mDeviceState); 474 mGLThreadManager.start(); 475 } 476 mGLThreadManager.waitUntilStarted(); 477 List<Pair<Surface, Size>> previews = new ArrayList<>(); 478 Iterator<Size> previewSizeIter = previewOutputSizes.iterator(); 479 for (Surface p : mPreviewOutputs) { 480 previews.add(new Pair<>(p, previewSizeIter.next())); 481 } 482 mGLThreadManager.setConfigurationAndWait(previews, mCaptureCollector); 483 mGLThreadManager.allowNewFrames(); 484 mPreviewTexture = mGLThreadManager.getCurrentSurfaceTexture(); 485 if (mPreviewTexture != null) { 486 mPreviewTexture.setOnFrameAvailableListener(mPreviewCallback); 487 } 488 489 try { 490 mCamera.setParameters(mParams); 491 } catch (RuntimeException e) { 492 Log.e(TAG, "Received device exception while configuring: ", e); 493 mDeviceState.setError( 494 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 495 496 } 497 } 498 resetJpegSurfaceFormats(Collection<Surface> surfaces)499 private void resetJpegSurfaceFormats(Collection<Surface> surfaces) { 500 if (!USE_BLOB_FORMAT_OVERRIDE || surfaces == null) { 501 return; 502 } 503 for(Surface s : surfaces) { 504 try { 505 LegacyCameraDevice.setSurfaceFormat(s, LegacyMetadataMapper.HAL_PIXEL_FORMAT_BLOB); 506 } catch (LegacyExceptionUtils.BufferQueueAbandonedException e) { 507 Log.w(TAG, "Surface abandoned, skipping...", e); 508 } 509 } 510 } 511 512 /** 513 * Find a JPEG size (that is supported by the legacy camera device) which is equal to or larger 514 * than all of the configured {@code JPEG} outputs (by both width and height). 515 * 516 * <p>If multiple supported JPEG sizes are larger, select the smallest of them which 517 * still satisfies the above constraint.</p> 518 * 519 * <p>As a result, the returned size is guaranteed to be usable without needing 520 * to upscale any of the outputs. If only one {@code JPEG} surface is used, 521 * then no scaling/cropping is necessary between the taken picture and 522 * the {@code JPEG} output surface.</p> 523 * 524 * @param callbackOutputs a non-{@code null} list of {@code Surface}s with any image formats 525 * @param params api1 parameters (used for reading only) 526 * 527 * @return a size large enough to fit all of the configured {@code JPEG} outputs, or 528 * {@code null} if the {@code callbackOutputs} did not have any {@code JPEG} 529 * surfaces. 530 */ calculatePictureSize( List<Surface> callbackOutputs, List<Size> callbackSizes, Camera.Parameters params)531 private Size calculatePictureSize( List<Surface> callbackOutputs, 532 List<Size> callbackSizes, Camera.Parameters params) { 533 /* 534 * Find the largest JPEG size (if any), from the configured outputs: 535 * - the api1 picture size should be set to the smallest legal size that's at least as large 536 * as the largest configured JPEG size 537 */ 538 if (callbackOutputs.size() != callbackSizes.size()) { 539 throw new IllegalStateException("Input collections must be same length"); 540 } 541 List<Size> configuredJpegSizes = new ArrayList<>(); 542 Iterator<Size> sizeIterator = callbackSizes.iterator(); 543 for (Surface callbackSurface : callbackOutputs) { 544 Size jpegSize = sizeIterator.next(); 545 if (!LegacyCameraDevice.containsSurfaceId(callbackSurface, mJpegSurfaceIds)) { 546 continue; // Ignore non-JPEG callback formats 547 } 548 549 configuredJpegSizes.add(jpegSize); 550 } 551 if (!configuredJpegSizes.isEmpty()) { 552 /* 553 * Find the largest configured JPEG width, and height, independently 554 * of the rest. 555 * 556 * The rest of the JPEG streams can be cropped out of this smallest bounding 557 * rectangle. 558 */ 559 int maxConfiguredJpegWidth = -1; 560 int maxConfiguredJpegHeight = -1; 561 for (Size jpegSize : configuredJpegSizes) { 562 maxConfiguredJpegWidth = jpegSize.getWidth() > maxConfiguredJpegWidth ? 563 jpegSize.getWidth() : maxConfiguredJpegWidth; 564 maxConfiguredJpegHeight = jpegSize.getHeight() > maxConfiguredJpegHeight ? 565 jpegSize.getHeight() : maxConfiguredJpegHeight; 566 } 567 Size smallestBoundJpegSize = new Size(maxConfiguredJpegWidth, maxConfiguredJpegHeight); 568 569 List<Size> supportedJpegSizes = ParameterUtils.convertSizeList( 570 params.getSupportedPictureSizes()); 571 572 /* 573 * Find the smallest supported JPEG size that can fit the smallest bounding 574 * rectangle for the configured JPEG sizes. 575 */ 576 List<Size> candidateSupportedJpegSizes = new ArrayList<>(); 577 for (Size supportedJpegSize : supportedJpegSizes) { 578 if (supportedJpegSize.getWidth() >= maxConfiguredJpegWidth && 579 supportedJpegSize.getHeight() >= maxConfiguredJpegHeight) { 580 candidateSupportedJpegSizes.add(supportedJpegSize); 581 } 582 } 583 584 if (candidateSupportedJpegSizes.isEmpty()) { 585 throw new AssertionError( 586 "Could not find any supported JPEG sizes large enough to fit " + 587 smallestBoundJpegSize); 588 } 589 590 Size smallestSupportedJpegSize = Collections.min(candidateSupportedJpegSizes, 591 new SizeAreaComparator()); 592 593 if (!smallestSupportedJpegSize.equals(smallestBoundJpegSize)) { 594 Log.w(TAG, 595 String.format( 596 "configureOutputs - Will need to crop picture %s into " 597 + "smallest bound size %s", 598 smallestSupportedJpegSize, smallestBoundJpegSize)); 599 } 600 601 return smallestSupportedJpegSize; 602 } 603 604 return null; 605 } 606 checkAspectRatiosMatch(Size a, Size b)607 private static boolean checkAspectRatiosMatch(Size a, Size b) { 608 float aAspect = a.getWidth() / (float) a.getHeight(); 609 float bAspect = b.getWidth() / (float) b.getHeight(); 610 611 return Math.abs(aAspect - bAspect) < ASPECT_RATIO_TOLERANCE; 612 } 613 614 // Calculate the highest FPS range supported getPhotoPreviewFpsRange(List<int[]> frameRates)615 private int[] getPhotoPreviewFpsRange(List<int[]> frameRates) { 616 if (frameRates.size() == 0) { 617 Log.e(TAG, "No supported frame rates returned!"); 618 return null; 619 } 620 621 int bestMin = 0; 622 int bestMax = 0; 623 int bestIndex = 0; 624 int index = 0; 625 for (int[] rate : frameRates) { 626 int minFps = rate[Camera.Parameters.PREVIEW_FPS_MIN_INDEX]; 627 int maxFps = rate[Camera.Parameters.PREVIEW_FPS_MAX_INDEX]; 628 if (maxFps > bestMax || (maxFps == bestMax && minFps > bestMin)) { 629 bestMin = minFps; 630 bestMax = maxFps; 631 bestIndex = index; 632 } 633 index++; 634 } 635 636 return frameRates.get(bestIndex); 637 } 638 639 private final Handler.Callback mRequestHandlerCb = new Handler.Callback() { 640 private boolean mCleanup = false; 641 private final LegacyResultMapper mMapper = new LegacyResultMapper(); 642 643 @Override 644 public boolean handleMessage(Message msg) { 645 if (mCleanup) { 646 return true; 647 } 648 649 if (DEBUG) { 650 Log.d(TAG, "Request thread handling message:" + msg.what); 651 } 652 long startTime = 0; 653 if (DEBUG) { 654 startTime = SystemClock.elapsedRealtimeNanos(); 655 } 656 switch (msg.what) { 657 case MSG_CONFIGURE_OUTPUTS: 658 ConfigureHolder config = (ConfigureHolder) msg.obj; 659 int sizes = config.surfaces != null ? config.surfaces.size() : 0; 660 Log.i(TAG, "Configure outputs: " + sizes + " surfaces configured."); 661 662 try { 663 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 664 TimeUnit.MILLISECONDS); 665 if (!success) { 666 Log.e(TAG, "Timed out while queueing configure request."); 667 mCaptureCollector.failAll(); 668 } 669 } catch (InterruptedException e) { 670 Log.e(TAG, "Interrupted while waiting for requests to complete."); 671 mDeviceState.setError( 672 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 673 break; 674 } 675 676 configureOutputs(config.surfaces); 677 config.condition.open(); 678 if (DEBUG) { 679 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 680 Log.d(TAG, "Configure took " + totalTime + " ns"); 681 } 682 break; 683 case MSG_SUBMIT_CAPTURE_REQUEST: 684 Handler handler = RequestThreadManager.this.mRequestThread.getHandler(); 685 686 // Get the next burst from the request queue. 687 Pair<BurstHolder, Long> nextBurst = mRequestQueue.getNext(); 688 689 if (nextBurst == null) { 690 // If there are no further requests queued, wait for any currently executing 691 // requests to complete, then switch to idle state. 692 try { 693 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 694 TimeUnit.MILLISECONDS); 695 if (!success) { 696 Log.e(TAG, 697 "Timed out while waiting for prior requests to complete."); 698 mCaptureCollector.failAll(); 699 } 700 } catch (InterruptedException e) { 701 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 702 mDeviceState.setError( 703 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 704 break; 705 } 706 707 synchronized (mIdleLock) { 708 // Retry the the request queue. 709 nextBurst = mRequestQueue.getNext(); 710 711 // If we still have no queued requests, go idle. 712 if (nextBurst == null) { 713 mDeviceState.setIdle(); 714 break; 715 } 716 } 717 } 718 719 if (nextBurst != null) { 720 // Queue another capture if we did not get the last burst. 721 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 722 } 723 724 // Complete each request in the burst 725 List<RequestHolder> requests = 726 nextBurst.first.produceRequestHolders(nextBurst.second); 727 for (RequestHolder holder : requests) { 728 CaptureRequest request = holder.getRequest(); 729 730 boolean paramsChanged = false; 731 732 // Only update parameters if the request has changed 733 if (mLastRequest == null || mLastRequest.captureRequest != request) { 734 735 // The intermediate buffer is sometimes null, but we always need 736 // the Camera1 API configured preview size 737 Size previewSize = ParameterUtils.convertSize(mParams.getPreviewSize()); 738 739 LegacyRequest legacyRequest = new LegacyRequest(mCharacteristics, 740 request, previewSize, mParams); // params are copied 741 742 743 // Parameters are mutated as a side-effect 744 LegacyMetadataMapper.convertRequestMetadata(/*inout*/legacyRequest); 745 746 // If the parameters have changed, set them in the Camera1 API. 747 if (!mParams.same(legacyRequest.parameters)) { 748 try { 749 mCamera.setParameters(legacyRequest.parameters); 750 } catch (RuntimeException e) { 751 // If setting the parameters failed, report a request error to 752 // the camera client, and skip any further work for this request 753 Log.e(TAG, "Exception while setting camera parameters: ", e); 754 holder.failRequest(); 755 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 756 CameraDeviceImpl.CameraDeviceCallbacks. 757 ERROR_CAMERA_REQUEST); 758 continue; 759 } 760 paramsChanged = true; 761 mParams = legacyRequest.parameters; 762 } 763 764 mLastRequest = legacyRequest; 765 } 766 767 try { 768 boolean success = mCaptureCollector.queueRequest(holder, 769 mLastRequest, JPEG_FRAME_TIMEOUT, TimeUnit.MILLISECONDS); 770 771 if (!success) { 772 // Report a request error if we timed out while queuing this. 773 Log.e(TAG, "Timed out while queueing capture request."); 774 holder.failRequest(); 775 mDeviceState.setCaptureStart(holder, /*timestamp*/0, 776 CameraDeviceImpl.CameraDeviceCallbacks. 777 ERROR_CAMERA_REQUEST); 778 continue; 779 } 780 781 // Starting the preview needs to happen before enabling 782 // face detection or auto focus 783 if (holder.hasPreviewTargets()) { 784 doPreviewCapture(holder); 785 } 786 if (holder.hasJpegTargets()) { 787 while(!mCaptureCollector.waitForPreviewsEmpty(PREVIEW_FRAME_TIMEOUT, 788 TimeUnit.MILLISECONDS)) { 789 // Fail preview requests until the queue is empty. 790 Log.e(TAG, "Timed out while waiting for preview requests to " + 791 "complete."); 792 mCaptureCollector.failNextPreview(); 793 } 794 mReceivedJpeg.close(); 795 doJpegCapturePrepare(holder); 796 } 797 798 /* 799 * Do all the actions that require a preview to have been started 800 */ 801 802 // Toggle face detection on/off 803 // - do this before AF to give AF a chance to use faces 804 mFaceDetectMapper.processFaceDetectMode(request, /*in*/mParams); 805 806 // Unconditionally process AF triggers, since they're non-idempotent 807 // - must be done after setting the most-up-to-date AF mode 808 mFocusStateMapper.processRequestTriggers(request, mParams); 809 810 if (holder.hasJpegTargets()) { 811 doJpegCapture(holder); 812 if (!mReceivedJpeg.block(JPEG_FRAME_TIMEOUT)) { 813 Log.e(TAG, "Hit timeout for jpeg callback!"); 814 mCaptureCollector.failNextJpeg(); 815 } 816 } 817 818 } catch (IOException e) { 819 Log.e(TAG, "Received device exception during capture call: ", e); 820 mDeviceState.setError( 821 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 822 break; 823 } catch (InterruptedException e) { 824 Log.e(TAG, "Interrupted during capture: ", e); 825 mDeviceState.setError( 826 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 827 break; 828 } catch (RuntimeException e) { 829 Log.e(TAG, "Received device exception during capture call: ", e); 830 mDeviceState.setError( 831 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 832 break; 833 } 834 835 if (paramsChanged) { 836 if (DEBUG) { 837 Log.d(TAG, "Params changed -- getting new Parameters from HAL."); 838 } 839 try { 840 mParams = mCamera.getParameters(); 841 } catch (RuntimeException e) { 842 Log.e(TAG, "Received device exception: ", e); 843 mDeviceState.setError( 844 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 845 break; 846 } 847 848 // Update parameters to the latest that we think the camera is using 849 mLastRequest.setParameters(mParams); 850 } 851 852 MutableLong timestampMutable = new MutableLong(/*value*/0L); 853 try { 854 boolean success = mCaptureCollector.waitForRequestCompleted(holder, 855 REQUEST_COMPLETE_TIMEOUT, TimeUnit.MILLISECONDS, 856 /*out*/timestampMutable); 857 858 if (!success) { 859 Log.e(TAG, "Timed out while waiting for request to complete."); 860 mCaptureCollector.failAll(); 861 } 862 } catch (InterruptedException e) { 863 Log.e(TAG, "Interrupted waiting for request completion: ", e); 864 mDeviceState.setError( 865 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 866 break; 867 } 868 869 CameraMetadataNative result = mMapper.cachedConvertResultMetadata( 870 mLastRequest, timestampMutable.value); 871 /* 872 * Order matters: The default result mapper is state-less; the 873 * other mappers carry state and may override keys set by the default 874 * mapper with their own values. 875 */ 876 877 // Update AF state 878 mFocusStateMapper.mapResultTriggers(result); 879 // Update face-related results 880 mFaceDetectMapper.mapResultFaces(result, mLastRequest); 881 882 if (!holder.requestFailed()) { 883 mDeviceState.setCaptureResult(holder, result, 884 CameraDeviceState.NO_CAPTURE_ERROR); 885 } 886 } 887 if (DEBUG) { 888 long totalTime = SystemClock.elapsedRealtimeNanos() - startTime; 889 Log.d(TAG, "Capture request took " + totalTime + " ns"); 890 mRequestCounter.countAndLog(); 891 } 892 break; 893 case MSG_CLEANUP: 894 mCleanup = true; 895 try { 896 boolean success = mCaptureCollector.waitForEmpty(JPEG_FRAME_TIMEOUT, 897 TimeUnit.MILLISECONDS); 898 if (!success) { 899 Log.e(TAG, "Timed out while queueing cleanup request."); 900 mCaptureCollector.failAll(); 901 } 902 } catch (InterruptedException e) { 903 Log.e(TAG, "Interrupted while waiting for requests to complete: ", e); 904 mDeviceState.setError( 905 CameraDeviceImpl.CameraDeviceCallbacks.ERROR_CAMERA_DEVICE); 906 } 907 if (mGLThreadManager != null) { 908 mGLThreadManager.quit(); 909 mGLThreadManager = null; 910 } 911 if (mCamera != null) { 912 mCamera.release(); 913 mCamera = null; 914 } 915 resetJpegSurfaceFormats(mCallbackOutputs); 916 break; 917 case RequestHandlerThread.MSG_POKE_IDLE_HANDLER: 918 // OK: Ignore message. 919 break; 920 default: 921 throw new AssertionError("Unhandled message " + msg.what + 922 " on RequestThread."); 923 } 924 return true; 925 } 926 }; 927 928 /** 929 * Create a new RequestThreadManager. 930 * 931 * @param cameraId the id of the camera to use. 932 * @param camera an open camera object. The RequestThreadManager takes ownership of this camera 933 * object, and is responsible for closing it. 934 * @param characteristics the static camera characteristics corresponding to this camera device 935 * @param deviceState a {@link CameraDeviceState} state machine. 936 */ RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, CameraDeviceState deviceState)937 public RequestThreadManager(int cameraId, Camera camera, CameraCharacteristics characteristics, 938 CameraDeviceState deviceState) { 939 mCamera = checkNotNull(camera, "camera must not be null"); 940 mCameraId = cameraId; 941 mCharacteristics = checkNotNull(characteristics, "characteristics must not be null"); 942 String name = String.format("RequestThread-%d", cameraId); 943 TAG = name; 944 mDeviceState = checkNotNull(deviceState, "deviceState must not be null"); 945 mFocusStateMapper = new LegacyFocusStateMapper(mCamera); 946 mFaceDetectMapper = new LegacyFaceDetectMapper(mCamera, mCharacteristics); 947 mCaptureCollector = new CaptureCollector(MAX_IN_FLIGHT_REQUESTS, mDeviceState); 948 mRequestThread = new RequestHandlerThread(name, mRequestHandlerCb); 949 mCamera.setErrorCallback(mErrorCallback); 950 } 951 952 /** 953 * Start the request thread. 954 */ start()955 public void start() { 956 mRequestThread.start(); 957 } 958 959 /** 960 * Flush any pending requests. 961 * 962 * @return the last frame number. 963 */ flush()964 public long flush() { 965 Log.i(TAG, "Flushing all pending requests."); 966 long lastFrame = mRequestQueue.stopRepeating(); 967 mCaptureCollector.failAll(); 968 return lastFrame; 969 } 970 971 /** 972 * Quit the request thread, and clean up everything. 973 */ quit()974 public void quit() { 975 if (!mQuit.getAndSet(true)) { // Avoid sending messages on dead thread's handler. 976 Handler handler = mRequestThread.waitAndGetHandler(); 977 handler.sendMessageAtFrontOfQueue(handler.obtainMessage(MSG_CLEANUP)); 978 mRequestThread.quitSafely(); 979 try { 980 mRequestThread.join(); 981 } catch (InterruptedException e) { 982 Log.e(TAG, String.format("Thread %s (%d) interrupted while quitting.", 983 mRequestThread.getName(), mRequestThread.getId())); 984 } 985 } 986 } 987 988 /** 989 * Submit the given burst of requests to be captured. 990 * 991 * <p>If the burst is repeating, replace the current repeating burst.</p> 992 * 993 * @param requests the burst of requests to add to the queue. 994 * @param repeating true if the burst is repeating. 995 * @param frameNumber an output argument that contains either the frame number of the last frame 996 * that will be returned for this request, or the frame number of the last 997 * frame that will be returned for the current repeating request if this 998 * burst is set to be repeating. 999 * @return the request id. 1000 */ submitCaptureRequests(List<CaptureRequest> requests, boolean repeating, LongParcelable frameNumber)1001 public int submitCaptureRequests(List<CaptureRequest> requests, boolean repeating, 1002 /*out*/LongParcelable frameNumber) { 1003 Handler handler = mRequestThread.waitAndGetHandler(); 1004 int ret; 1005 synchronized (mIdleLock) { 1006 ret = mRequestQueue.submit(requests, repeating, frameNumber); 1007 handler.sendEmptyMessage(MSG_SUBMIT_CAPTURE_REQUEST); 1008 } 1009 return ret; 1010 } 1011 1012 /** 1013 * Cancel a repeating request. 1014 * 1015 * @param requestId the id of the repeating request to cancel. 1016 * @return the last frame to be returned from the HAL for the given repeating request, or 1017 * {@code INVALID_FRAME} if none exists. 1018 */ cancelRepeating(int requestId)1019 public long cancelRepeating(int requestId) { 1020 return mRequestQueue.stopRepeating(requestId); 1021 } 1022 1023 /** 1024 * Configure with the current list of output Surfaces. 1025 * 1026 * <p> 1027 * This operation blocks until the configuration is complete. 1028 * </p> 1029 * 1030 * <p>Using a {@code null} or empty {@code outputs} list is the equivalent of unconfiguring.</p> 1031 * 1032 * @param outputs a {@link java.util.Collection} of outputs to configure. 1033 */ configure(Collection<Pair<Surface, Size>> outputs)1034 public void configure(Collection<Pair<Surface, Size>> outputs) { 1035 Handler handler = mRequestThread.waitAndGetHandler(); 1036 final ConditionVariable condition = new ConditionVariable(/*closed*/false); 1037 ConfigureHolder holder = new ConfigureHolder(condition, outputs); 1038 handler.sendMessage(handler.obtainMessage(MSG_CONFIGURE_OUTPUTS, 0, 0, holder)); 1039 condition.block(); 1040 } 1041 } 1042