1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3DEVICE_H 18 #define ANDROID_SERVERS_CAMERA3DEVICE_H 19 20 #include <utility> 21 #include <unordered_map> 22 #include <set> 23 #include <tuple> 24 25 #include <utils/Condition.h> 26 #include <utils/Errors.h> 27 #include <utils/List.h> 28 #include <utils/Mutex.h> 29 #include <utils/Thread.h> 30 #include <utils/KeyedVector.h> 31 #include <utils/Timers.h> 32 33 #include <camera/CaptureResult.h> 34 35 #include "CameraServiceWatchdog.h" 36 #include <aidl/android/hardware/camera/device/CameraBlob.h> 37 38 #include "common/CameraDeviceBase.h" 39 #include "device3/BufferUtils.h" 40 #include "device3/StatusTracker.h" 41 #include "device3/Camera3BufferManager.h" 42 #include "device3/DistortionMapper.h" 43 #include "device3/ZoomRatioMapper.h" 44 #include "device3/RotateAndCropMapper.h" 45 #include "device3/UHRCropAndMeteringRegionMapper.h" 46 #include "device3/InFlightRequest.h" 47 #include "device3/Camera3OutputInterface.h" 48 #include "device3/Camera3OfflineSession.h" 49 #include "device3/Camera3StreamInterface.h" 50 #include "utils/AttributionAndPermissionUtils.h" 51 #include "utils/TagMonitor.h" 52 #include "utils/IPCTransport.h" 53 #include "utils/LatencyHistogram.h" 54 #include "utils/CameraServiceProxyWrapper.h" 55 #include <camera_metadata_hidden.h> 56 57 using android::camera3::camera_capture_request_t; 58 using android::camera3::camera_request_template; 59 using android::camera3::camera_stream_buffer_t; 60 using android::camera3::camera_stream_configuration_t; 61 using android::camera3::camera_stream_configuration_mode_t; 62 using android::camera3::CAMERA_TEMPLATE_COUNT; 63 using android::camera3::OutputStreamInfo; 64 65 namespace android { 66 67 namespace camera3 { 68 69 class Camera3Stream; 70 class Camera3ZslStream; 71 class Camera3StreamInterface; 72 73 } // namespace camera3 74 75 /** 76 * CameraDevice for HAL devices with version CAMERA_DEVICE_API_VERSION_3_0 or higher. 77 */ 78 class Camera3Device : 79 public CameraDeviceBase, 80 public camera3::SetErrorInterface, 81 public camera3::InflightRequestUpdateInterface, 82 public camera3::RequestBufferInterface, 83 public camera3::FlushBufferInterface, 84 public AttributionAndPermissionUtilsEncapsulator { 85 friend class HidlCamera3Device; 86 friend class AidlCamera3Device; 87 public: 88 89 explicit Camera3Device(std::shared_ptr<CameraServiceProxyWrapper>& cameraServiceProxyWrapper, 90 std::shared_ptr<AttributionAndPermissionUtils> attributionAndPermissionUtils, 91 const std::string& id, bool overrideForPerfClass, int rotationOverride, 92 bool legacyClient = false); 93 94 virtual ~Camera3Device(); 95 // Delete and optionally close native handles and clear the input vector afterward 96 static void cleanupNativeHandles( 97 std::vector<native_handle_t*> *handles, bool closeFd = false); 98 getTransportType()99 virtual IPCTransport getTransportType() const override { 100 return mInterface->getTransportType(); 101 } 102 isHalBufferManagedStream(int32_t streamId)103 bool isHalBufferManagedStream(int32_t streamId) const { 104 return mInterface->isHalBufferManagedStream(streamId); 105 }; 106 107 /** 108 * CameraDeviceBase interface 109 */ 110 111 const std::string& getId() const override; 112 getVendorTagId()113 metadata_vendor_id_t getVendorTagId() const override { return mVendorTagId; } 114 115 // Watchdog thread 116 sp<CameraServiceWatchdog> mCameraServiceWatchdog; 117 118 // Transitions to idle state on success. 119 virtual status_t initialize(sp<CameraProviderManager> /*manager*/, 120 const std::string& /*monitorTags*/) = 0; 121 122 status_t disconnect() override; 123 status_t dump(int fd, const Vector<String16> &args) override; 124 status_t startWatchingTags(const std::string &tags) override; 125 status_t stopWatchingTags() override; 126 status_t dumpWatchedEventsToVector(std::vector<std::string> &out) override; 127 const CameraMetadata& info() const override; 128 const CameraMetadata& infoPhysical(const std::string& physicalId) const override; isCompositeJpegRDisabled()129 bool isCompositeJpegRDisabled() const override { return mIsCompositeJpegRDisabled; }; 130 131 // Capture and setStreamingRequest will configure streams if currently in 132 // idle state 133 status_t capture(CameraMetadata &request, int64_t *lastFrameNumber = NULL) override; 134 status_t captureList(const List<const PhysicalCameraSettingsList> &requestsList, 135 const std::list<const SurfaceMap> &surfaceMaps, 136 int64_t *lastFrameNumber = NULL) override; 137 status_t setStreamingRequest(const CameraMetadata &request, 138 int64_t *lastFrameNumber = NULL) override; 139 status_t setStreamingRequestList(const List<const PhysicalCameraSettingsList> &requestsList, 140 const std::list<const SurfaceMap> &surfaceMaps, 141 int64_t *lastFrameNumber = NULL) override; 142 status_t clearStreamingRequest(int64_t *lastFrameNumber = NULL) override; 143 144 status_t waitUntilRequestReceived(int32_t requestId, nsecs_t timeout) override; 145 146 // Actual stream creation/deletion is delayed until first request is submitted 147 // If adding streams while actively capturing, will pause device before adding 148 // stream, reconfiguring device, and unpausing. If the client create a stream 149 // with nullptr consumer surface, the client must then call setConsumers() 150 // and finish the stream configuration before starting output streaming. 151 status_t createStream(sp<Surface> consumer, 152 uint32_t width, uint32_t height, int format, 153 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 154 const std::string& physicalCameraId, 155 const std::unordered_set<int32_t> &sensorPixelModesUsed, 156 std::vector<int> *surfaceIds = nullptr, 157 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 158 bool isShared = false, bool isMultiResolution = false, 159 uint64_t consumerUsage = 0, 160 int64_t dynamicRangeProfile = 161 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 162 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 163 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 164 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 165 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 166 bool useReadoutTimestamp = false) 167 override; 168 169 status_t createStream(const std::vector<sp<Surface>>& consumers, 170 bool hasDeferredConsumer, uint32_t width, uint32_t height, int format, 171 android_dataspace dataSpace, camera_stream_rotation_t rotation, int *id, 172 const std::string& physicalCameraId, 173 const std::unordered_set<int32_t> &sensorPixelModesUsed, 174 std::vector<int> *surfaceIds = nullptr, 175 int streamSetId = camera3::CAMERA3_STREAM_SET_ID_INVALID, 176 bool isShared = false, bool isMultiResolution = false, 177 uint64_t consumerUsage = 0, 178 int64_t dynamicRangeProfile = 179 ANDROID_REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES_MAP_STANDARD, 180 int64_t streamUseCase = ANDROID_SCALER_AVAILABLE_STREAM_USE_CASES_DEFAULT, 181 int timestampBase = OutputConfiguration::TIMESTAMP_BASE_DEFAULT, 182 int mirrorMode = OutputConfiguration::MIRROR_MODE_AUTO, 183 int32_t colorSpace = ANDROID_REQUEST_AVAILABLE_COLOR_SPACE_PROFILES_MAP_UNSPECIFIED, 184 bool useReadoutTimestamp = false) 185 override; 186 187 status_t createInputStream( 188 uint32_t width, uint32_t height, int format, bool isMultiResolution, 189 int *id) override; 190 191 status_t getStreamInfo(int id, StreamInfo *streamInfo) override; 192 status_t setStreamTransform(int id, int transform) override; 193 194 status_t deleteStream(int id) override; 195 196 status_t configureStreams(const CameraMetadata& sessionParams, 197 int operatingMode = 198 camera_stream_configuration_mode_t::CAMERA_STREAM_CONFIGURATION_NORMAL_MODE) override; 199 status_t getInputBufferProducer( 200 sp<IGraphicBufferProducer> *producer) override; 201 202 void getOfflineStreamIds(std::vector<int> *offlineStreamIds) override; 203 204 status_t createDefaultRequest(camera_request_template_t templateId, 205 CameraMetadata *request) override; 206 207 // Transitions to the idle state on success 208 status_t waitUntilDrained() override; 209 210 status_t setNotifyCallback(wp<NotificationListener> listener) override; 211 bool willNotify3A() override; 212 status_t waitForNextFrame(nsecs_t timeout) override; 213 status_t getNextResult(CaptureResult *frame) override; 214 215 status_t triggerAutofocus(uint32_t id) override; 216 status_t triggerCancelAutofocus(uint32_t id) override; 217 status_t triggerPrecaptureMetering(uint32_t id) override; 218 219 status_t flush(int64_t *lastFrameNumber = NULL) override; 220 221 status_t prepare(int streamId) override; 222 223 status_t tearDown(int streamId) override; 224 225 status_t addBufferListenerForStream(int streamId, 226 wp<camera3::Camera3StreamBufferListener> listener) override; 227 228 status_t prepare(int maxCount, int streamId) override; 229 230 ssize_t getJpegBufferSize(const CameraMetadata &info, uint32_t width, 231 uint32_t height) const override; 232 ssize_t getPointCloudBufferSize(const CameraMetadata &info) const; 233 ssize_t getRawOpaqueBufferSize(const CameraMetadata &info, int32_t width, int32_t height, 234 bool maxResolution) const; 235 236 // Methods called by subclasses 237 void notifyStatus(bool idle); // updates from StatusTracker 238 239 /** 240 * Set the deferred consumer surfaces to the output stream and finish the deferred 241 * consumer configuration. 242 */ 243 status_t setConsumerSurfaces( 244 int streamId, const std::vector<sp<Surface>>& consumers, 245 std::vector<int> *surfaceIds /*out*/) override; 246 247 /** 248 * Update a given stream. 249 */ 250 status_t updateStream(int streamId, const std::vector<sp<Surface>> &newSurfaces, 251 const std::vector<OutputStreamInfo> &outputInfo, 252 const std::vector<size_t> &removedSurfaceIds, 253 KeyedVector<sp<Surface>, size_t> *outputMap/*out*/); 254 255 /** 256 * Drop buffers for stream of streamId if dropping is true. If dropping is false, do not 257 * drop buffers for stream of streamId. 258 */ 259 status_t dropStreamBuffers(bool dropping, int streamId) override; 260 261 nsecs_t getExpectedInFlightDuration() override; 262 switchToOffline(const std::vector<int32_t> &,sp<CameraOfflineSessionBase> *)263 virtual status_t switchToOffline(const std::vector<int32_t>& , 264 /*out*/ sp<CameraOfflineSessionBase>* ) override { 265 return INVALID_OPERATION; 266 }; 267 268 // RequestBufferInterface 269 bool startRequestBuffer() override; 270 void endRequestBuffer() override; 271 nsecs_t getWaitDuration() override; 272 273 // FlushBufferInterface 274 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override; 275 void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override; 276 std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override; 277 278 /** 279 * Set the current behavior for the ROTATE_AND_CROP control when in AUTO. 280 * 281 * The value must be one of the ROTATE_AND_CROP_* values besides AUTO, 282 * and defaults to NONE. 283 */ 284 status_t setRotateAndCropAutoBehavior( 285 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue, bool fromHal); 286 287 /** 288 * Set the current behavior for the AUTOFRAMING control when in AUTO. 289 * 290 * The value must be one of the AUTOFRAMING_* values besides AUTO. 291 */ 292 status_t setAutoframingAutoBehavior( 293 camera_metadata_enum_android_control_autoframing_t autoframingValue); 294 295 /** 296 * Whether camera muting (producing black-only output) is supported. 297 * 298 * Calling setCameraMute(true) when this returns false will return an 299 * INVALID_OPERATION error. 300 */ 301 bool supportsCameraMute(); 302 303 /** 304 * Mute the camera. 305 * 306 * When muted, black image data is output on all output streams. 307 */ 308 status_t setCameraMute(bool enabled); 309 310 /** 311 * Mute the camera. 312 * 313 * When muted, black image data is output on all output streams. 314 * This method assumes the caller already acquired the 'mInterfaceLock' 315 * and 'mLock' locks. 316 */ 317 status_t setCameraMuteLocked(bool enabled); 318 319 /** 320 * Enables/disables camera service watchdog 321 */ 322 status_t setCameraServiceWatchdog(bool enabled); 323 324 // Set stream use case overrides 325 void setStreamUseCaseOverrides( 326 const std::vector<int64_t>& useCaseOverrides); 327 328 // Clear stream use case overrides 329 void clearStreamUseCaseOverrides(); 330 331 /** 332 * Whether the camera device supports zoom override. 333 */ 334 bool supportsZoomOverride(); 335 336 // Set/reset zoom override 337 status_t setZoomOverride(int32_t zoomOverride); 338 339 // Get the status trackeer for the camera device getStatusTracker()340 wp<camera3::StatusTracker> getStatusTracker() { return mStatusTracker; } 341 342 // Whether the device is in error state 343 bool hasDeviceError(); 344 345 /** 346 * The injection camera session to replace the internal camera 347 * session. 348 */ 349 status_t injectCamera(const std::string& injectedCamId, 350 sp<CameraProviderManager> manager); 351 352 /** 353 * Stop the injection camera and restore to internal camera session. 354 */ 355 status_t stopInjection(); 356 357 /** 358 * Inject session params into the current client. 359 */ 360 status_t injectSessionParams(const CameraMetadata& sessionParams); 361 362 protected: 363 status_t disconnectImpl(); 364 static status_t removeFwkOnlyRegionKeys(CameraMetadata *request); 365 366 float getMaxPreviewFps(sp<camera3::Camera3OutputStreamInterface> stream); 367 368 static const size_t kDumpLockAttempts = 10; 369 static const size_t kDumpSleepDuration = 100000; // 0.10 sec 370 static const nsecs_t kActiveTimeout = 500000000; // 500 ms 371 static const nsecs_t kMinWarnInflightDuration = 5000000000; // 5 s 372 static const size_t kInFlightWarnLimit = 30; 373 static const size_t kInFlightWarnLimitHighSpeed = 256; // batch size 32 * pipe depth 8 374 static const nsecs_t kMinInflightDuration = 5000000000; // 5 s 375 static const nsecs_t kBaseGetBufferWait = 3000000000; // 3 sec. 376 377 struct RequestTrigger; 378 // minimal jpeg buffer size: 256KB + blob header 379 static const ssize_t kMinJpegBufferSize = 380 256 * 1024 + sizeof(aidl::android::hardware::camera::device::CameraBlob); 381 // Constant to use for stream ID when one doesn't exist 382 static const int NO_STREAM = -1; 383 384 std::shared_ptr<CameraServiceProxyWrapper> mCameraServiceProxyWrapper; 385 386 // A lock to enforce serialization on the input/configure side 387 // of the public interface. 388 // Only locked by public methods inherited from CameraDeviceBase. 389 // Not locked by methods guarded by mOutputLock, since they may act 390 // concurrently to the input/configure side of the interface. 391 // Must be locked before mLock if both will be locked by a method 392 Mutex mInterfaceLock; 393 394 // The main lock on internal state 395 Mutex mLock; 396 397 // Camera device ID 398 const std::string mId; 399 400 // Legacy camera client flag 401 bool mLegacyClient; 402 403 // Current stream configuration mode; 404 int mOperatingMode; 405 // Current session wide parameters 406 hardware::camera2::impl::CameraMetadataNative mSessionParams; 407 // Constant to use for no set operating mode 408 static const int NO_MODE = -1; 409 410 // Flag indicating is the current active stream configuration is constrained high speed. 411 bool mIsConstrainedHighSpeedConfiguration; 412 413 /**** Scope for mLock ****/ 414 415 class HalInterface : public camera3::Camera3StreamBufferFreedListener, 416 public camera3::BufferRecordsInterface { 417 public: HalInterface(bool useHalBufManager,bool supportOfflineProcessing)418 HalInterface(bool useHalBufManager, bool supportOfflineProcessing) : 419 mUseHalBufManager(useHalBufManager), 420 mIsReconfigurationQuerySupported(true), 421 mSupportOfflineProcessing(supportOfflineProcessing) 422 {}; 423 HalInterface(const HalInterface &other); 424 HalInterface(); 425 426 virtual IPCTransport getTransportType() const = 0; 427 428 // Returns true if constructed with a valid device or session, and not yet cleared 429 virtual bool valid() = 0; 430 431 // Reset this HalInterface object (does not call close()) 432 virtual void clear() = 0; 433 434 // Calls into the HAL interface 435 436 // Caller takes ownership of requestTemplate 437 virtual status_t constructDefaultRequestSettings(camera_request_template templateId, 438 /*out*/ camera_metadata_t **requestTemplate) = 0; 439 440 virtual status_t configureStreams(const camera_metadata_t * sessionParams, 441 /*inout*/ camera_stream_configuration_t * config, 442 const std::vector<uint32_t>& bufferSizes, int64_t logId) = 0; 443 444 // The injection camera configures the streams to hal. 445 virtual status_t configureInjectedStreams( 446 const camera_metadata_t* sessionParams, 447 /*inout*/ camera_stream_configuration_t* config, 448 const std::vector<uint32_t>& bufferSizes, 449 const CameraMetadata& cameraCharacteristics) = 0; 450 451 // When the call succeeds, the ownership of acquire fences in requests is transferred to 452 // HalInterface. More specifically, the current implementation will send the fence to 453 // HAL process and close the FD in cameraserver process. When the call fails, the ownership 454 // of the acquire fence still belongs to the caller. 455 virtual status_t processBatchCaptureRequests( 456 std::vector<camera_capture_request_t*>& requests, 457 /*out*/uint32_t* numRequestProcessed) = 0; 458 459 virtual status_t flush() = 0; 460 461 virtual status_t dump(int fd) = 0; 462 463 virtual status_t close() = 0; 464 465 virtual void signalPipelineDrain(const std::vector<int>& streamIds) = 0; 466 467 virtual bool isReconfigurationRequired(CameraMetadata& oldSessionParams, 468 CameraMetadata& newSessionParams) = 0; 469 470 virtual status_t repeatingRequestEnd(uint32_t frameNumber, 471 const std::vector<int32_t> &streamIds) = 0; 472 473 ///////////////////////////////////////////////////////////////////// 474 // Implements BufferRecordsInterface 475 476 std::pair<bool, uint64_t> getBufferId( 477 const buffer_handle_t& buf, int streamId) override; 478 479 uint64_t removeOneBufferCache(int streamId, const native_handle_t* handle) override; 480 481 status_t popInflightBuffer(int32_t frameNumber, int32_t streamId, 482 /*out*/ buffer_handle_t **buffer) override; 483 484 status_t pushInflightRequestBuffer( 485 uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override; 486 487 status_t popInflightRequestBuffer(uint64_t bufferId, 488 /*out*/ buffer_handle_t** buffer, 489 /*optional out*/ int32_t* streamId = nullptr) override; 490 491 ///////////////////////////////////////////////////////////////////// 492 493 //Check if a stream is hal buffer managed 494 bool isHalBufferManagedStream(int32_t streamId) const; 495 496 // Get a vector of (frameNumber, streamId) pair of currently inflight 497 // buffers 498 void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out); 499 500 // Get a vector of bufferId of currently inflight buffers 501 void getInflightRequestBufferKeys(std::vector<uint64_t>* out); 502 503 void onStreamReConfigured(int streamId); 504 505 protected: 506 507 // Return true if the input caches match what we have; otherwise false 508 bool verifyBufferIds(int32_t streamId, std::vector<uint64_t>& inBufIds); 509 510 template <typename OfflineSessionInfoT> verifyBufferCaches(const OfflineSessionInfoT * offlineSessionInfo,camera3::BufferRecords * bufferRecords)511 status_t verifyBufferCaches( 512 const OfflineSessionInfoT *offlineSessionInfo, camera3::BufferRecords *bufferRecords) { 513 // Validate buffer caches 514 std::vector<int32_t> streams; 515 streams.reserve(offlineSessionInfo->offlineStreams.size()); 516 for (auto offlineStream : offlineSessionInfo->offlineStreams) { 517 int32_t id = offlineStream.id; 518 streams.push_back(id); 519 // Verify buffer caches 520 std::vector<uint64_t> bufIds(offlineStream.circulatingBufferIds.begin(), 521 offlineStream.circulatingBufferIds.end()); 522 { 523 // Due to timing it is possible that we may not have any remaining pending 524 // capture requests that can update the caches on Hal side. This can result in 525 // buffer cache mismatch between the service and the Hal and must be accounted 526 // for. 527 std::lock_guard<std::mutex> l(mFreedBuffersLock); 528 for (const auto& it : mFreedBuffers) { 529 if (it.first == id) { 530 ALOGV("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 531 "pending", __FUNCTION__, id, it.second); 532 const auto& cachedEntry = std::find(bufIds.begin(), bufIds.end(), 533 it.second); 534 if (cachedEntry != bufIds.end()) { 535 bufIds.erase(cachedEntry); 536 } else { 537 ALOGE("%s: stream ID %d buffer id %" PRIu64 " cache removal still " 538 "pending however buffer is no longer in the offline stream " 539 "info!", __FUNCTION__, id, it.second); 540 } 541 } 542 } 543 } 544 if (!verifyBufferIds(id, bufIds)) { 545 ALOGE("%s: stream ID %d buffer cache records mismatch!", __FUNCTION__, id); 546 return UNKNOWN_ERROR; 547 } 548 } 549 550 // Move buffer records 551 bufferRecords->takeBufferCaches(mBufferRecords, streams); 552 bufferRecords->takeInflightBufferMap(mBufferRecords); 553 bufferRecords->takeRequestedBufferMap(mBufferRecords); 554 return OK; 555 } 556 557 virtual void onBufferFreed(int streamId, const native_handle_t* handle) override; 558 559 std::mutex mFreedBuffersLock; 560 std::vector<std::pair<int, uint64_t>> mFreedBuffers; 561 562 // Keep track of buffer cache and inflight buffer records 563 camera3::BufferRecords mBufferRecords; 564 565 uint32_t mNextStreamConfigCounter = 1; 566 567 // TODO: This can be removed after flags::session_hal_buf_manager is removed 568 bool mUseHalBufManager = false; 569 std::set<int32_t > mHalBufManagedStreamIds; 570 bool mIsReconfigurationQuerySupported; 571 572 const bool mSupportOfflineProcessing; 573 }; // class HalInterface 574 575 sp<HalInterface> mInterface; 576 577 CameraMetadata mDeviceInfo; 578 bool mSupportNativeZoomRatio; 579 bool mIsCompositeJpegRDisabled; 580 std::unordered_map<std::string, CameraMetadata> mPhysicalDeviceInfoMap; 581 582 CameraMetadata mRequestTemplateCache[CAMERA_TEMPLATE_COUNT]; 583 584 struct Size { 585 uint32_t width; 586 uint32_t height; widthSize587 explicit Size(uint32_t w = 0, uint32_t h = 0) : width(w), height(h){} 588 }; 589 590 enum Status { 591 STATUS_ERROR, 592 STATUS_UNINITIALIZED, 593 STATUS_UNCONFIGURED, 594 STATUS_CONFIGURED, 595 STATUS_ACTIVE 596 } mStatus; 597 598 struct StatusInfo { 599 Status status; 600 bool isInternal; // status triggered by internal reconfigureCamera. 601 }; 602 603 bool mStatusIsInternal; 604 605 // Only clear mRecentStatusUpdates, mStatusWaiters from waitUntilStateThenRelock 606 Vector<StatusInfo> mRecentStatusUpdates; 607 int mStatusWaiters; 608 609 Condition mStatusChanged; 610 611 // Tracking cause of fatal errors when in STATUS_ERROR 612 std::string mErrorCause; 613 614 camera3::StreamSet mOutputStreams; 615 sp<camera3::Camera3Stream> mInputStream; 616 bool mIsInputStreamMultiResolution; 617 SessionStatsBuilder mSessionStatsBuilder; 618 // Map from stream group ID to physical cameras backing the stream group 619 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 620 621 int mNextStreamId; 622 bool mNeedConfig; 623 624 int mFakeStreamId; 625 626 // Whether to send state updates upstream 627 // Pause when doing transparent reconfiguration 628 bool mPauseStateNotify; 629 630 // Need to hold on to stream references until configure completes. 631 Vector<sp<camera3::Camera3StreamInterface> > mDeletedStreams; 632 633 // Whether the HAL will send partial result 634 bool mUsePartialResult; 635 636 // Number of partial results that will be delivered by the HAL. 637 uint32_t mNumPartialResults; 638 639 /**** End scope for mLock ****/ 640 641 bool mDeviceTimeBaseIsRealtime; 642 // The offset converting from clock domain of other subsystem 643 // (video/hardware composer) to that of camera. Assumption is that this 644 // offset won't change during the life cycle of the camera device. In other 645 // words, camera device shouldn't be open during CPU suspend. 646 nsecs_t mTimestampOffset; 647 648 class CaptureRequest : public LightRefBase<CaptureRequest> { 649 public: 650 PhysicalCameraSettingsList mSettingsList; 651 sp<camera3::Camera3Stream> mInputStream; 652 camera_stream_buffer_t mInputBuffer; 653 camera3::Size mInputBufferSize; 654 Vector<sp<camera3::Camera3OutputStreamInterface> > 655 mOutputStreams; 656 SurfaceMap mOutputSurfaces; 657 CaptureResultExtras mResultExtras; 658 // The number of requests that should be submitted to HAL at a time. 659 // For example, if batch size is 8, this request and the following 7 660 // requests will be submitted to HAL at a time. The batch size for 661 // the following 7 requests will be ignored by the request thread. 662 int mBatchSize; 663 // Whether this request is from a repeating or repeating burst. 664 bool mRepeating; 665 // Whether this request has ROTATE_AND_CROP_AUTO set, so needs both 666 // overriding of ROTATE_AND_CROP value and adjustment of coordinates 667 // in several other controls in both the request and the result 668 bool mRotateAndCropAuto; 669 // Indicates that the ROTATE_AND_CROP value within 'mSettingsList' was modified 670 // irrespective of the original value. 671 bool mRotateAndCropChanged = false; 672 // Whether this request has AUTOFRAMING_AUTO set, so need to override the AUTOFRAMING value 673 // in the capture request. 674 bool mAutoframingAuto; 675 // Indicates that the auto framing value within 'mSettingsList' was modified 676 bool mAutoframingChanged = false; 677 // Indicates that the camera test pattern setting is modified 678 bool mTestPatternChanged = false; 679 680 // Whether this capture request has its zoom ratio set to 1.0x before 681 // the framework overrides it for camera HAL consumption. 682 bool mZoomRatioIs1x; 683 // The systemTime timestamp when the request is created. 684 nsecs_t mRequestTimeNs; 685 686 // Whether this capture request's distortion correction update has 687 // been done. 688 bool mDistortionCorrectionUpdated = false; 689 // Whether this capture request's rotation and crop update has been 690 // done. 691 bool mRotationAndCropUpdated = false; 692 // Whether this capture request's autoframing has been done. 693 bool mAutoframingUpdated = false; 694 // Whether this capture request's zoom ratio update has been done. 695 bool mZoomRatioUpdated = false; 696 // Whether this max resolution capture request's crop / metering region update has been 697 // done. 698 bool mUHRCropAndMeteringRegionsUpdated = false; 699 }; 700 typedef List<sp<CaptureRequest> > RequestList; 701 702 status_t checkStatusOkToCaptureLocked(); 703 704 status_t convertMetadataListToRequestListLocked( 705 const List<const PhysicalCameraSettingsList> &metadataList, 706 const std::list<const SurfaceMap> &surfaceMaps, 707 bool repeating, nsecs_t requestTimeNs, 708 /*out*/ 709 RequestList *requestList); 710 711 void convertToRequestList(List<const PhysicalCameraSettingsList>& requestsList, 712 std::list<const SurfaceMap>& surfaceMaps, 713 const CameraMetadata& request); 714 715 status_t submitRequestsHelper(const List<const PhysicalCameraSettingsList> &requestsList, 716 const std::list<const SurfaceMap> &surfaceMaps, 717 bool repeating, 718 int64_t *lastFrameNumber = NULL); 719 720 // lock to ensure only one processCaptureResult is called at a time. 721 Mutex mProcessCaptureResultLock; 722 723 /** 724 * Common initialization code shared by both HAL paths 725 * 726 * Must be called with mLock and mInterfaceLock held. 727 */ 728 status_t initializeCommonLocked(); 729 730 /** 731 * Update capture request list so that each batch size honors the batch_size_max report from 732 * the HAL. Set the batch size to output stream for buffer operations. 733 * 734 * Must be called with mLock held. 735 */ 736 virtual void applyMaxBatchSizeLocked( 737 RequestList* requestList, const sp<camera3::Camera3OutputStreamInterface>& stream) = 0; 738 739 /** 740 * Get the last request submitted to the hal by the request thread. 741 * 742 * Must be called with mLock held. 743 */ 744 virtual CameraMetadata getLatestRequestLocked(); 745 746 virtual status_t injectionCameraInitialize(const std::string &injectCamId, 747 sp<CameraProviderManager> manager) = 0; 748 749 /** 750 * Update the current device status and wake all waiting threads. 751 * 752 * Must be called with mLock held. 753 */ 754 void internalUpdateStatusLocked(Status status); 755 756 /** 757 * Pause processing and flush everything, but don't tell the clients. 758 * This is for reconfiguring outputs transparently when according to the 759 * CameraDeviceBase interface we shouldn't need to. 760 * Must be called with mLock and mInterfaceLock both held. 761 */ 762 status_t internalPauseAndWaitLocked(nsecs_t maxExpectedDuration, 763 bool requestThreadInvocation); 764 765 /** 766 * Resume work after internalPauseAndWaitLocked() 767 * Must be called with mLock and mInterfaceLock both held. 768 */ 769 status_t internalResumeLocked(); 770 771 /** 772 * Wait until status tracker tells us we've transitioned to the target state 773 * set, which is either ACTIVE when active==true or IDLE (which is any 774 * non-ACTIVE state) when active==false. 775 * 776 * Needs to be called with mLock and mInterfaceLock held. This means there 777 * can ever only be one waiter at most. 778 * 779 * During the wait mLock is released. 780 * 781 */ 782 status_t waitUntilStateThenRelock(bool active, nsecs_t timeout, 783 bool requestThreadInvocation); 784 785 /** 786 * Implementation of waitUntilDrained. On success, will transition to IDLE state. 787 * 788 * Need to be called with mLock and mInterfaceLock held. 789 */ 790 status_t waitUntilDrainedLocked(nsecs_t maxExpectedDuration); 791 792 /** 793 * Do common work for setting up a streaming or single capture request. 794 * On success, will transition to ACTIVE if in IDLE. 795 */ 796 sp<CaptureRequest> setUpRequestLocked(const PhysicalCameraSettingsList &request, 797 const SurfaceMap &surfaceMap); 798 799 /** 800 * Build a CaptureRequest request from the CameraDeviceBase request 801 * settings. 802 */ 803 sp<CaptureRequest> createCaptureRequest(const PhysicalCameraSettingsList &request, 804 const SurfaceMap &surfaceMap); 805 806 /** 807 * Internally re-configure camera device using new session parameters. 808 * This will get triggered by the request thread. 809 */ 810 bool reconfigureCamera(const CameraMetadata& sessionParams, int clientStatusId); 811 812 /** 813 * Return true in case of any output or input abandoned streams, 814 * otherwise return false. 815 */ 816 bool checkAbandonedStreamsLocked(); 817 818 /** 819 * Filter stream session parameters and configure camera HAL. 820 */ 821 status_t filterParamsAndConfigureLocked(const CameraMetadata& sessionParams, 822 int operatingMode); 823 824 /** 825 * Take the currently-defined set of streams and configure the HAL to use 826 * them. This is a long-running operation (may be several hundered ms). 827 */ 828 status_t configureStreamsLocked(int operatingMode, 829 const CameraMetadata& sessionParams, bool notifyRequestThread = true); 830 831 /** 832 * Cancel stream configuration that did not finish successfully. 833 */ 834 void cancelStreamsConfigurationLocked(); 835 836 /** 837 * Add a fake stream to the current stream set as a workaround for 838 * not allowing 0 streams in the camera HAL spec. 839 */ 840 status_t addFakeStreamLocked(); 841 842 /** 843 * Remove a fake stream if the current config includes real streams. 844 */ 845 status_t tryRemoveFakeStreamLocked(); 846 847 /** 848 * Set device into an error state due to some fatal failure, and set an 849 * error message to indicate why. Only the first call's message will be 850 * used. The message is also sent to the log. 851 */ 852 void setErrorState(const char *fmt, ...) override; 853 void setErrorStateLocked(const char *fmt, ...) override; 854 void setErrorStateV(const char *fmt, va_list args); 855 void setErrorStateLockedV(const char *fmt, va_list args); 856 857 ///////////////////////////////////////////////////////////////////// 858 // Implements InflightRequestUpdateInterface 859 860 void onInflightEntryRemovedLocked(nsecs_t duration) override; 861 void checkInflightMapLengthLocked() override; 862 void onInflightMapFlushedLocked() override; 863 864 ///////////////////////////////////////////////////////////////////// 865 866 /** 867 * Debugging trylock/spin method 868 * Try to acquire a lock a few times with sleeps between before giving up. 869 */ 870 bool tryLockSpinRightRound(Mutex& lock); 871 872 /** 873 * Helper function to get the offset between MONOTONIC and BOOTTIME 874 * timestamp. 875 */ 876 static nsecs_t getMonoToBoottimeOffset(); 877 878 // Override rotate_and_crop control if needed 879 static bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/, 880 int rotationOverride, 881 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropOverride); 882 883 // Override auto framing control if needed 884 static bool overrideAutoframing(const sp<CaptureRequest> &request /*out*/, 885 camera_metadata_enum_android_control_autoframing_t autoframingOverride); 886 887 struct RequestTrigger { 888 // Metadata tag number, e.g. android.control.aePrecaptureTrigger 889 uint32_t metadataTag; 890 // Metadata value, e.g. 'START' or the trigger ID 891 int32_t entryValue; 892 893 // The last part of the fully qualified path, e.g. afTrigger getTagNameRequestTrigger894 const char *getTagName() const { 895 return get_camera_metadata_tag_name(metadataTag) ?: "NULL"; 896 } 897 898 // e.g. TYPE_BYTE, TYPE_INT32, etc. getTagTypeRequestTrigger899 int getTagType() const { 900 return get_camera_metadata_tag_type(metadataTag); 901 } 902 }; 903 904 /** 905 * Thread for managing capture request submission to HAL device. 906 */ 907 class RequestThread : public Thread { 908 909 public: 910 911 RequestThread(wp<Camera3Device> parent, 912 sp<camera3::StatusTracker> statusTracker, 913 sp<HalInterface> interface, 914 const Vector<int32_t>& sessionParamKeys, 915 bool useHalBufManager, 916 bool supportCameraMute, 917 int rotationOverride, 918 bool supportSettingsOverride); 919 ~RequestThread(); 920 921 void setNotificationListener(wp<NotificationListener> listener); 922 923 /** 924 * Call after stream (re)-configuration is completed. 925 */ 926 void configurationComplete(bool isConstrainedHighSpeed, 927 const CameraMetadata& sessionParams, 928 const std::map<int32_t, std::set<std::string>>& groupIdPhysicalCameraMap); 929 930 /** 931 * Set or clear the list of repeating requests. Does not block 932 * on either. Use waitUntilPaused to wait until request queue 933 * has emptied out. 934 */ 935 status_t setRepeatingRequests(const RequestList& requests, 936 /*out*/ 937 int64_t *lastFrameNumber = NULL); 938 status_t clearRepeatingRequests(/*out*/ 939 int64_t *lastFrameNumber = NULL); 940 941 status_t queueRequestList(List<sp<CaptureRequest> > &requests, 942 /*out*/ 943 int64_t *lastFrameNumber = NULL); 944 945 /** 946 * Remove all queued and repeating requests, and pending triggers 947 */ 948 status_t clear(/*out*/int64_t *lastFrameNumber = NULL); 949 950 /** 951 * Flush all pending requests in HAL. 952 */ 953 status_t flush(); 954 955 /** 956 * Queue a trigger to be dispatched with the next outgoing 957 * process_capture_request. The settings for that request only 958 * will be temporarily rewritten to add the trigger tag/value. 959 * Subsequent requests will not be rewritten (for this tag). 960 */ 961 status_t queueTrigger(RequestTrigger trigger[], size_t count); 962 963 /** 964 * Pause/unpause the capture thread. Doesn't block, so use 965 * waitUntilPaused to wait until the thread is paused. 966 */ 967 void setPaused(bool paused); 968 969 /** 970 * Set Hal buffer managed streams 971 * @param halBufferManagedStreams The streams for which hal buffer manager is enabled 972 * 973 */ 974 void setHalBufferManagedStreams(const std::set<int32_t> &halBufferManagedStreams); 975 976 /** 977 * Wait until thread processes the capture request with settings' 978 * android.request.id == requestId. 979 * 980 * Returns TIMED_OUT in case the thread does not process the request 981 * within the timeout. 982 */ 983 status_t waitUntilRequestProcessed(int32_t requestId, nsecs_t timeout); 984 985 /** 986 * Shut down the thread. Shutdown is asynchronous, so thread may 987 * still be running once this method returns. 988 */ 989 virtual void requestExit(); 990 991 /** 992 * Get the latest request that was sent to the HAL 993 * with process_capture_request. 994 */ 995 CameraMetadata getLatestRequest() const; 996 997 /** 998 * Returns true if the stream is a target of any queued or repeating 999 * capture request 1000 */ 1001 bool isStreamPending(sp<camera3::Camera3StreamInterface>& stream); 1002 1003 /** 1004 * Returns true if the surface is a target of any queued or repeating 1005 * capture request 1006 */ 1007 bool isOutputSurfacePending(int streamId, size_t surfaceId); 1008 1009 // dump processCaptureRequest latency dumpCaptureRequestLatency(int fd,const char * name)1010 void dumpCaptureRequestLatency(int fd, const char* name) { 1011 mRequestLatency.dump(fd, name); 1012 } 1013 1014 void signalPipelineDrain(const std::vector<int>& streamIds); 1015 void resetPipelineDrain(); 1016 1017 void clearPreviousRequest(); 1018 1019 status_t setRotateAndCropAutoBehavior( 1020 camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue); 1021 1022 status_t setAutoframingAutoBehaviour( 1023 camera_metadata_enum_android_control_autoframing_t autoframingValue); 1024 1025 status_t setComposerSurface(bool composerSurfacePresent); 1026 1027 status_t setCameraMute(int32_t muteMode); 1028 1029 status_t setZoomOverride(int32_t zoomOverride); 1030 1031 status_t setHalInterface(sp<HalInterface> newHalInterface); 1032 1033 status_t setInjectedSessionParams(const CameraMetadata& sessionParams); 1034 1035 void injectSessionParams( 1036 const sp<CaptureRequest> &request, 1037 const CameraMetadata& injectedSessionParams); 1038 1039 /** 1040 * signal mLatestRequestmutex 1041 **/ 1042 void wakeupLatestRequest(bool latestRequestFailed, int32_t latestRequestId); 1043 1044 protected: 1045 1046 virtual bool threadLoop(); 1047 1048 static const std::string& getId(const wp<Camera3Device> &device); 1049 1050 status_t queueTriggerLocked(RequestTrigger trigger); 1051 // Mix-in queued triggers into this request 1052 int32_t insertTriggers(const sp<CaptureRequest> &request); 1053 // Purge the queued triggers from this request, 1054 // restoring the old field values for those tags. 1055 status_t removeTriggers(const sp<CaptureRequest> &request); 1056 1057 // HAL workaround: Make sure a trigger ID always exists if 1058 // a trigger does 1059 status_t addFakeTriggerIds(const sp<CaptureRequest> &request); 1060 1061 // Override rotate_and_crop control if needed; returns true if the current value was changed 1062 bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request /*out*/); 1063 1064 // Override autoframing control if needed; returns true if the current value was changed 1065 bool overrideAutoframing(const sp<CaptureRequest> &request); 1066 1067 // Override test_pattern control if needed for camera mute; returns true 1068 // if the current value was changed 1069 bool overrideTestPattern(const sp<CaptureRequest> &request); 1070 1071 // Override settings override if needed for lower zoom latency; return 1072 // true if the current value was changed 1073 bool overrideSettingsOverride(const sp<CaptureRequest> &request); 1074 1075 static const nsecs_t kRequestTimeout = 50e6; // 50 ms 1076 1077 // TODO: does this need to be adjusted for long exposure requests? 1078 static const nsecs_t kRequestSubmitTimeout = 500e6; // 500 ms 1079 1080 // Used to prepare a batch of requests. 1081 struct NextRequest { 1082 sp<CaptureRequest> captureRequest; 1083 camera_capture_request_t halRequest; 1084 Vector<camera_stream_buffer_t> outputBuffers; 1085 bool submitted; 1086 }; 1087 1088 // Wait for the next batch of requests and put them in mNextRequests. mNextRequests will 1089 // be empty if it times out. 1090 void waitForNextRequestBatch(); 1091 1092 // Waits for a request, or returns NULL if times out. Must be called with mRequestLock hold. 1093 sp<CaptureRequest> waitForNextRequestLocked(); 1094 1095 // Prepare HAL requests and output buffers in mNextRequests. Return TIMED_OUT if getting any 1096 // output buffer timed out. If an error is returned, the caller should clean up the pending 1097 // request batch. 1098 status_t prepareHalRequests(); 1099 1100 // Return buffers, etc, for requests in mNextRequests that couldn't be fully constructed and 1101 // send request errors if sendRequestError is true. The buffers will be returned in the 1102 // ERROR state to mark them as not having valid data. mNextRequests will be cleared. 1103 void cleanUpFailedRequests(bool sendRequestError); 1104 1105 // Stop the repeating request if any of its output streams is abandoned. 1106 void checkAndStopRepeatingRequest(); 1107 1108 // Release physical camera settings and camera id resources. 1109 void cleanupPhysicalSettings(sp<CaptureRequest> request, 1110 /*out*/camera_capture_request_t *halRequest); 1111 1112 // Pause handling 1113 bool waitIfPaused(); 1114 void unpauseForNewRequests(); 1115 1116 // Relay error to parent device object setErrorState 1117 void setErrorState(const char *fmt, ...); 1118 1119 // If the input request is in mRepeatingRequests. Must be called with mRequestLock hold 1120 bool isRepeatingRequestLocked(const sp<CaptureRequest>&); 1121 1122 // Clear repeating requests. Must be called with mRequestLock held. 1123 status_t clearRepeatingRequestsLocked(/*out*/ int64_t *lastFrameNumber = NULL); 1124 1125 // send request in mNextRequests to HAL in a batch. Return true = sucssess 1126 bool sendRequestsBatch(); 1127 1128 // Calculate the expected (minimum, maximum, isFixedFps) duration info for a request 1129 struct ExpectedDurationInfo { 1130 nsecs_t minDuration; 1131 nsecs_t maxDuration; 1132 bool isFixedFps; 1133 }; 1134 ExpectedDurationInfo calculateExpectedDurationRange( 1135 const camera_metadata_t *request); 1136 1137 // Check and update latest session parameters based on the current request settings. 1138 bool updateSessionParameters(const CameraMetadata& settings); 1139 1140 // Check whether FPS range session parameter re-configuration is needed in constrained 1141 // high speed recording camera sessions. 1142 bool skipHFRTargetFPSUpdate(int32_t tag, const camera_metadata_ro_entry_t& newEntry, 1143 const camera_metadata_entry_t& currentEntry); 1144 1145 // Update next request sent to HAL 1146 void updateNextRequest(NextRequest& nextRequest); 1147 1148 wp<Camera3Device> mParent; 1149 wp<camera3::StatusTracker> mStatusTracker; 1150 sp<HalInterface> mInterface; 1151 1152 wp<NotificationListener> mListener; 1153 1154 const std::string mId; // The camera ID 1155 int mStatusId; // The RequestThread's component ID for 1156 // status tracking 1157 1158 Mutex mRequestLock; 1159 Condition mRequestSignal; 1160 bool mRequestClearing; 1161 1162 Condition mRequestSubmittedSignal; 1163 RequestList mRequestQueue; 1164 RequestList mRepeatingRequests; 1165 bool mFirstRepeating; 1166 // The next batch of requests being prepped for submission to the HAL, no longer 1167 // on the request queue. Read-only even with mRequestLock held, outside 1168 // of threadLoop 1169 Vector<NextRequest> mNextRequests; 1170 1171 // To protect flush() and sending a request batch to HAL. 1172 Mutex mFlushLock; 1173 1174 bool mReconfigured; 1175 1176 // Used by waitIfPaused, waitForNextRequest, waitUntilPaused, and signalPipelineDrain 1177 Mutex mPauseLock; 1178 bool mDoPause; 1179 Condition mDoPauseSignal; 1180 bool mPaused; 1181 bool mNotifyPipelineDrain; 1182 std::vector<int> mStreamIdsToBeDrained; 1183 1184 sp<CaptureRequest> mPrevRequest; 1185 int32_t mPrevTriggers; 1186 std::set<std::string> mPrevCameraIdsWithZoom; 1187 1188 uint32_t mFrameNumber; 1189 1190 mutable Mutex mLatestRequestMutex; 1191 Condition mLatestRequestSignal; 1192 // android.request.id for latest process_capture_request 1193 int32_t mLatestRequestId; 1194 int32_t mLatestFailedRequestId; 1195 CameraMetadata mLatestRequest; 1196 std::unordered_map<std::string, CameraMetadata> mLatestPhysicalRequest; 1197 1198 typedef KeyedVector<uint32_t/*tag*/, RequestTrigger> TriggerMap; 1199 Mutex mTriggerMutex; 1200 TriggerMap mTriggerMap; 1201 TriggerMap mTriggerRemovedMap; 1202 TriggerMap mTriggerReplacedMap; 1203 uint32_t mCurrentAfTriggerId; 1204 uint32_t mCurrentPreCaptureTriggerId; 1205 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1206 camera_metadata_enum_android_control_autoframing_t mAutoframingOverride; 1207 bool mComposerOutput; 1208 int32_t mCameraMute; // 0 = no mute, otherwise the TEST_PATTERN_MODE to use 1209 int32_t mSettingsOverride; // -1 = use original, otherwise 1210 // the settings override to use. 1211 1212 int64_t mRepeatingLastFrameNumber; 1213 1214 // Flag indicating if we should prepare video stream for video requests. 1215 bool mPrepareVideoStream; 1216 1217 bool mConstrainedMode; 1218 1219 static const int32_t kRequestLatencyBinSize = 40; // in ms 1220 CameraLatencyHistogram mRequestLatency; 1221 1222 Vector<int32_t> mSessionParamKeys; 1223 CameraMetadata mLatestSessionParams; 1224 CameraMetadata mInjectedSessionParams; 1225 bool mForceNewRequestAfterReconfigure; 1226 1227 std::map<int32_t, std::set<std::string>> mGroupIdPhysicalCameraMap; 1228 1229 bool mUseHalBufManager = false; 1230 std::set<int32_t > mHalBufManagedStreamIds; 1231 const bool mSupportCameraMute; 1232 const bool mRotationOverride; 1233 const bool mSupportSettingsOverride; 1234 int32_t mVndkVersion = -1; 1235 }; 1236 1237 virtual sp<RequestThread> createNewRequestThread(wp<Camera3Device> /*parent*/, 1238 sp<camera3::StatusTracker> /*statusTracker*/, 1239 sp<HalInterface> /*interface*/, 1240 const Vector<int32_t>& /*sessionParamKeys*/, 1241 bool /*useHalBufManager*/, 1242 bool /*supportCameraMute*/, 1243 int /*rotationOverride*/, 1244 bool /*supportSettingsOverride*/) = 0; 1245 1246 sp<RequestThread> mRequestThread; 1247 1248 /** 1249 * In-flight queue for tracking completion of capture requests. 1250 */ 1251 std::mutex mInFlightLock; 1252 camera3::InFlightRequestMap mInFlightMap; 1253 nsecs_t mExpectedInflightDuration = 0; 1254 int64_t mLastCompletedRegularFrameNumber = -1; 1255 int64_t mLastCompletedReprocessFrameNumber = -1; 1256 int64_t mLastCompletedZslFrameNumber = -1; 1257 // End of mInFlightLock protection scope 1258 1259 int mInFlightStatusId; // const after initialize 1260 1261 status_t registerInFlight(uint32_t frameNumber, 1262 int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput, 1263 bool callback, nsecs_t minExpectedDuration, nsecs_t maxExpectedDuration, 1264 bool isFixedFps, const std::set<std::set<std::string>>& physicalCameraIds, 1265 bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto, bool autoframingAuto, 1266 const std::set<std::string>& cameraIdsWithZoom, const SurfaceMap& outputSurfaces, 1267 nsecs_t requestTimeNs); 1268 1269 /** 1270 * Tracking for idle detection 1271 */ 1272 sp<camera3::StatusTracker> mStatusTracker; 1273 1274 /** 1275 * Graphic buffer manager for output streams. Each device has a buffer manager, which is used 1276 * by the output streams to get and return buffers if these streams are registered to this 1277 * buffer manager. 1278 */ 1279 sp<camera3::Camera3BufferManager> mBufferManager; 1280 1281 /** 1282 * Thread for preparing streams 1283 */ 1284 class PreparerThread : private Thread, public virtual RefBase { 1285 public: 1286 PreparerThread(); 1287 ~PreparerThread(); 1288 1289 void setNotificationListener(wp<NotificationListener> listener); 1290 1291 /** 1292 * Queue up a stream to be prepared. Streams are processed by a background thread in FIFO 1293 * order. Pre-allocate up to maxCount buffers for the stream, or the maximum number needed 1294 * for the pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 1295 */ 1296 status_t prepare(int maxCount, sp<camera3::Camera3StreamInterface>& stream); 1297 1298 /** 1299 * Cancel all current and pending stream preparation 1300 */ 1301 status_t clear(); 1302 1303 /** 1304 * Pause all preparation activities 1305 */ 1306 void pause(); 1307 1308 /** 1309 * Resume preparation activities 1310 */ 1311 status_t resume(); 1312 1313 private: 1314 Mutex mLock; 1315 Condition mThreadActiveSignal; 1316 1317 virtual bool threadLoop(); 1318 1319 // Guarded by mLock 1320 1321 wp<NotificationListener> mListener; 1322 std::list<std::tuple<int, sp<camera3::Camera3StreamInterface>>> mPendingStreams; 1323 bool mActive; 1324 bool mCancelNow; 1325 1326 // Only accessed by threadLoop and the destructor 1327 1328 sp<camera3::Camera3StreamInterface> mCurrentStream; 1329 int mCurrentMaxCount; 1330 bool mCurrentPrepareComplete; 1331 }; 1332 sp<PreparerThread> mPreparerThread; 1333 1334 /** 1335 * Output result queue and current HAL device 3A state 1336 */ 1337 1338 // Lock for output side of device 1339 std::mutex mOutputLock; 1340 1341 /**** Scope for mOutputLock ****/ 1342 // the minimal frame number of the next non-reprocess result 1343 uint32_t mNextResultFrameNumber; 1344 // the minimal frame number of the next reprocess result 1345 uint32_t mNextReprocessResultFrameNumber; 1346 // the minimal frame number of the next ZSL still capture result 1347 uint32_t mNextZslStillResultFrameNumber; 1348 // the minimal frame number of the next non-reprocess shutter 1349 uint32_t mNextShutterFrameNumber; 1350 // the minimal frame number of the next reprocess shutter 1351 uint32_t mNextReprocessShutterFrameNumber; 1352 // the minimal frame number of the next ZSL still capture shutter 1353 uint32_t mNextZslStillShutterFrameNumber; 1354 std::list<CaptureResult> mResultQueue; 1355 std::condition_variable mResultSignal; 1356 wp<NotificationListener> mListener; 1357 1358 /**** End scope for mOutputLock ****/ 1359 1360 /**** Scope for mInFlightLock ****/ 1361 1362 // Remove the in-flight map entry of the given index from mInFlightMap. 1363 // It must only be called with mInFlightLock held. 1364 void removeInFlightMapEntryLocked(int idx); 1365 1366 // Remove all in-flight requests and return all buffers. 1367 // This is used after HAL interface is closed to cleanup any request/buffers 1368 // not returned by HAL. 1369 void flushInflightRequests(); 1370 1371 /**** End scope for mInFlightLock ****/ 1372 1373 /** 1374 * Distortion correction support 1375 */ 1376 // Map from camera IDs to its corresponding distortion mapper. Only contains 1377 // 1 ID if the device isn't a logical multi-camera. Otherwise contains both 1378 // logical camera and its physical subcameras. 1379 std::unordered_map<std::string, camera3::DistortionMapper> mDistortionMappers; 1380 1381 /** 1382 * Zoom ratio mapper support 1383 */ 1384 std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers; 1385 1386 /** 1387 * UHR request crop / metering region mapper support 1388 */ 1389 std::unordered_map<std::string, camera3::UHRCropAndMeteringRegionMapper> 1390 mUHRCropAndMeteringRegionMappers; 1391 1392 /** 1393 * RotateAndCrop mapper support 1394 */ 1395 std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers; 1396 1397 // Debug tracker for metadata tag value changes 1398 // - Enabled with the -m <taglist> option to dumpsys, such as 1399 // dumpsys -m android.control.aeState,android.control.aeMode 1400 // - Disabled with -m off 1401 // - dumpsys -m 3a is a shortcut for ae/af/awbMode, State, and Triggers 1402 TagMonitor mTagMonitor; 1403 1404 void monitorMetadata(TagMonitor::eventSource source, int64_t frameNumber, 1405 nsecs_t timestamp, const CameraMetadata& metadata, 1406 const std::unordered_map<std::string, CameraMetadata>& physicalMetadata, 1407 const camera_stream_buffer_t *outputBuffers, uint32_t numOutputBuffers, 1408 int32_t inputStreamId); 1409 1410 // Collect any statistics that are based on the stream of capture requests sent 1411 // to the HAL 1412 void collectRequestStats(int64_t frameNumber, const CameraMetadata& request); 1413 1414 metadata_vendor_id_t mVendorTagId; 1415 1416 // Cached last requested template id 1417 int mLastTemplateId; 1418 1419 // Synchronizes access to status tracker between inflight updates and disconnect. 1420 // b/79972865 1421 Mutex mTrackerLock; 1422 1423 // Whether HAL request buffers through requestStreamBuffers API 1424 bool mUseHalBufManager = false; 1425 std::set<int32_t > mHalBufManagedStreamIds; 1426 bool mSessionHalBufManager = false; 1427 // Lock to ensure requestStreamBuffers() callbacks are serialized 1428 std::mutex mRequestBufferInterfaceLock; 1429 1430 // The state machine to control when requestStreamBuffers should allow 1431 // HAL to request buffers. 1432 enum RequestBufferState { 1433 /** 1434 * This is the initial state. 1435 * requestStreamBuffers call will return FAILED_CONFIGURING in this state. 1436 * Will switch to RB_STATUS_READY after a successful configureStreams or 1437 * processCaptureRequest call. 1438 */ 1439 RB_STATUS_STOPPED, 1440 1441 /** 1442 * requestStreamBuffers call will proceed in this state. 1443 * When device is asked to stay idle via waitUntilStateThenRelock() call: 1444 * - Switch to RB_STATUS_STOPPED if there is no inflight requests and 1445 * request thread is paused. 1446 * - Switch to RB_STATUS_PENDING_STOP otherwise 1447 */ 1448 RB_STATUS_READY, 1449 1450 /** 1451 * requestStreamBuffers call will proceed in this state. 1452 * Switch to RB_STATUS_STOPPED when all inflight requests are fulfilled 1453 * and request thread is paused 1454 */ 1455 RB_STATUS_PENDING_STOP, 1456 }; 1457 1458 class RequestBufferStateMachine { 1459 public: 1460 status_t initialize(sp<camera3::StatusTracker> statusTracker); 1461 1462 status_t deInit(); 1463 1464 // Return if the state machine currently allows for requestBuffers 1465 // If the state allows for it, mRequestBufferOngoing will be set to true 1466 // and caller must call endRequestBuffer() later to unset the flag 1467 bool startRequestBuffer(); 1468 void endRequestBuffer(); 1469 1470 // Events triggered by application API call 1471 void onStreamsConfigured(); 1472 void onWaitUntilIdle(); 1473 1474 // Events usually triggered by hwBinder processCaptureResult callback thread 1475 // But can also be triggered on request thread for failed request, or on 1476 // hwbinder notify callback thread for shutter/error callbacks 1477 void onInflightMapEmpty(); 1478 1479 // Events triggered by RequestThread 1480 void onSubmittingRequest(); 1481 void onRequestThreadPaused(); 1482 1483 // Events triggered by successful switchToOffline call 1484 // Return true is there is no ongoing requestBuffer call. 1485 bool onSwitchToOfflineSuccess(); 1486 1487 private: 1488 void notifyTrackerLocked(bool active); 1489 1490 // Switch to STOPPED state and return true if all conditions allows for it. 1491 // Otherwise do nothing and return false. 1492 bool checkSwitchToStopLocked(); 1493 1494 std::mutex mLock; 1495 RequestBufferState mStatus = RB_STATUS_STOPPED; 1496 1497 bool mRequestThreadPaused = true; 1498 bool mInflightMapEmpty = true; 1499 bool mRequestBufferOngoing = false; 1500 bool mSwitchedToOffline = false; 1501 1502 wp<camera3::StatusTracker> mStatusTracker; 1503 int mRequestBufferStatusId; 1504 } mRequestBufferSM; 1505 1506 // Fix up result metadata for monochrome camera. 1507 bool mNeedFixupMonochromeTags; 1508 1509 // Whether HAL supports offline processing capability. 1510 bool mSupportOfflineProcessing = false; 1511 1512 // Whether the HAL supports camera muting via test pattern 1513 bool mSupportCameraMute = false; 1514 // Whether the HAL supports SOLID_COLOR or BLACK if mSupportCameraMute is true 1515 bool mSupportTestPatternSolidColor = false; 1516 // Whether the HAL supports zoom settings override 1517 bool mSupportZoomOverride = false; 1518 1519 // Whether the camera framework overrides the device characteristics for 1520 // performance class. 1521 bool mOverrideForPerfClass; 1522 1523 // Whether the camera framework overrides the device characteristics for 1524 // app compatibility reasons. 1525 int mRotationOverride; 1526 camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride; 1527 bool mComposerOutput; 1528 1529 // Auto framing override value 1530 camera_metadata_enum_android_control_autoframing mAutoframingOverride; 1531 1532 // Initial camera mute state stored before the request thread 1533 // is active. 1534 bool mCameraMuteInitial = false; 1535 1536 // Settings override value 1537 int32_t mSettingsOverride; // -1 = use original, otherwise 1538 // the settings override to use. 1539 1540 // Current active physical id of the logical multi-camera, if any 1541 std::string mActivePhysicalId; 1542 1543 // The current minimum expected frame duration based on AE_TARGET_FPS_RANGE 1544 nsecs_t mMinExpectedDuration = 0; 1545 // Whether the camera device runs at fixed frame rate based on AE_MODE and 1546 // AE_TARGET_FPS_RANGE 1547 bool mIsFixedFps = false; 1548 1549 // Flag to indicate that we shouldn't forward extension related metadata 1550 bool mSupportsExtensionKeys = false; 1551 1552 // Injection camera related methods. 1553 class Camera3DeviceInjectionMethods : public virtual RefBase { 1554 public: 1555 Camera3DeviceInjectionMethods(wp<Camera3Device> parent); 1556 1557 ~Camera3DeviceInjectionMethods(); 1558 1559 // Injection camera will replace the internal camera and configure streams 1560 // when device is IDLE and request thread is paused. 1561 status_t injectCamera( 1562 camera3::camera_stream_configuration& injectionConfig, 1563 const std::vector<uint32_t>& injectionBufferSizes); 1564 1565 // Stop the injection camera and switch back to backup hal interface. 1566 status_t stopInjection(); 1567 1568 bool isInjecting(); 1569 1570 bool isStreamConfigCompleteButNotInjected(); 1571 1572 const std::string& getInjectedCamId() const; 1573 1574 void getInjectionConfig(/*out*/ camera3::camera_stream_configuration* injectionConfig, 1575 /*out*/ std::vector<uint32_t>* injectionBufferSizes); 1576 1577 // When the streaming configuration is completed and the camera device is active, but the 1578 // injection camera has not yet been injected, the streaming configuration of the internal 1579 // camera will be stored first. 1580 void storeInjectionConfig( 1581 const camera3::camera_stream_configuration& injectionConfig, 1582 const std::vector<uint32_t>& injectionBufferSizes); 1583 1584 protected: 1585 // Configure the streams of injection camera, it need wait until the 1586 // output streams are created and configured to the original camera before 1587 // proceeding. 1588 status_t injectionConfigureStreams( 1589 camera3::camera_stream_configuration& injectionConfig, 1590 const std::vector<uint32_t>& injectionBufferSizes); 1591 1592 // Disconnect the injection camera and delete the hal interface. 1593 void injectionDisconnectImpl(); 1594 1595 // Use injection camera hal interface to replace and backup original 1596 // camera hal interface. 1597 virtual status_t replaceHalInterface(sp<HalInterface> /*newHalInterface*/, 1598 bool /*keepBackup*/) = 0; 1599 1600 wp<Camera3Device> mParent; 1601 1602 // Backup of the original camera hal interface. 1603 sp<HalInterface> mBackupHalInterface; 1604 1605 // Generated injection camera hal interface. 1606 sp<HalInterface> mInjectedCamHalInterface; 1607 1608 // The flag indicates that the stream configuration is complete, the camera device is 1609 // active, but the injection camera has not yet been injected. 1610 bool mIsStreamConfigCompleteButNotInjected = false; 1611 1612 // Copy the configuration of the internal camera. 1613 camera3::camera_stream_configuration mInjectionConfig; 1614 1615 // Copy the streams of the internal camera. 1616 Vector<camera3::camera_stream_t*> mInjectionStreams; 1617 1618 // Copy the bufferSizes of the output streams of the internal camera. 1619 std::vector<uint32_t> mInjectionBufferSizes; 1620 1621 // Synchronizes access to injection camera between initialize and 1622 // disconnect. 1623 Mutex mInjectionLock; 1624 1625 // The injection camera ID. 1626 std::string mInjectedCamId; 1627 }; 1628 1629 virtual sp<Camera3DeviceInjectionMethods> 1630 createCamera3DeviceInjectionMethods(wp<Camera3Device>) = 0; 1631 1632 sp<Camera3DeviceInjectionMethods> mInjectionMethods; 1633 1634 void overrideStreamUseCaseLocked(); 1635 1636 1637 }; // class Camera3Device 1638 1639 }; // namespace android 1640 1641 #endif 1642