1 /* 2 * Copyright (C) 2013-2018 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef ANDROID_SERVERS_CAMERA3_STREAM_H 18 #define ANDROID_SERVERS_CAMERA3_STREAM_H 19 20 #include <gui/Surface.h> 21 #include <utils/RefBase.h> 22 #include <utils/String8.h> 23 #include <utils/String16.h> 24 #include <utils/List.h> 25 26 #include "hardware/camera3.h" 27 28 #include "utils/LatencyHistogram.h" 29 #include "Camera3StreamBufferListener.h" 30 #include "Camera3StreamInterface.h" 31 32 namespace android { 33 34 namespace camera3 { 35 36 /** 37 * A class for managing a single stream of input or output data from the camera 38 * device. 39 * 40 * The stream has an internal state machine to track whether it's 41 * connected/configured/etc. 42 * 43 * States: 44 * 45 * STATE_ERROR: A serious error has occurred, stream is unusable. Outstanding 46 * buffers may still be returned. 47 * 48 * STATE_CONSTRUCTED: The stream is ready for configuration, but buffers cannot 49 * be gotten yet. Not connected to any endpoint, no buffers are registered 50 * with the HAL. 51 * 52 * STATE_IN_CONFIG: Configuration has started, but not yet concluded. During this 53 * time, the usage, max_buffers, and priv fields of camera3_stream returned by 54 * startConfiguration() may be modified. 55 * 56 * STATE_IN_RE_CONFIG: Configuration has started, and the stream has been 57 * configured before. Need to track separately from IN_CONFIG to avoid 58 * re-registering buffers with HAL. 59 * 60 * STATE_CONFIGURED: Stream is configured, and has registered buffers with the 61 * HAL (if necessary). The stream's getBuffer/returnBuffer work. The priv 62 * pointer may still be modified. 63 * 64 * STATE_PREPARING: The stream's buffers are being pre-allocated for use. On 65 * older HALs, this is done as part of configuration, but in newer HALs 66 * buffers may be allocated at time of first use. But some use cases require 67 * buffer allocation upfront, to minmize disruption due to lengthy allocation 68 * duration. In this state, only prepareNextBuffer() and cancelPrepare() 69 * may be called. 70 * 71 * STATE_IN_IDLE: This is a temporary state only intended to be used for input 72 * streams and only for the case where we need to re-configure the camera device 73 * while the input stream has an outstanding buffer. All other streams should not 74 * be able to switch to this state. For them this is invalid and should be handled 75 * as an unknown state. 76 * 77 * Transition table: 78 * 79 * <none> => STATE_CONSTRUCTED: 80 * When constructed with valid arguments 81 * <none> => STATE_ERROR: 82 * When constructed with invalid arguments 83 * STATE_CONSTRUCTED => STATE_IN_CONFIG: 84 * When startConfiguration() is called 85 * STATE_IN_CONFIG => STATE_CONFIGURED: 86 * When finishConfiguration() is called 87 * STATE_IN_CONFIG => STATE_ERROR: 88 * When finishConfiguration() fails to allocate or register buffers. 89 * STATE_CONFIGURED => STATE_IN_RE_CONFIG: * 90 * When startConfiguration() is called again, after making sure stream is 91 * idle with waitUntilIdle(). 92 * STATE_IN_RE_CONFIG => STATE_CONFIGURED: 93 * When finishConfiguration() is called. 94 * STATE_IN_RE_CONFIG => STATE_ERROR: 95 * When finishConfiguration() fails to allocate or register buffers. 96 * STATE_CONFIGURED => STATE_CONSTRUCTED: 97 * When disconnect() is called after making sure stream is idle with 98 * waitUntilIdle(). 99 * STATE_CONFIGURED => STATE_PREPARING: 100 * When startPrepare is called before the stream has a buffer 101 * queued back into it for the first time. 102 * STATE_PREPARING => STATE_CONFIGURED: 103 * When sufficient prepareNextBuffer calls have been made to allocate 104 * all stream buffers, or cancelPrepare is called. 105 * STATE_CONFIGURED => STATE_ABANDONED: 106 * When the buffer queue of the stream is abandoned. 107 * STATE_CONFIGURED => STATE_IN_IDLE: 108 * Only for an input stream which has an outstanding buffer. 109 * STATE_IN_IDLE => STATE_CONFIGURED: 110 * After the internal re-configuration, the input should revert back to 111 * the configured state. 112 * 113 * Status Tracking: 114 * Each stream is tracked by StatusTracker as a separate component, 115 * depending on the handed out buffer count. The state must be STATE_CONFIGURED 116 * in order for the component to be marked. 117 * 118 * It's marked in one of two ways: 119 * 120 * - ACTIVE: One or more buffers have been handed out (with #getBuffer). 121 * - IDLE: All buffers have been returned (with #returnBuffer), and their 122 * respective release_fence(s) have been signaled. The only exception to this 123 * rule is an input stream that moves to "STATE_IN_IDLE" during internal 124 * re-configuration. 125 * 126 * A typical use case is output streams. When the HAL has any buffers 127 * dequeued, the stream is marked ACTIVE. When the HAL returns all buffers 128 * (e.g. if no capture requests are active), the stream is marked IDLE. 129 * In this use case, the app consumer does not affect the component status. 130 * 131 */ 132 class Camera3Stream : 133 protected camera3_stream, 134 public virtual Camera3StreamInterface, 135 public virtual RefBase { 136 public: 137 138 virtual ~Camera3Stream(); 139 140 static Camera3Stream* cast(camera3_stream *stream); 141 static const Camera3Stream* cast(const camera3_stream *stream); 142 143 /** 144 * Get the stream's ID 145 */ 146 int getId() const; 147 148 /** 149 * Get the output stream set id. 150 */ 151 int getStreamSetId() const; 152 153 /** 154 * Get the stream's dimensions and format 155 */ 156 uint32_t getWidth() const; 157 uint32_t getHeight() const; 158 int getFormat() const; 159 android_dataspace getDataSpace() const; 160 uint64_t getUsage() const; 161 void setUsage(uint64_t usage); 162 void setFormatOverride(bool formatOverriden); 163 bool isFormatOverridden() const; 164 int getOriginalFormat() const; 165 void setDataSpaceOverride(bool dataSpaceOverriden); 166 bool isDataSpaceOverridden() const; 167 android_dataspace getOriginalDataSpace() const; 168 const String8& physicalCameraId() const; 169 asHalStream()170 camera3_stream* asHalStream() override { 171 return this; 172 } 173 174 /** 175 * Start the stream configuration process. Returns a handle to the stream's 176 * information to be passed into the HAL device's configure_streams call. 177 * 178 * Until finishConfiguration() is called, no other methods on the stream may be 179 * called. The usage and max_buffers fields of camera3_stream may be modified 180 * between start/finishConfiguration, but may not be changed after that. 181 * The priv field of camera3_stream may be modified at any time after 182 * startConfiguration. 183 * 184 * Returns NULL in case of error starting configuration. 185 */ 186 camera3_stream* startConfiguration(); 187 188 /** 189 * Check if the stream is mid-configuration (start has been called, but not 190 * finish). Used for lazy completion of configuration. 191 */ 192 bool isConfiguring() const; 193 194 /** 195 * Completes the stream configuration process. The stream information 196 * structure returned by startConfiguration() may no longer be modified 197 * after this call, but can still be read until the destruction of the 198 * stream. 199 * 200 * Returns: 201 * OK on a successful configuration 202 * NO_INIT in case of a serious error from the HAL device 203 * NO_MEMORY in case of an error registering buffers 204 * INVALID_OPERATION in case connecting to the consumer failed or consumer 205 * doesn't exist yet. 206 */ 207 status_t finishConfiguration(); 208 209 /** 210 * Cancels the stream configuration process. This returns the stream to the 211 * initial state, allowing it to be configured again later. 212 * This is done if the HAL rejects the proposed combined stream configuration 213 */ 214 status_t cancelConfiguration(); 215 216 /** 217 * Determine whether the stream has already become in-use (has received 218 * a valid filled buffer), which determines if a stream can still have 219 * prepareNextBuffer called on it. 220 */ 221 bool isUnpreparable(); 222 223 /** 224 * Start stream preparation. May only be called in the CONFIGURED state, 225 * when no valid buffers have yet been returned to this stream. Prepares 226 * up to maxCount buffers, or the maximum number of buffers needed by the 227 * pipeline if maxCount is ALLOCATE_PIPELINE_MAX. 228 * 229 * If no prepartion is necessary, returns OK and does not transition to 230 * PREPARING state. Otherwise, returns NOT_ENOUGH_DATA and transitions 231 * to PREPARING. 232 * 233 * This call performs no allocation, so is quick to call. 234 * 235 * Returns: 236 * OK if no more buffers need to be preallocated 237 * NOT_ENOUGH_DATA if calls to prepareNextBuffer are needed to finish 238 * buffer pre-allocation, and transitions to the PREPARING state. 239 * NO_INIT in case of a serious error from the HAL device 240 * INVALID_OPERATION if called when not in CONFIGURED state, or a 241 * valid buffer has already been returned to this stream. 242 */ 243 status_t startPrepare(int maxCount); 244 245 /** 246 * Check if the stream is mid-preparing. 247 */ 248 bool isPreparing() const; 249 250 /** 251 * Continue stream buffer preparation by allocating the next 252 * buffer for this stream. May only be called in the PREPARED state. 253 * 254 * Returns OK and transitions to the CONFIGURED state if all buffers 255 * are allocated after the call concludes. Otherwise returns NOT_ENOUGH_DATA. 256 * 257 * This call allocates one buffer, which may take several milliseconds for 258 * large buffers. 259 * 260 * Returns: 261 * OK if no more buffers need to be preallocated, and transitions 262 * to the CONFIGURED state. 263 * NOT_ENOUGH_DATA if more calls to prepareNextBuffer are needed to finish 264 * buffer pre-allocation. 265 * NO_INIT in case of a serious error from the HAL device 266 * INVALID_OPERATION if called when not in CONFIGURED state, or a 267 * valid buffer has already been returned to this stream. 268 */ 269 status_t prepareNextBuffer(); 270 271 /** 272 * Cancel stream preparation early. In case allocation needs to be 273 * stopped, this method transitions the stream back to the CONFIGURED state. 274 * Buffers that have been allocated with prepareNextBuffer remain that way, 275 * but a later use of prepareNextBuffer will require just as many 276 * calls as if the earlier prepare attempt had not existed. 277 * 278 * Returns: 279 * OK if cancellation succeeded, and transitions to the CONFIGURED state 280 * INVALID_OPERATION if not in the PREPARING state 281 * NO_INIT in case of a serious error from the HAL device 282 */ 283 status_t cancelPrepare(); 284 285 /** 286 * Tear down memory for this stream. This frees all unused gralloc buffers 287 * allocated for this stream, but leaves it ready for operation afterward. 288 * 289 * May only be called in the CONFIGURED state, and keeps the stream in 290 * the CONFIGURED state. 291 * 292 * Returns: 293 * OK if teardown succeeded. 294 * INVALID_OPERATION if not in the CONFIGURED state 295 * NO_INIT in case of a serious error from the HAL device 296 */ 297 status_t tearDown(); 298 299 /** 300 * Fill in the camera3_stream_buffer with the next valid buffer for this 301 * stream, to hand over to the HAL. 302 * 303 * Multiple surfaces could share the same HAL stream, but a request may 304 * be only for a subset of surfaces. In this case, the 305 * Camera3StreamInterface object needs the surface ID information to acquire 306 * buffers for those surfaces. 307 * 308 * This method may only be called once finishConfiguration has been called. 309 * For bidirectional streams, this method applies to the output-side 310 * buffers. 311 * 312 */ 313 status_t getBuffer(camera3_stream_buffer *buffer, 314 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 315 316 /** 317 * Return a buffer to the stream after use by the HAL. 318 * 319 * This method may only be called for buffers provided by getBuffer(). 320 * For bidirectional streams, this method applies to the output-side buffers 321 */ 322 status_t returnBuffer(const camera3_stream_buffer &buffer, 323 nsecs_t timestamp); 324 325 /** 326 * Fill in the camera3_stream_buffer with the next valid buffer for this 327 * stream, to hand over to the HAL. 328 * 329 * This method may only be called once finishConfiguration has been called. 330 * For bidirectional streams, this method applies to the input-side 331 * buffers. 332 * 333 * Normally this call will block until the handed out buffer count is less than the stream 334 * max buffer count; if respectHalLimit is set to false, this is ignored. 335 */ 336 status_t getInputBuffer(camera3_stream_buffer *buffer, bool respectHalLimit = true); 337 338 /** 339 * Return a buffer to the stream after use by the HAL. 340 * 341 * This method may only be called for buffers provided by getBuffer(). 342 * For bidirectional streams, this method applies to the input-side buffers 343 */ 344 status_t returnInputBuffer(const camera3_stream_buffer &buffer); 345 346 // get the buffer producer of the input buffer queue. 347 // only apply to input streams. 348 status_t getInputBufferProducer(sp<IGraphicBufferProducer> *producer); 349 350 /** 351 * Whether any of the stream's buffers are currently in use by the HAL, 352 * including buffers that have been returned but not yet had their 353 * release fence signaled. 354 */ 355 bool hasOutstandingBuffers() const; 356 357 enum { 358 TIMEOUT_NEVER = -1 359 }; 360 361 /** 362 * Set the status tracker to notify about idle transitions 363 */ 364 virtual status_t setStatusTracker(sp<StatusTracker> statusTracker); 365 366 /** 367 * Disconnect stream from its non-HAL endpoint. After this, 368 * start/finishConfiguration must be called before the stream can be used 369 * again. This cannot be called if the stream has outstanding dequeued 370 * buffers. 371 */ 372 status_t disconnect(); 373 374 /** 375 * Debug dump of the stream's state. 376 */ 377 virtual void dump(int fd, const Vector<String16> &args) const; 378 379 /** 380 * Add a camera3 buffer listener. Adding the same listener twice has 381 * no effect. 382 */ 383 void addBufferListener( 384 wp<Camera3StreamBufferListener> listener); 385 386 /** 387 * Remove a camera3 buffer listener. Removing the same listener twice 388 * or the listener that was never added has no effect. 389 */ 390 void removeBufferListener( 391 const sp<Camera3StreamBufferListener>& listener); 392 393 394 // Setting listener will remove previous listener (if exists) 395 virtual void setBufferFreedListener( 396 wp<Camera3StreamBufferFreedListener> listener) override; 397 398 /** 399 * Return if the buffer queue of the stream is abandoned. 400 */ 401 bool isAbandoned() const; 402 403 /** 404 * Switch a configured stream with possibly outstanding buffers in idle 405 * state. Configuration for such streams will be skipped assuming there 406 * are no changes to the stream parameters. 407 */ 408 status_t forceToIdle(); 409 410 /** 411 * Restore a forced idle stream to configured state, marking it active 412 * in case it contains outstanding buffers. 413 */ 414 status_t restoreConfiguredState(); 415 416 protected: 417 const int mId; 418 /** 419 * Stream set id, used to indicate which group of this stream belongs to for buffer sharing 420 * across multiple streams. 421 * 422 * The default value is set to CAMERA3_STREAM_SET_ID_INVALID, which indicates that this stream 423 * doesn't intend to share buffers with any other streams, and this stream will fall back to 424 * the existing BufferQueue mechanism to manage the buffer allocations and buffer circulation. 425 * When a valid stream set id is set, this stream intends to use the Camera3BufferManager to 426 * manage the buffer allocations; the BufferQueue will only handle the buffer transaction 427 * between the producer and consumer. For this case, upon successfully registration, the streams 428 * with the same stream set id will potentially share the buffers allocated by 429 * Camera3BufferManager. 430 */ 431 const int mSetId; 432 433 const String8 mName; 434 // Zero for formats with fixed buffer size for given dimensions. 435 const size_t mMaxSize; 436 437 enum { 438 STATE_ERROR, 439 STATE_CONSTRUCTED, 440 STATE_IN_CONFIG, 441 STATE_IN_RECONFIG, 442 STATE_CONFIGURED, 443 STATE_PREPARING, 444 STATE_ABANDONED, 445 STATE_IN_IDLE 446 } mState; 447 448 mutable Mutex mLock; 449 450 Camera3Stream(int id, camera3_stream_type type, 451 uint32_t width, uint32_t height, size_t maxSize, int format, 452 android_dataspace dataSpace, camera3_stream_rotation_t rotation, 453 const String8& physicalCameraId, int setId); 454 455 wp<Camera3StreamBufferFreedListener> mBufferFreedListener; 456 457 /** 458 * Interface to be implemented by derived classes 459 */ 460 461 // getBuffer / returnBuffer implementations 462 463 // Since camera3_stream_buffer includes a raw pointer to the stream, 464 // cast to camera3_stream*, implementations must increment the 465 // refcount of the stream manually in getBufferLocked, and decrement it in 466 // returnBufferLocked. 467 virtual status_t getBufferLocked(camera3_stream_buffer *buffer, 468 const std::vector<size_t>& surface_ids = std::vector<size_t>()); 469 virtual status_t returnBufferLocked(const camera3_stream_buffer &buffer, 470 nsecs_t timestamp); 471 virtual status_t getInputBufferLocked(camera3_stream_buffer *buffer); 472 virtual status_t returnInputBufferLocked( 473 const camera3_stream_buffer &buffer); 474 virtual bool hasOutstandingBuffersLocked() const = 0; 475 // Get the buffer producer of the input buffer queue. Only apply to input streams. 476 virtual status_t getInputBufferProducerLocked(sp<IGraphicBufferProducer> *producer); 477 478 // Can return -ENOTCONN when we are already disconnected (not an error) 479 virtual status_t disconnectLocked() = 0; 480 481 // Configure the buffer queue interface to the other end of the stream, 482 // after the HAL has provided usage and max_buffers values. After this call, 483 // the stream must be ready to produce all buffers for registration with 484 // HAL. 485 virtual status_t configureQueueLocked() = 0; 486 487 // Get the total number of buffers in the queue 488 virtual size_t getBufferCountLocked() = 0; 489 490 // Get handout output buffer count. 491 virtual size_t getHandoutOutputBufferCountLocked() = 0; 492 493 // Get handout input buffer count. 494 virtual size_t getHandoutInputBufferCountLocked() = 0; 495 496 // Get the usage flags for the other endpoint, or return 497 // INVALID_OPERATION if they cannot be obtained. 498 virtual status_t getEndpointUsage(uint64_t *usage) const = 0; 499 500 // Return whether the buffer is in the list of outstanding buffers. 501 bool isOutstandingBuffer(const camera3_stream_buffer& buffer) const; 502 503 // Tracking for idle state 504 wp<StatusTracker> mStatusTracker; 505 // Status tracker component ID 506 int mStatusId; 507 508 // Tracking for stream prepare - whether this stream can still have 509 // prepareNextBuffer called on it. 510 bool mStreamUnpreparable; 511 512 uint64_t mUsage; 513 514 private: 515 uint64_t mOldUsage; 516 uint32_t mOldMaxBuffers; 517 Condition mOutputBufferReturnedSignal; 518 Condition mInputBufferReturnedSignal; 519 static const nsecs_t kWaitForBufferDuration = 3000000000LL; // 3000 ms 520 521 void fireBufferListenersLocked(const camera3_stream_buffer& buffer, 522 bool acquired, bool output); 523 List<wp<Camera3StreamBufferListener> > mBufferListenerList; 524 525 status_t cancelPrepareLocked(); 526 527 // Remove the buffer from the list of outstanding buffers. 528 void removeOutstandingBuffer(const camera3_stream_buffer& buffer); 529 530 // Tracking for PREPARING state 531 532 // State of buffer preallocation. Only true if either prepareNextBuffer 533 // has been called sufficient number of times, or stream configuration 534 // had to register buffers with the HAL 535 bool mPrepared; 536 537 Vector<camera3_stream_buffer_t> mPreparedBuffers; 538 size_t mPreparedBufferIdx; 539 540 // Number of buffers allocated on last prepare call. 541 size_t mLastMaxCount; 542 543 mutable Mutex mOutstandingBuffersLock; 544 // Outstanding buffers dequeued from the stream's buffer queue. 545 List<buffer_handle_t> mOutstandingBuffers; 546 547 // Latency histogram of the wait time for handout buffer count to drop below 548 // max_buffers. 549 static const int32_t kBufferLimitLatencyBinSize = 33; //in ms 550 CameraLatencyHistogram mBufferLimitLatency; 551 552 //Keep track of original format in case it gets overridden 553 bool mFormatOverridden; 554 int mOriginalFormat; 555 556 //Keep track of original dataSpace in case it gets overridden 557 bool mDataSpaceOverridden; 558 android_dataspace mOriginalDataSpace; 559 560 String8 mPhysicalCameraId; 561 }; // class Camera3Stream 562 563 }; // namespace camera3 564 565 }; // namespace android 566 567 #endif 568