1 /* 2 * Copyright 2019 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #pragma once 18 19 #include <android/hardware_buffer.h> 20 #include <gui/BufferQueueDefs.h> 21 #include <gui/ConsumerBase.h> 22 23 #include <gui/IGraphicBufferProducer.h> 24 #include <sys/cdefs.h> 25 #include <system/graphics.h> 26 #include <ui/FenceTime.h> 27 #include <ui/GraphicBuffer.h> 28 #include <utils/Mutex.h> 29 #include <utils/String8.h> 30 31 #include "EGLConsumer.h" 32 #include "ImageConsumer.h" 33 34 namespace android { 35 36 /* 37 * SurfaceTexture consumes buffers of graphics data from a BufferQueue, 38 * and makes them available to HWUI render thread as a SkImage and to 39 * an application GL render thread as an OpenGL texture. 40 * 41 * When attached to an application GL render thread, a typical usage 42 * pattern is to set up the SurfaceTexture with the 43 * desired options, and call updateTexImage() when a new frame is desired. 44 * If a new frame is available, the texture will be updated. If not, 45 * the previous contents are retained. 46 * 47 * When attached to a HWUI render thread, the TextureView implementation 48 * calls dequeueBuffer, which either pulls a new buffer or returns the 49 * last cached buffer if BufferQueue is empty. 50 * When attached to HWUI render thread, SurfaceTexture is compatible to 51 * both Vulkan and GL drawing pipelines. 52 */ 53 class ANDROID_API SurfaceTexture : public ConsumerBase { 54 public: 55 /** 56 * Callback function needed by dequeueBuffer. It creates a fence, 57 * that is signalled, when the previous buffer is no longer in use by HWUI 58 * and can be used by written by the producer. 59 */ 60 typedef status_t (*SurfaceTexture_createReleaseFence)(bool useFenceSync, EGLSyncKHR* eglFence, 61 EGLDisplay* display, int* releaseFence, 62 void* passThroughHandle); 63 64 /** 65 * Callback function needed by dequeueBuffer. It waits for the new buffer 66 * fence to signal, before issuing any draw commands. 67 */ 68 typedef status_t (*SurfaceTexture_fenceWait)(int fence, void* passThroughHandle); 69 70 enum { TEXTURE_EXTERNAL = 0x8D65 }; // GL_TEXTURE_EXTERNAL_OES 71 typedef ConsumerBase::FrameAvailableListener FrameAvailableListener; 72 73 /** 74 * SurfaceTexture constructs a new SurfaceTexture object. If the constructor 75 * with the tex parameter is used, tex indicates the name of the OpenGL ES 76 * texture to which images are to be streamed. texTarget specifies the 77 * OpenGL ES texture target to which the texture will be bound in 78 * updateTexImage. useFenceSync specifies whether fences should be used to 79 * synchronize access to buffers if that behavior is enabled at 80 * compile-time. 81 * 82 * A SurfaceTexture may be detached from one OpenGL ES context and then 83 * attached to a different context using the detachFromContext and 84 * attachToContext methods, respectively. The intention of these methods is 85 * purely to allow a SurfaceTexture to be transferred from one consumer 86 * context to another. If such a transfer is not needed there is no 87 * requirement that either of these methods be called. 88 * 89 * If the constructor with the tex parameter is used, the SurfaceTexture is 90 * created in a state where it is considered attached to an OpenGL ES 91 * context for the purposes of the attachToContext and detachFromContext 92 * methods. However, despite being considered "attached" to a context, the 93 * specific OpenGL ES context doesn't get latched until the first call to 94 * updateTexImage. After that point, all calls to updateTexImage must be 95 * made with the same OpenGL ES context current. 96 * 97 * If the constructor without the tex parameter is used, the SurfaceTexture 98 * is created in a detached state, and attachToContext must be called before 99 * calls to updateTexImage. 100 */ 101 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t tex, uint32_t textureTarget, 102 bool useFenceSync, bool isControlledByApp); 103 104 SurfaceTexture(const sp<IGraphicBufferConsumer>& bq, uint32_t textureTarget, bool useFenceSync, 105 bool isControlledByApp); 106 107 /** 108 * updateTexImage acquires the most recently queued buffer, and sets the 109 * image contents of the target texture to it. 110 * 111 * This call may only be made while the OpenGL ES context to which the 112 * target texture belongs is bound to the calling thread. 113 * 114 * This calls doGLFenceWait to ensure proper synchronization. 115 */ 116 status_t updateTexImage(); 117 118 /** 119 * releaseTexImage releases the texture acquired in updateTexImage(). 120 * This is intended to be used in single buffer mode. 121 * 122 * This call may only be made while the OpenGL ES context to which the 123 * target texture belongs is bound to the calling thread. 124 */ 125 status_t releaseTexImage(); 126 127 /** 128 * getTransformMatrix retrieves the 4x4 texture coordinate transform matrix 129 * associated with the texture image set by the most recent call to 130 * updateTexImage. 131 * 132 * This transform matrix maps 2D homogeneous texture coordinates of the form 133 * (s, t, 0, 1) with s and t in the inclusive range [0, 1] to the texture 134 * coordinate that should be used to sample that location from the texture. 135 * Sampling the texture outside of the range of this transform is undefined. 136 * 137 * This transform is necessary to compensate for transforms that the stream 138 * content producer may implicitly apply to the content. By forcing users of 139 * a SurfaceTexture to apply this transform we avoid performing an extra 140 * copy of the data that would be needed to hide the transform from the 141 * user. 142 * 143 * The matrix is stored in column-major order so that it may be passed 144 * directly to OpenGL ES via the glLoadMatrixf or glUniformMatrix4fv 145 * functions. 146 */ 147 void getTransformMatrix(float mtx[16]); 148 149 /** 150 * Computes the transform matrix documented by getTransformMatrix 151 * from the BufferItem sub parts. 152 */ 153 static void computeTransformMatrix(float outTransform[16], const sp<GraphicBuffer>& buf, 154 const Rect& cropRect, uint32_t transform, bool filtering); 155 156 /** 157 * Scale the crop down horizontally or vertically such that it has the 158 * same aspect ratio as the buffer does. 159 */ 160 static Rect scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight); 161 162 /** 163 * getTimestamp retrieves the timestamp associated with the texture image 164 * set by the most recent call to updateTexImage. 165 * 166 * The timestamp is in nanoseconds, and is monotonically increasing. Its 167 * other semantics (zero point, etc) are source-dependent and should be 168 * documented by the source. 169 */ 170 int64_t getTimestamp(); 171 172 /** 173 * getDataSpace retrieves the DataSpace associated with the texture image 174 * set by the most recent call to updateTexImage. 175 */ 176 android_dataspace getCurrentDataSpace(); 177 178 /** 179 * getFrameNumber retrieves the frame number associated with the texture 180 * image set by the most recent call to updateTexImage. 181 * 182 * The frame number is an incrementing counter set to 0 at the creation of 183 * the BufferQueue associated with this consumer. 184 */ 185 uint64_t getFrameNumber(); 186 187 /** 188 * setDefaultBufferSize is used to set the size of buffers returned by 189 * requestBuffers when a with and height of zero is requested. 190 * A call to setDefaultBufferSize() may trigger requestBuffers() to 191 * be called from the client. 192 * The width and height parameters must be no greater than the minimum of 193 * GL_MAX_VIEWPORT_DIMS and GL_MAX_TEXTURE_SIZE (see: glGetIntegerv). 194 * An error due to invalid dimensions might not be reported until 195 * updateTexImage() is called. 196 */ 197 status_t setDefaultBufferSize(uint32_t width, uint32_t height); 198 199 /** 200 * setFilteringEnabled sets whether the transform matrix should be computed 201 * for use with bilinear filtering. 202 */ 203 void setFilteringEnabled(bool enabled); 204 205 /** 206 * getCurrentTextureTarget returns the texture target of the current 207 * texture as returned by updateTexImage(). 208 */ 209 uint32_t getCurrentTextureTarget() const; 210 211 /** 212 * getCurrentCrop returns the cropping rectangle of the current buffer. 213 */ 214 Rect getCurrentCrop() const; 215 216 /** 217 * getCurrentTransform returns the transform of the current buffer. 218 */ 219 uint32_t getCurrentTransform() const; 220 221 /** 222 * getCurrentScalingMode returns the scaling mode of the current buffer. 223 */ 224 uint32_t getCurrentScalingMode() const; 225 226 /** 227 * getCurrentFence returns the fence indicating when the current buffer is 228 * ready to be read from. 229 */ 230 sp<Fence> getCurrentFence() const; 231 232 /** 233 * getCurrentFence returns the FenceTime indicating when the current 234 * buffer is ready to be read from. 235 */ 236 std::shared_ptr<FenceTime> getCurrentFenceTime() const; 237 238 /** 239 * setConsumerUsageBits overrides the ConsumerBase method to OR 240 * DEFAULT_USAGE_FLAGS to usage. 241 */ 242 status_t setConsumerUsageBits(uint64_t usage); 243 244 /** 245 * detachFromContext detaches the SurfaceTexture from the calling thread's 246 * current OpenGL ES context. This context must be the same as the context 247 * that was current for previous calls to updateTexImage. 248 * 249 * Detaching a SurfaceTexture from an OpenGL ES context will result in the 250 * deletion of the OpenGL ES texture object into which the images were being 251 * streamed. After a SurfaceTexture has been detached from the OpenGL ES 252 * context calls to updateTexImage will fail returning INVALID_OPERATION 253 * until the SurfaceTexture is attached to a new OpenGL ES context using the 254 * attachToContext method. 255 */ 256 status_t detachFromContext(); 257 258 /** 259 * attachToContext attaches a SurfaceTexture that is currently in the 260 * 'detached' state to the current OpenGL ES context. A SurfaceTexture is 261 * in the 'detached' state iff detachFromContext has successfully been 262 * called and no calls to attachToContext have succeeded since the last 263 * detachFromContext call. Calls to attachToContext made on a 264 * SurfaceTexture that is not in the 'detached' state will result in an 265 * INVALID_OPERATION error. 266 * 267 * The tex argument specifies the OpenGL ES texture object name in the 268 * new context into which the image contents will be streamed. A successful 269 * call to attachToContext will result in this texture object being bound to 270 * the texture target and populated with the image contents that were 271 * current at the time of the last call to detachFromContext. 272 */ 273 status_t attachToContext(uint32_t tex); 274 275 sp<GraphicBuffer> dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, 276 HdrMetadata* outHdrMetadata, float* outTransformMatrix, 277 uint32_t* outTransform, bool* outQueueEmpty, 278 SurfaceTexture_createReleaseFence createFence, 279 SurfaceTexture_fenceWait fenceWait, 280 void* fencePassThroughHandle, ARect* currentCrop); 281 282 /** 283 * takeConsumerOwnership attaches a SurfaceTexture that is currently in the 284 * 'detached' state to a consumer context (usually HWUI RenderThread). 285 */ 286 void takeConsumerOwnership(); 287 288 /** 289 * releaseConsumerOwnership detaches a SurfaceTexture from a consumer 290 * context (usually HWUI RenderThread). 291 */ 292 void releaseConsumerOwnership(); 293 294 /** 295 * Interface for SurfaceTexture callback(s). 296 */ 297 struct SurfaceTextureListener : public RefBase { 298 virtual void onFrameAvailable(const BufferItem& item) = 0; 299 virtual void onSetFrameRate(float frameRate, int8_t compatibility, 300 int8_t changeFrameRateStrategy) = 0; 301 }; 302 303 /** 304 * setSurfaceTextureListener registers a SurfaceTextureListener. 305 */ 306 void setSurfaceTextureListener(const sp<SurfaceTextureListener>&); 307 308 protected: 309 /** 310 * abandonLocked overrides the ConsumerBase method to clear 311 * mCurrentTextureImage in addition to the ConsumerBase behavior. 312 */ 313 virtual void abandonLocked(); 314 315 /** 316 * dumpLocked overrides the ConsumerBase method to dump SurfaceTexture- 317 * specific info in addition to the ConsumerBase behavior. 318 */ 319 virtual void dumpLocked(String8& result, const char* prefix) const override; 320 321 /** 322 * acquireBufferLocked overrides the ConsumerBase method to update the 323 * mEglSlots array in addition to the ConsumerBase behavior. 324 */ 325 virtual status_t acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, 326 uint64_t maxFrameNumber = 0) override; 327 328 /** 329 * releaseBufferLocked overrides the ConsumerBase method to update the 330 * mEglSlots array in addition to the ConsumerBase. 331 */ 332 virtual status_t releaseBufferLocked(int slot, const sp<GraphicBuffer> graphicBuffer, 333 EGLDisplay display, EGLSyncKHR eglFence) override; 334 335 /** 336 * freeBufferLocked frees up the given buffer slot. If the slot has been 337 * initialized this will release the reference to the GraphicBuffer in that 338 * slot and destroy the EGLImage in that slot. Otherwise it has no effect. 339 * 340 * This method must be called with mMutex locked. 341 */ 342 virtual void freeBufferLocked(int slotIndex); 343 344 /** 345 * computeCurrentTransformMatrixLocked computes the transform matrix for the 346 * current texture. It uses mCurrentTransform and the current GraphicBuffer 347 * to compute this matrix and stores it in mCurrentTransformMatrix. 348 * mCurrentTextureImage must not be NULL. 349 */ 350 void computeCurrentTransformMatrixLocked(); 351 352 /** 353 * onSetFrameRate Notifies the consumer of a setFrameRate call from the producer side. 354 */ 355 #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_SETFRAMERATE) 356 void onSetFrameRate(float frameRate, int8_t compatibility, 357 int8_t changeFrameRateStrategy) override; 358 #endif 359 360 /** 361 * The default consumer usage flags that SurfaceTexture always sets on its 362 * BufferQueue instance; these will be OR:d with any additional flags passed 363 * from the SurfaceTexture user. In particular, SurfaceTexture will always 364 * consume buffers as hardware textures. 365 */ 366 static const uint64_t DEFAULT_USAGE_FLAGS = GraphicBuffer::USAGE_HW_TEXTURE; 367 368 /** 369 * mCurrentCrop is the crop rectangle that applies to the current texture. 370 * It gets set each time updateTexImage is called. 371 */ 372 Rect mCurrentCrop; 373 374 /** 375 * mCurrentTransform is the transform identifier for the current texture. It 376 * gets set each time updateTexImage is called. 377 */ 378 uint32_t mCurrentTransform; 379 380 /** 381 * mCurrentScalingMode is the scaling mode for the current texture. It gets 382 * set each time updateTexImage is called. 383 */ 384 uint32_t mCurrentScalingMode; 385 386 /** 387 * mCurrentFence is the fence received from BufferQueue in updateTexImage. 388 */ 389 sp<Fence> mCurrentFence; 390 391 /** 392 * The FenceTime wrapper around mCurrentFence. 393 */ 394 std::shared_ptr<FenceTime> mCurrentFenceTime{FenceTime::NO_FENCE}; 395 396 /** 397 * mCurrentTransformMatrix is the transform matrix for the current texture. 398 * It gets computed by computeTransformMatrix each time updateTexImage is 399 * called. 400 */ 401 float mCurrentTransformMatrix[16]; 402 403 /** 404 * mCurrentTimestamp is the timestamp for the current texture. It 405 * gets set each time updateTexImage is called. 406 */ 407 int64_t mCurrentTimestamp; 408 409 /** 410 * mCurrentDataSpace is the dataspace for the current texture. It 411 * gets set each time updateTexImage is called. 412 */ 413 android_dataspace mCurrentDataSpace; 414 415 /** 416 * mCurrentFrameNumber is the frame counter for the current texture. 417 * It gets set each time updateTexImage is called. 418 */ 419 uint64_t mCurrentFrameNumber; 420 421 uint32_t mDefaultWidth, mDefaultHeight; 422 423 /** 424 * mFilteringEnabled indicates whether the transform matrix is computed for 425 * use with bilinear filtering. It defaults to true and is changed by 426 * setFilteringEnabled(). 427 */ 428 bool mFilteringEnabled; 429 430 /** 431 * mTexName is the name of the OpenGL texture to which streamed images will 432 * be bound when updateTexImage is called. It is set at construction time 433 * and can be changed with a call to attachToContext. 434 */ 435 uint32_t mTexName; 436 437 /** 438 * mUseFenceSync indicates whether creation of the EGL_KHR_fence_sync 439 * extension should be used to prevent buffers from being dequeued before 440 * it's safe for them to be written. It gets set at construction time and 441 * never changes. 442 */ 443 const bool mUseFenceSync; 444 445 /** 446 * mTexTarget is the GL texture target with which the GL texture object is 447 * associated. It is set in the constructor and never changed. It is 448 * almost always GL_TEXTURE_EXTERNAL_OES except for one use case in Android 449 * Browser. In that case it is set to GL_TEXTURE_2D to allow 450 * glCopyTexSubImage to read from the texture. This is a hack to work 451 * around a GL driver limitation on the number of FBO attachments, which the 452 * browser's tile cache exceeds. 453 */ 454 const uint32_t mTexTarget; 455 456 /** 457 * mCurrentTexture is the buffer slot index of the buffer that is currently 458 * bound to the OpenGL texture. It is initialized to INVALID_BUFFER_SLOT, 459 * indicating that no buffer slot is currently bound to the texture. Note, 460 * however, that a value of INVALID_BUFFER_SLOT does not necessarily mean 461 * that no buffer is bound to the texture. A call to setBufferCount will 462 * reset mCurrentTexture to INVALID_BUFFER_SLOT. 463 */ 464 int mCurrentTexture; 465 466 enum class OpMode { detached, attachedToConsumer, attachedToGL }; 467 /** 468 * mOpMode indicates whether the SurfaceTexture is currently attached to 469 * an OpenGL ES context or the consumer context. For legacy reasons, this 470 * is initialized to, "attachedToGL" indicating that the SurfaceTexture is 471 * considered to be attached to whatever GL context is current at the time 472 * of the first updateTexImage call. 473 * It is set to "detached" by detachFromContext, and then set to 474 * "attachedToGL" again by attachToContext. 475 * takeConsumerOwnership/releaseConsumerOwnership are used to attach/detach 476 * from a consumer context - usually HWUI RenderThread. 477 */ 478 OpMode mOpMode; 479 480 /** 481 * mEGLConsumer has SurfaceTexture logic used when attached to GL context. 482 */ 483 EGLConsumer mEGLConsumer; 484 485 /** 486 * mImageConsumer has SurfaceTexture logic used when attached to a consumer 487 * context (usually HWUI RenderThread). 488 */ 489 ImageConsumer mImageConsumer; 490 491 /** 492 * mSurfaceTextureListener holds the registered SurfaceTextureListener. 493 * Note that SurfaceTexture holds the lister with an sp<>, which means that the listener 494 * must only hold a wp<> to SurfaceTexture and not an sp<>. 495 */ 496 sp<SurfaceTextureListener> mSurfaceTextureListener; 497 498 friend class ImageConsumer; 499 friend class EGLConsumer; 500 501 private: 502 // Proxy listener to avoid having SurfaceTexture directly implement FrameAvailableListener as it 503 // is extending ConsumerBase which also implements FrameAvailableListener. 504 class FrameAvailableListenerProxy : public ConsumerBase::FrameAvailableListener { 505 public: FrameAvailableListenerProxy(const wp<SurfaceTextureListener> & listener)506 FrameAvailableListenerProxy(const wp<SurfaceTextureListener>& listener) 507 : mSurfaceTextureListener(listener) {} 508 509 private: 510 void onFrameAvailable(const BufferItem& item) override; 511 512 const wp<SurfaceTextureListener> mSurfaceTextureListener; 513 }; 514 sp<FrameAvailableListenerProxy> mFrameAvailableListenerProxy; 515 }; 516 517 // ---------------------------------------------------------------------------- 518 } // namespace android 519