/* * Copyright 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include #include #include namespace android { // Macros for including the SurfaceTexture name in log messages #define SFT_LOGV(x, ...) ALOGV("[%s] " x, mName.c_str(), ##__VA_ARGS__) #define SFT_LOGD(x, ...) ALOGD("[%s] " x, mName.c_str(), ##__VA_ARGS__) #define SFT_LOGW(x, ...) ALOGW("[%s] " x, mName.c_str(), ##__VA_ARGS__) #define SFT_LOGE(x, ...) ALOGE("[%s] " x, mName.c_str(), ##__VA_ARGS__) static const mat4 mtxIdentity; SurfaceTexture::SurfaceTexture(const sp& bq, uint32_t tex, uint32_t texTarget, bool useFenceSync, bool isControlledByApp) : ConsumerBase(bq, isControlledByApp), mCurrentCrop(Rect::EMPTY_RECT), mCurrentTransform(0), mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), mCurrentFence(Fence::NO_FENCE), mCurrentTimestamp(0), mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), mCurrentFrameNumber(0), mDefaultWidth(1), mDefaultHeight(1), mFilteringEnabled(true), mTexName(tex), mUseFenceSync(useFenceSync), mTexTarget(texTarget), mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), mOpMode(OpMode::attachedToGL) { SFT_LOGV("SurfaceTexture"); memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); } SurfaceTexture::SurfaceTexture(const sp& bq, uint32_t texTarget, bool useFenceSync, bool isControlledByApp) : ConsumerBase(bq, isControlledByApp), mCurrentCrop(Rect::EMPTY_RECT), mCurrentTransform(0), mCurrentScalingMode(NATIVE_WINDOW_SCALING_MODE_FREEZE), mCurrentFence(Fence::NO_FENCE), mCurrentTimestamp(0), mCurrentDataSpace(HAL_DATASPACE_UNKNOWN), mCurrentFrameNumber(0), mDefaultWidth(1), mDefaultHeight(1), mFilteringEnabled(true), mTexName(0), mUseFenceSync(useFenceSync), mTexTarget(texTarget), mCurrentTexture(BufferQueue::INVALID_BUFFER_SLOT), mOpMode(OpMode::detached) { SFT_LOGV("SurfaceTexture"); memcpy(mCurrentTransformMatrix, mtxIdentity.asArray(), sizeof(mCurrentTransformMatrix)); mConsumer->setConsumerUsageBits(DEFAULT_USAGE_FLAGS); } status_t SurfaceTexture::setDefaultBufferSize(uint32_t w, uint32_t h) { Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("setDefaultBufferSize: SurfaceTexture is abandoned!"); return NO_INIT; } mDefaultWidth = w; mDefaultHeight = h; return mConsumer->setDefaultBufferSize(w, h); } status_t SurfaceTexture::updateTexImage() { ATRACE_CALL(); SFT_LOGV("updateTexImage"); Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("updateTexImage: SurfaceTexture is abandoned!"); return NO_INIT; } return mEGLConsumer.updateTexImage(*this); } status_t SurfaceTexture::releaseTexImage() { // releaseTexImage can be invoked even when not attached to a GL context. ATRACE_CALL(); SFT_LOGV("releaseTexImage"); Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("releaseTexImage: SurfaceTexture is abandoned!"); return NO_INIT; } return mEGLConsumer.releaseTexImage(*this); } status_t SurfaceTexture::acquireBufferLocked(BufferItem* item, nsecs_t presentWhen, uint64_t maxFrameNumber) { status_t err = ConsumerBase::acquireBufferLocked(item, presentWhen, maxFrameNumber); if (err != NO_ERROR) { return err; } switch (mOpMode) { case OpMode::attachedToConsumer: break; case OpMode::attachedToGL: mEGLConsumer.onAcquireBufferLocked(item, *this); break; case OpMode::detached: break; } return NO_ERROR; } status_t SurfaceTexture::releaseBufferLocked(int buf, sp graphicBuffer, EGLDisplay display, EGLSyncKHR eglFence) { // release the buffer if it hasn't already been discarded by the // BufferQueue. This can happen, for example, when the producer of this // buffer has reallocated the original buffer slot after this buffer // was acquired. status_t err = ConsumerBase::releaseBufferLocked(buf, graphicBuffer, display, eglFence); // We could be releasing an EGL/Vulkan buffer, even if not currently // attached to a GL context. mImageConsumer.onReleaseBufferLocked(buf); mEGLConsumer.onReleaseBufferLocked(buf); return err; } status_t SurfaceTexture::detachFromContext() { ATRACE_CALL(); SFT_LOGV("detachFromContext"); Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("detachFromContext: abandoned SurfaceTexture"); return NO_INIT; } if (mOpMode != OpMode::attachedToGL) { SFT_LOGE("detachFromContext: SurfaceTexture is not attached to a GL context"); return INVALID_OPERATION; } status_t err = mEGLConsumer.detachFromContext(*this); if (err == OK) { mOpMode = OpMode::detached; } return err; } status_t SurfaceTexture::attachToContext(uint32_t tex) { ATRACE_CALL(); SFT_LOGV("attachToContext"); Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("attachToContext: abandoned SurfaceTexture"); return NO_INIT; } if (mOpMode != OpMode::detached) { SFT_LOGE("attachToContext: SurfaceTexture is already attached to a " "context"); return INVALID_OPERATION; } return mEGLConsumer.attachToContext(tex, *this); } void SurfaceTexture::takeConsumerOwnership() { ATRACE_CALL(); Mutex::Autolock _l(mMutex); if (mAbandoned) { SFT_LOGE("attachToView: abandoned SurfaceTexture"); return; } if (mOpMode == OpMode::detached) { mOpMode = OpMode::attachedToConsumer; if (mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) { // release possible EGLConsumer texture cache mEGLConsumer.onFreeBufferLocked(mCurrentTexture); mEGLConsumer.onAbandonLocked(); } } else { SFT_LOGE("attachToView: already attached"); } } void SurfaceTexture::releaseConsumerOwnership() { ATRACE_CALL(); Mutex::Autolock _l(mMutex); if (mAbandoned) { SFT_LOGE("detachFromView: abandoned SurfaceTexture"); return; } if (mOpMode == OpMode::attachedToConsumer) { mOpMode = OpMode::detached; } else { SFT_LOGE("detachFromView: not attached to View"); } } uint32_t SurfaceTexture::getCurrentTextureTarget() const { return mTexTarget; } void SurfaceTexture::getTransformMatrix(float mtx[16]) { Mutex::Autolock lock(mMutex); memcpy(mtx, mCurrentTransformMatrix, sizeof(mCurrentTransformMatrix)); } void SurfaceTexture::setFilteringEnabled(bool enabled) { Mutex::Autolock lock(mMutex); if (mAbandoned) { SFT_LOGE("setFilteringEnabled: SurfaceTexture is abandoned!"); return; } bool needsRecompute = mFilteringEnabled != enabled; mFilteringEnabled = enabled; if (needsRecompute && mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT) { SFT_LOGD("setFilteringEnabled called with no current item"); } if (needsRecompute && mCurrentTexture != BufferQueue::INVALID_BUFFER_SLOT) { computeCurrentTransformMatrixLocked(); } } void SurfaceTexture::computeCurrentTransformMatrixLocked() { SFT_LOGV("computeCurrentTransformMatrixLocked"); sp buf = (mCurrentTexture == BufferQueue::INVALID_BUFFER_SLOT) ? nullptr : mSlots[mCurrentTexture].mGraphicBuffer; if (buf == nullptr) { SFT_LOGD("computeCurrentTransformMatrixLocked: no current item"); } computeTransformMatrix(mCurrentTransformMatrix, buf, mCurrentCrop, mCurrentTransform, mFilteringEnabled); } void SurfaceTexture::computeTransformMatrix(float outTransform[16], const sp& buf, const Rect& cropRect, uint32_t transform, bool filtering) { // Transform matrices static const mat4 mtxFlipH(-1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1); static const mat4 mtxFlipV(1, 0, 0, 0, 0, -1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1); static const mat4 mtxRot90(0, 1, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1); mat4 xform; if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_H) { xform *= mtxFlipH; } if (transform & NATIVE_WINDOW_TRANSFORM_FLIP_V) { xform *= mtxFlipV; } if (transform & NATIVE_WINDOW_TRANSFORM_ROT_90) { xform *= mtxRot90; } if (!cropRect.isEmpty() && buf.get()) { float tx = 0.0f, ty = 0.0f, sx = 1.0f, sy = 1.0f; float bufferWidth = buf->getWidth(); float bufferHeight = buf->getHeight(); float shrinkAmount = 0.0f; if (filtering) { // In order to prevent bilinear sampling beyond the edge of the // crop rectangle we may need to shrink it by 2 texels in each // dimension. Normally this would just need to take 1/2 a texel // off each end, but because the chroma channels of YUV420 images // are subsampled we may need to shrink the crop region by a whole // texel on each side. switch (buf->getPixelFormat()) { case PIXEL_FORMAT_RGBA_8888: case PIXEL_FORMAT_RGBX_8888: case PIXEL_FORMAT_RGBA_FP16: case PIXEL_FORMAT_RGBA_1010102: case PIXEL_FORMAT_RGB_888: case PIXEL_FORMAT_RGB_565: case PIXEL_FORMAT_BGRA_8888: // We know there's no subsampling of any channels, so we // only need to shrink by a half a pixel. shrinkAmount = 0.5; break; default: // If we don't recognize the format, we must assume the // worst case (that we care about), which is YUV420. shrinkAmount = 1.0; break; } } // Only shrink the dimensions that are not the size of the buffer. if (cropRect.width() < bufferWidth) { tx = (float(cropRect.left) + shrinkAmount) / bufferWidth; sx = (float(cropRect.width()) - (2.0f * shrinkAmount)) / bufferWidth; } if (cropRect.height() < bufferHeight) { ty = (float(bufferHeight - cropRect.bottom) + shrinkAmount) / bufferHeight; sy = (float(cropRect.height()) - (2.0f * shrinkAmount)) / bufferHeight; } mat4 crop(sx, 0, 0, 0, 0, sy, 0, 0, 0, 0, 1, 0, tx, ty, 0, 1); xform = crop * xform; } // SurfaceFlinger expects the top of its window textures to be at a Y // coordinate of 0, so SurfaceTexture must behave the same way. We don't // want to expose this to applications, however, so we must add an // additional vertical flip to the transform after all the other transforms. xform = mtxFlipV * xform; memcpy(outTransform, xform.asArray(), sizeof(xform)); } Rect SurfaceTexture::scaleDownCrop(const Rect& crop, uint32_t bufferWidth, uint32_t bufferHeight) { Rect outCrop = crop; uint32_t newWidth = static_cast(crop.width()); uint32_t newHeight = static_cast(crop.height()); if (newWidth * bufferHeight > newHeight * bufferWidth) { newWidth = newHeight * bufferWidth / bufferHeight; ALOGV("too wide: newWidth = %d", newWidth); } else if (newWidth * bufferHeight < newHeight * bufferWidth) { newHeight = newWidth * bufferHeight / bufferWidth; ALOGV("too tall: newHeight = %d", newHeight); } uint32_t currentWidth = static_cast(crop.width()); uint32_t currentHeight = static_cast(crop.height()); // The crop is too wide if (newWidth < currentWidth) { uint32_t dw = currentWidth - newWidth; auto halfdw = dw / 2; outCrop.left += halfdw; // Not halfdw because it would subtract 1 too few when dw is odd outCrop.right -= (dw - halfdw); // The crop is too tall } else if (newHeight < currentHeight) { uint32_t dh = currentHeight - newHeight; auto halfdh = dh / 2; outCrop.top += halfdh; // Not halfdh because it would subtract 1 too few when dh is odd outCrop.bottom -= (dh - halfdh); } ALOGV("getCurrentCrop final crop [%d,%d,%d,%d]", outCrop.left, outCrop.top, outCrop.right, outCrop.bottom); return outCrop; } nsecs_t SurfaceTexture::getTimestamp() { SFT_LOGV("getTimestamp"); Mutex::Autolock lock(mMutex); return mCurrentTimestamp; } android_dataspace SurfaceTexture::getCurrentDataSpace() { SFT_LOGV("getCurrentDataSpace"); Mutex::Autolock lock(mMutex); return mCurrentDataSpace; } uint64_t SurfaceTexture::getFrameNumber() { SFT_LOGV("getFrameNumber"); Mutex::Autolock lock(mMutex); return mCurrentFrameNumber; } Rect SurfaceTexture::getCurrentCrop() const { Mutex::Autolock lock(mMutex); return (mCurrentScalingMode == NATIVE_WINDOW_SCALING_MODE_SCALE_CROP) ? scaleDownCrop(mCurrentCrop, mDefaultWidth, mDefaultHeight) : mCurrentCrop; } uint32_t SurfaceTexture::getCurrentTransform() const { Mutex::Autolock lock(mMutex); return mCurrentTransform; } uint32_t SurfaceTexture::getCurrentScalingMode() const { Mutex::Autolock lock(mMutex); return mCurrentScalingMode; } sp SurfaceTexture::getCurrentFence() const { Mutex::Autolock lock(mMutex); return mCurrentFence; } std::shared_ptr SurfaceTexture::getCurrentFenceTime() const { Mutex::Autolock lock(mMutex); return mCurrentFenceTime; } void SurfaceTexture::freeBufferLocked(int slotIndex) { SFT_LOGV("freeBufferLocked: slotIndex=%d", slotIndex); if (slotIndex == mCurrentTexture) { mCurrentTexture = BufferQueue::INVALID_BUFFER_SLOT; } // The slotIndex buffer could have EGL cache, but there is no way to tell // for sure. Buffers can be freed after SurfaceTexture has detached from GL // context or View. mEGLConsumer.onFreeBufferLocked(slotIndex); ConsumerBase::freeBufferLocked(slotIndex); } void SurfaceTexture::abandonLocked() { SFT_LOGV("abandonLocked"); mEGLConsumer.onAbandonLocked(); ConsumerBase::abandonLocked(); } status_t SurfaceTexture::setConsumerUsageBits(uint64_t usage) { return ConsumerBase::setConsumerUsageBits(usage | DEFAULT_USAGE_FLAGS); } void SurfaceTexture::dumpLocked(String8& result, const char* prefix) const { result.appendFormat("%smTexName=%d mCurrentTexture=%d\n" "%smCurrentCrop=[%d,%d,%d,%d] mCurrentTransform=%#x\n", prefix, mTexName, mCurrentTexture, prefix, mCurrentCrop.left, mCurrentCrop.top, mCurrentCrop.right, mCurrentCrop.bottom, mCurrentTransform); ConsumerBase::dumpLocked(result, prefix); } sp SurfaceTexture::dequeueBuffer(int* outSlotid, android_dataspace* outDataspace, HdrMetadata* outHdrMetadata, float* outTransformMatrix, uint32_t* outTransform, bool* outQueueEmpty, SurfaceTexture_createReleaseFence createFence, SurfaceTexture_fenceWait fenceWait, void* fencePassThroughHandle, ARect* currentCrop) { Mutex::Autolock _l(mMutex); sp buffer; if (mAbandoned) { SFT_LOGE("dequeueImage: SurfaceTexture is abandoned!"); return buffer; } if (mOpMode != OpMode::attachedToConsumer) { SFT_LOGE("dequeueImage: SurfaceTexture is not attached to a View"); return buffer; } buffer = mImageConsumer.dequeueBuffer(outSlotid, outDataspace, outHdrMetadata, outQueueEmpty, *this, createFence, fenceWait, fencePassThroughHandle); memcpy(outTransformMatrix, mCurrentTransformMatrix, sizeof(mCurrentTransformMatrix)); *outTransform = mCurrentTransform; *currentCrop = mCurrentCrop; return buffer; } void SurfaceTexture::setSurfaceTextureListener( const sp& listener) { SFT_LOGV("setSurfaceTextureListener"); Mutex::Autolock _l(mMutex); mSurfaceTextureListener = listener; if (mSurfaceTextureListener != nullptr) { mFrameAvailableListenerProxy = sp::make(mSurfaceTextureListener); setFrameAvailableListener(mFrameAvailableListenerProxy); } else { mFrameAvailableListenerProxy.clear(); } } void SurfaceTexture::FrameAvailableListenerProxy::onFrameAvailable(const BufferItem& item) { const auto listener = mSurfaceTextureListener.promote(); if (listener) { listener->onFrameAvailable(item); } } #if COM_ANDROID_GRAPHICS_LIBGUI_FLAGS(BQ_SETFRAMERATE) void SurfaceTexture::onSetFrameRate(float frameRate, int8_t compatibility, int8_t changeFrameRateStrategy) { SFT_LOGV("onSetFrameRate: %.2f", frameRate); auto listener = [&] { Mutex::Autolock _l(mMutex); return mSurfaceTextureListener; }(); if (listener) { listener->onSetFrameRate(frameRate, compatibility, changeFrameRateStrategy); } } #endif } // namespace android