1 /*
2 * Copyright (C) 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "HardwareBitmapUploader.h"
18
19 #include "hwui/Bitmap.h"
20 #include "renderthread/EglManager.h"
21 #include "renderthread/VulkanManager.h"
22 #include "thread/ThreadBase.h"
23 #include "utils/TimeUtils.h"
24
25 #include <EGL/eglext.h>
26 #include <GLES2/gl2.h>
27 #include <GLES2/gl2ext.h>
28 #include <GLES3/gl3.h>
29 #include <GrContext.h>
30 #include <SkCanvas.h>
31 #include <SkImage.h>
32 #include <utils/GLUtils.h>
33 #include <utils/Trace.h>
34 #include <utils/TraceUtils.h>
35 #include <thread>
36
37 namespace android::uirenderer {
38
39 class AHBUploader;
40 // This helper uploader classes allows us to upload using either EGL or Vulkan using the same
41 // interface.
42 static sp<AHBUploader> sUploader = nullptr;
43
44 struct FormatInfo {
45 PixelFormat pixelFormat;
46 GLint format, type;
47 VkFormat vkFormat;
48 bool isSupported = false;
49 bool valid = true;
50 };
51
52 class AHBUploader : public RefBase {
53 public:
~AHBUploader()54 virtual ~AHBUploader() {}
55
56 // Called to start creation of the Vulkan and EGL contexts on another thread before we actually
57 // need to do an upload.
initialize()58 void initialize() {
59 onInitialize();
60 }
61
destroy()62 void destroy() {
63 std::lock_guard _lock{mLock};
64 LOG_ALWAYS_FATAL_IF(mPendingUploads, "terminate called while uploads in progress");
65 if (mUploadThread) {
66 mUploadThread->requestExit();
67 mUploadThread->join();
68 mUploadThread = nullptr;
69 }
70 onDestroy();
71 }
72
uploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,sp<GraphicBuffer> graphicBuffer)73 bool uploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
74 sp<GraphicBuffer> graphicBuffer) {
75 ATRACE_CALL();
76 beginUpload();
77 bool result = onUploadHardwareBitmap(bitmap, format, graphicBuffer);
78 endUpload();
79 return result;
80 }
81
postIdleTimeoutCheck()82 void postIdleTimeoutCheck() {
83 mUploadThread->queue().postDelayed(5000_ms, [this](){ this->idleTimeoutCheck(); });
84 }
85
86 protected:
87 std::mutex mLock;
88 sp<ThreadBase> mUploadThread = nullptr;
89
90 private:
91 virtual void onInitialize() = 0;
92 virtual void onIdle() = 0;
93 virtual void onDestroy() = 0;
94
95 virtual bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
96 sp<GraphicBuffer> graphicBuffer) = 0;
97 virtual void onBeginUpload() = 0;
98
shouldTimeOutLocked()99 bool shouldTimeOutLocked() {
100 nsecs_t durationSince = systemTime() - mLastUpload;
101 return durationSince > 2000_ms;
102 }
103
idleTimeoutCheck()104 void idleTimeoutCheck() {
105 std::lock_guard _lock{mLock};
106 if (mPendingUploads == 0 && shouldTimeOutLocked()) {
107 onIdle();
108 } else {
109 this->postIdleTimeoutCheck();
110 }
111 }
112
beginUpload()113 void beginUpload() {
114 std::lock_guard _lock{mLock};
115 mPendingUploads++;
116
117 if (!mUploadThread) {
118 mUploadThread = new ThreadBase{};
119 }
120 if (!mUploadThread->isRunning()) {
121 mUploadThread->start("GrallocUploadThread");
122 }
123
124 onBeginUpload();
125 }
126
endUpload()127 void endUpload() {
128 std::lock_guard _lock{mLock};
129 mPendingUploads--;
130 mLastUpload = systemTime();
131 }
132
133 int mPendingUploads = 0;
134 nsecs_t mLastUpload = 0;
135 };
136
137 #define FENCE_TIMEOUT 2000000000
138
139 class EGLUploader : public AHBUploader {
140 private:
onInitialize()141 void onInitialize() override {}
onDestroy()142 void onDestroy() override {
143 mEglManager.destroy();
144 }
onIdle()145 void onIdle() override {
146 mEglManager.destroy();
147 }
148
onBeginUpload()149 void onBeginUpload() override {
150 if (!mEglManager.hasEglContext()) {
151 mUploadThread->queue().runSync([this]() {
152 this->mEglManager.initialize();
153 glPixelStorei(GL_UNPACK_ALIGNMENT, 1);
154 });
155
156 this->postIdleTimeoutCheck();
157 }
158 }
159
160
getUploadEglDisplay()161 EGLDisplay getUploadEglDisplay() {
162 std::lock_guard _lock{mLock};
163 LOG_ALWAYS_FATAL_IF(!mEglManager.hasEglContext(), "Forgot to begin an upload?");
164 return mEglManager.eglDisplay();
165 }
166
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,sp<GraphicBuffer> graphicBuffer)167 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
168 sp<GraphicBuffer> graphicBuffer) override {
169 ATRACE_CALL();
170
171 EGLDisplay display = getUploadEglDisplay();
172
173 LOG_ALWAYS_FATAL_IF(display == EGL_NO_DISPLAY, "Failed to get EGL_DEFAULT_DISPLAY! err=%s",
174 uirenderer::renderthread::EglManager::eglErrorString());
175 // We use an EGLImage to access the content of the GraphicBuffer
176 // The EGL image is later bound to a 2D texture
177 EGLClientBuffer clientBuffer = (EGLClientBuffer)graphicBuffer->getNativeBuffer();
178 AutoEglImage autoImage(display, clientBuffer);
179 if (autoImage.image == EGL_NO_IMAGE_KHR) {
180 ALOGW("Could not create EGL image, err =%s",
181 uirenderer::renderthread::EglManager::eglErrorString());
182 return false;
183 }
184
185 {
186 ATRACE_FORMAT("CPU -> gralloc transfer (%dx%d)", bitmap.width(), bitmap.height());
187 EGLSyncKHR fence = mUploadThread->queue().runSync([&]() -> EGLSyncKHR {
188 AutoSkiaGlTexture glTexture;
189 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, autoImage.image);
190 if (GLUtils::dumpGLErrors()) {
191 return EGL_NO_SYNC_KHR;
192 }
193
194 // glTexSubImage2D is synchronous in sense that it memcpy() from pointer that we
195 // provide.
196 // But asynchronous in sense that driver may upload texture onto hardware buffer
197 // when we first use it in drawing
198 glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, bitmap.width(), bitmap.height(),
199 format.format, format.type, bitmap.getPixels());
200 if (GLUtils::dumpGLErrors()) {
201 return EGL_NO_SYNC_KHR;
202 }
203
204 EGLSyncKHR uploadFence =
205 eglCreateSyncKHR(eglGetCurrentDisplay(), EGL_SYNC_FENCE_KHR, NULL);
206 if (uploadFence == EGL_NO_SYNC_KHR) {
207 ALOGW("Could not create sync fence %#x", eglGetError());
208 };
209 glFlush();
210 GLUtils::dumpGLErrors();
211 return uploadFence;
212 });
213
214 if (fence == EGL_NO_SYNC_KHR) {
215 return false;
216 }
217 EGLint waitStatus = eglClientWaitSyncKHR(display, fence, 0, FENCE_TIMEOUT);
218 ALOGE_IF(waitStatus != EGL_CONDITION_SATISFIED_KHR,
219 "Failed to wait for the fence %#x", eglGetError());
220
221 eglDestroySyncKHR(display, fence);
222 }
223 return true;
224 }
225
226 renderthread::EglManager mEglManager;
227 };
228
229 class VkUploader : public AHBUploader {
230 private:
onInitialize()231 void onInitialize() override {
232 std::lock_guard _lock{mLock};
233 if (!mUploadThread) {
234 mUploadThread = new ThreadBase{};
235 }
236 if (!mUploadThread->isRunning()) {
237 mUploadThread->start("GrallocUploadThread");
238 }
239
240 mUploadThread->queue().post([this]() {
241 std::lock_guard _lock{mVkLock};
242 if (!mVulkanManager.hasVkContext()) {
243 mVulkanManager.initialize();
244 }
245 });
246 }
onDestroy()247 void onDestroy() override {
248 mGrContext.reset();
249 mVulkanManager.destroy();
250 }
onIdle()251 void onIdle() override {
252 mGrContext.reset();
253 }
254
onBeginUpload()255 void onBeginUpload() override {
256 {
257 std::lock_guard _lock{mVkLock};
258 if (!mVulkanManager.hasVkContext()) {
259 LOG_ALWAYS_FATAL_IF(mGrContext,
260 "GrContext exists with no VulkanManager for vulkan uploads");
261 mUploadThread->queue().runSync([this]() {
262 mVulkanManager.initialize();
263 });
264 }
265 }
266 if (!mGrContext) {
267 GrContextOptions options;
268 mGrContext = mVulkanManager.createContext(options);
269 LOG_ALWAYS_FATAL_IF(!mGrContext, "failed to create GrContext for vulkan uploads");
270 this->postIdleTimeoutCheck();
271 }
272 }
273
onUploadHardwareBitmap(const SkBitmap & bitmap,const FormatInfo & format,sp<GraphicBuffer> graphicBuffer)274 bool onUploadHardwareBitmap(const SkBitmap& bitmap, const FormatInfo& format,
275 sp<GraphicBuffer> graphicBuffer) override {
276 ATRACE_CALL();
277
278 std::lock_guard _lock{mLock};
279
280 sk_sp<SkImage> image = SkImage::MakeFromAHardwareBufferWithData(mGrContext.get(),
281 bitmap.pixmap(), reinterpret_cast<AHardwareBuffer*>(graphicBuffer.get()));
282 return (image.get() != nullptr);
283 }
284
285 sk_sp<GrContext> mGrContext;
286 renderthread::VulkanManager mVulkanManager;
287 std::mutex mVkLock;
288 };
289
hasFP16Support()290 bool HardwareBitmapUploader::hasFP16Support() {
291 static std::once_flag sOnce;
292 static bool hasFP16Support = false;
293
294 // Gralloc shouldn't let us create a USAGE_HW_TEXTURE if GLES is unable to consume it, so
295 // we don't need to double-check the GLES version/extension.
296 std::call_once(sOnce, []() {
297 sp<GraphicBuffer> buffer = new GraphicBuffer(1, 1, PIXEL_FORMAT_RGBA_FP16,
298 GraphicBuffer::USAGE_HW_TEXTURE |
299 GraphicBuffer::USAGE_SW_WRITE_NEVER |
300 GraphicBuffer::USAGE_SW_READ_NEVER,
301 "tempFp16Buffer");
302 status_t error = buffer->initCheck();
303 hasFP16Support = !error;
304 });
305
306 return hasFP16Support;
307 }
308
determineFormat(const SkBitmap & skBitmap,bool usingGL)309 static FormatInfo determineFormat(const SkBitmap& skBitmap, bool usingGL) {
310 FormatInfo formatInfo;
311 switch (skBitmap.info().colorType()) {
312 case kRGBA_8888_SkColorType:
313 formatInfo.isSupported = true;
314 [[fallthrough]];
315 // ARGB_4444 is upconverted to RGBA_8888
316 case kARGB_4444_SkColorType:
317 formatInfo.pixelFormat = PIXEL_FORMAT_RGBA_8888;
318 formatInfo.format = GL_RGBA;
319 formatInfo.type = GL_UNSIGNED_BYTE;
320 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
321 break;
322 case kRGBA_F16_SkColorType:
323 formatInfo.isSupported = HardwareBitmapUploader::hasFP16Support();
324 if (formatInfo.isSupported) {
325 formatInfo.type = GL_HALF_FLOAT;
326 formatInfo.pixelFormat = PIXEL_FORMAT_RGBA_FP16;
327 formatInfo.vkFormat = VK_FORMAT_R16G16B16A16_SFLOAT;
328 } else {
329 formatInfo.type = GL_UNSIGNED_BYTE;
330 formatInfo.pixelFormat = PIXEL_FORMAT_RGBA_8888;
331 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
332 }
333 formatInfo.format = GL_RGBA;
334 break;
335 case kRGB_565_SkColorType:
336 formatInfo.isSupported = true;
337 formatInfo.pixelFormat = PIXEL_FORMAT_RGB_565;
338 formatInfo.format = GL_RGB;
339 formatInfo.type = GL_UNSIGNED_SHORT_5_6_5;
340 formatInfo.vkFormat = VK_FORMAT_R5G6B5_UNORM_PACK16;
341 break;
342 case kGray_8_SkColorType:
343 formatInfo.isSupported = usingGL;
344 formatInfo.pixelFormat = PIXEL_FORMAT_RGBA_8888;
345 formatInfo.format = GL_LUMINANCE;
346 formatInfo.type = GL_UNSIGNED_BYTE;
347 formatInfo.vkFormat = VK_FORMAT_R8G8B8A8_UNORM;
348 break;
349 default:
350 ALOGW("unable to create hardware bitmap of colortype: %d", skBitmap.info().colorType());
351 formatInfo.valid = false;
352 }
353 return formatInfo;
354 }
355
makeHwCompatible(const FormatInfo & format,const SkBitmap & source)356 static SkBitmap makeHwCompatible(const FormatInfo& format, const SkBitmap& source) {
357 if (format.isSupported) {
358 return source;
359 } else {
360 SkBitmap bitmap;
361 const SkImageInfo& info = source.info();
362 bitmap.allocPixels(info.makeColorType(kN32_SkColorType));
363
364 SkCanvas canvas(bitmap);
365 canvas.drawColor(0);
366 canvas.drawBitmap(source, 0.0f, 0.0f, nullptr);
367
368 return bitmap;
369 }
370 }
371
372
createUploader(bool usingGL)373 static void createUploader(bool usingGL) {
374 static std::mutex lock;
375 std::lock_guard _lock{lock};
376 if (!sUploader.get()) {
377 if (usingGL) {
378 sUploader = new EGLUploader();
379 } else {
380 sUploader = new VkUploader();
381 }
382 }
383 }
384
allocateHardwareBitmap(const SkBitmap & sourceBitmap)385 sk_sp<Bitmap> HardwareBitmapUploader::allocateHardwareBitmap(const SkBitmap& sourceBitmap) {
386 ATRACE_CALL();
387
388 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
389 uirenderer::RenderPipelineType::SkiaGL;
390
391 FormatInfo format = determineFormat(sourceBitmap, usingGL);
392 if (!format.valid) {
393 return nullptr;
394 }
395
396 SkBitmap bitmap = makeHwCompatible(format, sourceBitmap);
397 sp<GraphicBuffer> buffer = new GraphicBuffer(
398 static_cast<uint32_t>(bitmap.width()), static_cast<uint32_t>(bitmap.height()),
399 format.pixelFormat,
400 GraphicBuffer::USAGE_HW_TEXTURE | GraphicBuffer::USAGE_SW_WRITE_NEVER |
401 GraphicBuffer::USAGE_SW_READ_NEVER,
402 std::string("Bitmap::allocateHardwareBitmap pid [") + std::to_string(getpid()) +
403 "]");
404
405 status_t error = buffer->initCheck();
406 if (error < 0) {
407 ALOGW("createGraphicBuffer() failed in GraphicBuffer.create()");
408 return nullptr;
409 }
410
411 createUploader(usingGL);
412
413 if (!sUploader->uploadHardwareBitmap(bitmap, format, buffer)) {
414 return nullptr;
415 }
416 return Bitmap::createFrom(buffer->toAHardwareBuffer(), bitmap.colorType(),
417 bitmap.refColorSpace(), bitmap.alphaType(),
418 Bitmap::computePalette(bitmap));
419 }
420
initialize()421 void HardwareBitmapUploader::initialize() {
422 bool usingGL = uirenderer::Properties::getRenderPipelineType() ==
423 uirenderer::RenderPipelineType::SkiaGL;
424 createUploader(usingGL);
425 sUploader->initialize();
426 }
427
terminate()428 void HardwareBitmapUploader::terminate() {
429 if (sUploader) {
430 sUploader->destroy();
431 }
432 }
433
434 } // namespace android::uirenderer
435