1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #include "SurroundViewServiceCallback.h" 18 19 #include <android-base/logging.h> 20 #include <math/mat4.h> 21 #include <ui/GraphicBuffer.h> 22 #include <utils/Log.h> 23 24 #include "shader_simpleTex.h" 25 #include "shader.h" 26 27 using android::GraphicBuffer; 28 using android::hardware::automotive::evs::V1_0::DisplayState; 29 using android::hardware::automotive::evs::V1_0::EvsResult; 30 using android::hardware::Return; 31 using android::sp; 32 using std::string; 33 34 EGLDisplay SurroundViewServiceCallback::sGLDisplay; 35 GLuint SurroundViewServiceCallback::sFrameBuffer; 36 GLuint SurroundViewServiceCallback::sColorBuffer; 37 GLuint SurroundViewServiceCallback::sDepthBuffer; 38 GLuint SurroundViewServiceCallback::sTextureId; 39 EGLImageKHR SurroundViewServiceCallback::sKHRimage; 40 41 const char* SurroundViewServiceCallback::getEGLError(void) { 42 switch (eglGetError()) { 43 case EGL_SUCCESS: 44 return "EGL_SUCCESS"; 45 case EGL_NOT_INITIALIZED: 46 return "EGL_NOT_INITIALIZED"; 47 case EGL_BAD_ACCESS: 48 return "EGL_BAD_ACCESS"; 49 case EGL_BAD_ALLOC: 50 return "EGL_BAD_ALLOC"; 51 case EGL_BAD_ATTRIBUTE: 52 return "EGL_BAD_ATTRIBUTE"; 53 case EGL_BAD_CONTEXT: 54 return "EGL_BAD_CONTEXT"; 55 case EGL_BAD_CONFIG: 56 return "EGL_BAD_CONFIG"; 57 case EGL_BAD_CURRENT_SURFACE: 58 return "EGL_BAD_CURRENT_SURFACE"; 59 case EGL_BAD_DISPLAY: 60 return "EGL_BAD_DISPLAY"; 61 case EGL_BAD_SURFACE: 62 return "EGL_BAD_SURFACE"; 63 case EGL_BAD_MATCH: 64 return "EGL_BAD_MATCH"; 65 case EGL_BAD_PARAMETER: 66 return "EGL_BAD_PARAMETER"; 67 case EGL_BAD_NATIVE_PIXMAP: 68 return "EGL_BAD_NATIVE_PIXMAP"; 69 case EGL_BAD_NATIVE_WINDOW: 70 return "EGL_BAD_NATIVE_WINDOW"; 71 case EGL_CONTEXT_LOST: 72 return "EGL_CONTEXT_LOST"; 73 default: 74 return "Unknown error"; 75 } 76 } 77 78 const string SurroundViewServiceCallback::getGLFramebufferError(void) { 79 switch (glCheckFramebufferStatus(GL_FRAMEBUFFER)) { 80 case GL_FRAMEBUFFER_COMPLETE: 81 return "GL_FRAMEBUFFER_COMPLETE"; 82 case GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT: 83 return "GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT"; 84 case GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT: 85 return "GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT"; 86 case GL_FRAMEBUFFER_UNSUPPORTED: 87 return "GL_FRAMEBUFFER_UNSUPPORTED"; 88 case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS: 89 return "GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS"; 90 default: 91 return std::to_string(glCheckFramebufferStatus(GL_FRAMEBUFFER)); 92 } 93 } 94 95 bool SurroundViewServiceCallback::prepareGL() { 96 LOG(DEBUG) << __FUNCTION__; 97 98 // Just trivially return success if we're already prepared 99 if (sGLDisplay != EGL_NO_DISPLAY) { 100 return true; 101 } 102 103 // Hardcoded to RGBx output display 104 const EGLint config_attribs[] = { 105 // Tag Value 106 EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, 107 EGL_RED_SIZE, 8, 108 EGL_GREEN_SIZE, 8, 109 EGL_BLUE_SIZE, 8, 110 EGL_NONE 111 }; 112 113 // Select OpenGL ES v 3 114 const EGLint context_attribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE}; 115 116 // Set up our OpenGL ES context associated with the default display 117 // (though we won't be visible) 118 EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY); 119 if (display == EGL_NO_DISPLAY) { 120 LOG(ERROR) << "Failed to get egl display"; 121 return false; 122 } 123 124 EGLint major = 0; 125 EGLint minor = 0; 126 if (!eglInitialize(display, &major, &minor)) { 127 LOG(ERROR) << "Failed to initialize EGL: " 128 << getEGLError(); 129 return false; 130 } else { 131 LOG(INFO) << "Initialized EGL at " 132 << major 133 << "." 134 << minor; 135 } 136 137 // Select the configuration that "best" matches our desired characteristics 138 EGLConfig egl_config; 139 EGLint num_configs; 140 if (!eglChooseConfig(display, config_attribs, &egl_config, 1, 141 &num_configs)) { 142 LOG(ERROR) << "eglChooseConfig() failed with error: " 143 << getEGLError(); 144 return false; 145 } 146 147 // Create a dummy pbuffer so we have a surface to bind -- we never intend 148 // to draw to this because attachRenderTarget will be called first. 149 EGLint surface_attribs[] = { EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE }; 150 EGLSurface sDummySurface = eglCreatePbufferSurface(display, egl_config, 151 surface_attribs); 152 if (sDummySurface == EGL_NO_SURFACE) { 153 LOG(ERROR) << "Failed to create OpenGL ES Dummy surface: " 154 << getEGLError(); 155 return false; 156 } else { 157 LOG(INFO) << "Dummy surface looks good! :)"; 158 } 159 160 // 161 // Create the EGL context 162 // 163 EGLContext context = eglCreateContext(display, egl_config, 164 EGL_NO_CONTEXT, context_attribs); 165 if (context == EGL_NO_CONTEXT) { 166 LOG(ERROR) << "Failed to create OpenGL ES Context: " 167 << getEGLError(); 168 return false; 169 } 170 171 // Activate our render target for drawing 172 if (!eglMakeCurrent(display, sDummySurface, sDummySurface, context)) { 173 LOG(ERROR) << "Failed to make the OpenGL ES Context current: " 174 << getEGLError(); 175 return false; 176 } else { 177 LOG(INFO) << "We made our context current! :)"; 178 } 179 180 // Report the extensions available on this implementation 181 const char* gl_extensions = (const char*) glGetString(GL_EXTENSIONS); 182 LOG(INFO) << "GL EXTENSIONS:\n " 183 << gl_extensions; 184 185 // Reserve handles for the color and depth targets we'll be setting up 186 glGenRenderbuffers(1, &sColorBuffer); 187 glGenRenderbuffers(1, &sDepthBuffer); 188 189 // Set up the frame buffer object we can modify and use for off screen 190 // rendering 191 glGenFramebuffers(1, &sFrameBuffer); 192 glBindFramebuffer(GL_FRAMEBUFFER, sFrameBuffer); 193 194 LOG(INFO) << "FrameBuffer is bound to " 195 << sFrameBuffer; 196 197 // New (from TextWrapper) 198 glGenTextures(1, &sTextureId); 199 200 // Now that we're assured success, store object handles we constructed 201 sGLDisplay = display; 202 203 GLuint mShaderProgram = 0; 204 // Load our shader program if we don't have it already 205 if (!mShaderProgram) { 206 mShaderProgram = buildShaderProgram(kVtxShaderSimpleTexture, 207 kPixShaderSimpleTexture, 208 "simpleTexture"); 209 if (!mShaderProgram) { 210 LOG(ERROR) << "Error building shader program"; 211 return false; 212 } 213 } 214 215 // Select our screen space simple texture shader 216 glUseProgram(mShaderProgram); 217 218 // Set up the model to clip space transform (identity matrix if we're 219 // modeling in screen space) 220 GLint loc = glGetUniformLocation(mShaderProgram, "cameraMat"); 221 if (loc < 0) { 222 LOG(ERROR) << "Couldn't set shader parameter 'cameraMat'"; 223 } else { 224 const android::mat4 identityMatrix; 225 glUniformMatrix4fv(loc, 1, false, identityMatrix.asArray()); 226 } 227 228 GLint sampler = glGetUniformLocation(mShaderProgram, "tex"); 229 if (sampler < 0) { 230 LOG(ERROR) << "Couldn't set shader parameter 'tex'"; 231 } else { 232 // Tell the sampler we looked up from the shader to use texture slot 0 233 // as its source 234 glUniform1i(sampler, 0); 235 } 236 237 return true; 238 } 239 240 BufferDesc SurroundViewServiceCallback::convertBufferDesc( 241 const BufferDesc_1_0& src) { 242 BufferDesc dst = {}; 243 AHardwareBuffer_Desc* pDesc = 244 reinterpret_cast<AHardwareBuffer_Desc *>(&dst.buffer.description); 245 pDesc->width = src.width; 246 pDesc->height = src.height; 247 pDesc->layers = 1; 248 pDesc->format = src.format; 249 pDesc->usage = static_cast<uint64_t>(src.usage); 250 pDesc->stride = src.stride; 251 252 dst.buffer.nativeHandle = src.memHandle; 253 dst.pixelSize = src.pixelSize; 254 dst.bufferId = src.bufferId; 255 256 return dst; 257 } 258 259 bool SurroundViewServiceCallback::attachRenderTarget( 260 const BufferDesc& tgtBuffer) { 261 const AHardwareBuffer_Desc* pDesc = 262 reinterpret_cast<const AHardwareBuffer_Desc *>( 263 &tgtBuffer.buffer.description); 264 // Hardcoded to RGBx for now 265 if (pDesc->format != HAL_PIXEL_FORMAT_RGBA_8888) { 266 LOG(ERROR) << "Unsupported target buffer format"; 267 return false; 268 } 269 270 // create a GraphicBuffer from the existing handle 271 sp<GraphicBuffer> pGfxBuffer = 272 new GraphicBuffer(tgtBuffer.buffer.nativeHandle, 273 GraphicBuffer::CLONE_HANDLE, 274 pDesc->width, 275 pDesc->height, 276 pDesc->format, 277 pDesc->layers, 278 GRALLOC_USAGE_HW_RENDER, 279 pDesc->stride); 280 if (pGfxBuffer == nullptr) { 281 LOG(ERROR) << "Failed to allocate GraphicBuffer to wrap image handle"; 282 return false; 283 } 284 285 // Get a GL compatible reference to the graphics buffer we've been given 286 EGLint eglImageAttributes[] = {EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, EGL_NONE}; 287 EGLClientBuffer clientBuf = static_cast<EGLClientBuffer>( 288 pGfxBuffer->getNativeBuffer()); 289 290 // Destroy current KHR image due to new request. 291 if (sKHRimage != EGL_NO_IMAGE_KHR) { 292 eglDestroyImageKHR(sGLDisplay, sKHRimage); 293 } 294 295 sKHRimage = eglCreateImageKHR(sGLDisplay, EGL_NO_CONTEXT, 296 EGL_NATIVE_BUFFER_ANDROID, clientBuf, 297 eglImageAttributes); 298 if (sKHRimage == EGL_NO_IMAGE_KHR) { 299 LOG(ERROR) << "error creating EGLImage for target buffer: " 300 << getEGLError(); 301 return false; 302 } 303 304 glBindFramebuffer(GL_FRAMEBUFFER, sFrameBuffer); 305 306 // Construct a render buffer around the external buffer 307 glBindRenderbuffer(GL_RENDERBUFFER, sColorBuffer); 308 glEGLImageTargetRenderbufferStorageOES( 309 GL_RENDERBUFFER, static_cast<GLeglImageOES>(sKHRimage)); 310 if (eglGetError() != EGL_SUCCESS) { 311 LOG(INFO) << "glEGLImageTargetRenderbufferStorageOES => %s" 312 << getEGLError(); 313 return false; 314 } 315 316 glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, 317 GL_RENDERBUFFER, sColorBuffer); 318 if (eglGetError() != EGL_SUCCESS) { 319 LOG(ERROR) << "glFramebufferRenderbuffer => %s", getEGLError(); 320 return false; 321 } 322 323 GLenum checkResult = glCheckFramebufferStatus(GL_FRAMEBUFFER); 324 if (checkResult != GL_FRAMEBUFFER_COMPLETE) { 325 LOG(ERROR) << "Offscreen framebuffer not configured successfully (" 326 << checkResult 327 << ": " 328 << getGLFramebufferError().c_str() 329 << ")"; 330 if (eglGetError() != EGL_SUCCESS) { 331 LOG(ERROR) << "glCheckFramebufferStatus => " 332 << getEGLError(); 333 } 334 return false; 335 } 336 337 // Set the viewport 338 glViewport(0, 0, pDesc->width, pDesc->height); 339 340 // We don't actually need the clear if we're going to cover the whole 341 // screen anyway 342 // Clear the color buffer 343 glClearColor(0.8f, 0.1f, 0.2f, 1.0f); 344 glClear(GL_COLOR_BUFFER_BIT); 345 346 return true; 347 } 348 349 void SurroundViewServiceCallback::detachRenderTarget() { 350 // Drop our external render target 351 if (sKHRimage != EGL_NO_IMAGE_KHR) { 352 eglDestroyImageKHR(sGLDisplay, sKHRimage); 353 sKHRimage = EGL_NO_IMAGE_KHR; 354 } 355 } 356 357 SurroundViewServiceCallback::SurroundViewServiceCallback( 358 sp<IEvsDisplay> pDisplay, 359 sp<ISurroundViewSession> pSession) : 360 mDisplay(pDisplay), 361 mSession(pSession) { 362 // Nothing but member initialization 363 } 364 365 Return<void> SurroundViewServiceCallback::notify(SvEvent svEvent) { 366 // Waiting for STREAM_STARTED event. 367 if (svEvent == SvEvent::STREAM_STARTED) { 368 LOG(INFO) << "Received STREAM_STARTED event"; 369 370 // Set the display state to VISIBLE_ON_NEXT_FRAME 371 if (mDisplay != nullptr) { 372 Return<EvsResult> result = 373 mDisplay->setDisplayState(DisplayState::VISIBLE_ON_NEXT_FRAME); 374 if (result != EvsResult::OK) { 375 LOG(ERROR) << "Failed to setDisplayState"; 376 } 377 } else { 378 LOG(WARNING) << "setDisplayState is ignored since EVS display" 379 << " is null"; 380 } 381 382 // Set up OpenGL (exit if fail) 383 if (!prepareGL()) { 384 LOG(ERROR) << "Error while setting up OpenGL!"; 385 exit(EXIT_FAILURE); 386 } 387 } else if (svEvent == SvEvent::CONFIG_UPDATED) { 388 LOG(INFO) << "Received CONFIG_UPDATED event"; 389 } else if (svEvent == SvEvent::STREAM_STOPPED) { 390 LOG(INFO) << "Received STREAM_STOPPED event"; 391 } else if (svEvent == SvEvent::FRAME_DROPPED) { 392 LOG(INFO) << "Received FRAME_DROPPED event"; 393 } else if (svEvent == SvEvent::TIMEOUT) { 394 LOG(INFO) << "Received TIMEOUT event"; 395 } else { 396 LOG(INFO) << "Received unknown event"; 397 } 398 return {}; 399 } 400 401 Return<void> SurroundViewServiceCallback::receiveFrames( 402 const SvFramesDesc& svFramesDesc) { 403 LOG(INFO) << "Incoming frames with svBuffers size: " 404 << svFramesDesc.svBuffers.size(); 405 if (svFramesDesc.svBuffers.size() == 0) { 406 return {}; 407 } 408 409 // Now we assume there is only one frame for both 2d and 3d. 410 auto handle = 411 svFramesDesc.svBuffers[0].hardwareBuffer.nativeHandle 412 .getNativeHandle(); 413 const AHardwareBuffer_Desc* pDesc = 414 reinterpret_cast<const AHardwareBuffer_Desc *>( 415 &svFramesDesc.svBuffers[0].hardwareBuffer.description); 416 417 LOG(INFO) << "App received frames"; 418 LOG(INFO) << "descData: " 419 << pDesc->width 420 << pDesc->height 421 << pDesc->layers 422 << pDesc->format 423 << pDesc->usage 424 << pDesc->stride; 425 LOG(INFO) << "nativeHandle: " 426 << handle; 427 428 // Only process the frame when EVS display is valid. If 429 // not, ignore the coming frame. 430 if (mDisplay == nullptr) { 431 LOG(WARNING) << "Display is not ready. Skip the frame"; 432 } else { 433 // Get display buffer from EVS display 434 BufferDesc_1_0 tgtBuffer = {}; 435 mDisplay->getTargetBuffer([&tgtBuffer](const BufferDesc_1_0& buff) { 436 tgtBuffer = buff; 437 }); 438 439 if (!attachRenderTarget(convertBufferDesc(tgtBuffer))) { 440 LOG(ERROR) << "Failed to attach render target"; 441 return {}; 442 } else { 443 LOG(INFO) << "Successfully attached render target"; 444 } 445 446 // Call HIDL API "doneWithFrames" to return the ownership 447 // back to SV service 448 if (mSession == nullptr) { 449 LOG(ERROR) << "SurroundViewSession in callback is invalid"; 450 return {}; 451 } else { 452 mSession->doneWithFrames(svFramesDesc); 453 } 454 455 // Render frame to EVS display 456 LOG(INFO) << "Rendering to display buffer"; 457 sp<GraphicBuffer> graphicBuffer = 458 new GraphicBuffer(handle, 459 GraphicBuffer::CLONE_HANDLE, 460 pDesc->width, 461 pDesc->height, 462 pDesc->format, 463 pDesc->layers, // layer count 464 pDesc->usage, 465 pDesc->stride); 466 467 EGLImageKHR KHRimage = EGL_NO_IMAGE_KHR; 468 469 // Get a GL compatible reference to the graphics buffer we've been given 470 EGLint eglImageAttributes[] = { 471 EGL_IMAGE_PRESERVED_KHR, 472 EGL_TRUE, 473 EGL_NONE 474 }; 475 EGLClientBuffer clientBuf = static_cast<EGLClientBuffer>( 476 graphicBuffer->getNativeBuffer()); 477 KHRimage = eglCreateImageKHR(sGLDisplay, EGL_NO_CONTEXT, 478 EGL_NATIVE_BUFFER_ANDROID, clientBuf, 479 eglImageAttributes); 480 if (KHRimage == EGL_NO_IMAGE_KHR) { 481 const char *msg = getEGLError(); 482 LOG(ERROR) << "error creating EGLImage: " 483 << msg; 484 return {}; 485 } else { 486 LOG(INFO) << "Successfully created EGLImage"; 487 488 // Update the texture handle we already created to refer to 489 // this gralloc buffer 490 glActiveTexture(GL_TEXTURE0); 491 glBindTexture(GL_TEXTURE_2D, sTextureId); 492 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, 493 static_cast<GLeglImageOES>(KHRimage)); 494 495 // Initialize the sampling properties (it seems the sample may 496 // not work if this isn't done) 497 // The user of this texture may very well want to set their own 498 // filtering, but we're going to pay the (minor) price of 499 // setting this up for them to avoid the dreaded "black image" 500 // if they forget. 501 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, 502 GL_LINEAR); 503 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, 504 GL_NEAREST); 505 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, 506 GL_CLAMP_TO_EDGE); 507 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, 508 GL_CLAMP_TO_EDGE); 509 } 510 511 // Bind the texture and assign it to the shader's sampler 512 glActiveTexture(GL_TEXTURE0); 513 glBindTexture(GL_TEXTURE_2D, sTextureId); 514 515 // We want our image to show up opaque regardless of alpha values 516 glDisable(GL_BLEND); 517 518 // Draw a rectangle on the screen 519 const GLfloat vertsCarPos[] = { 520 -1.0, 1.0, 0.0f, // left top in window space 521 1.0, 1.0, 0.0f, // right top 522 -1.0, -1.0, 0.0f, // left bottom 523 1.0, -1.0, 0.0f // right bottom 524 }; 525 const GLfloat vertsCarTex[] = { 526 0.0f, 0.0f, // left top 527 1.0f, 0.0f, // right top 528 0.0f, 1.0f, // left bottom 529 1.0f, 1.0f // right bottom 530 }; 531 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, vertsCarPos); 532 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, vertsCarTex); 533 glEnableVertexAttribArray(0); 534 glEnableVertexAttribArray(1); 535 536 glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); 537 538 glDisableVertexAttribArray(0); 539 glDisableVertexAttribArray(1); 540 541 // Now that everything is submitted, release our hold on the 542 // texture resource 543 detachRenderTarget(); 544 545 // Wait for the rendering to finish 546 glFinish(); 547 detachRenderTarget(); 548 549 // Drop our external render target 550 if (KHRimage != EGL_NO_IMAGE_KHR) { 551 eglDestroyImageKHR(sGLDisplay, KHRimage); 552 KHRimage = EGL_NO_IMAGE_KHR; 553 } 554 555 LOG(DEBUG) << "Rendering finished. Going to return the buffer"; 556 557 // Call HIDL API "doneWithFrames" to return the ownership 558 // back to SV service 559 if (mSession == nullptr) { 560 LOG(WARNING) << "SurroundViewSession in callback is invalid"; 561 } else { 562 mSession->doneWithFrames(svFramesDesc); 563 } 564 565 // Return display buffer back to EVS display 566 mDisplay->returnTargetBufferForDisplay(tgtBuffer); 567 } 568 return {}; 569 } 570