1 /*
2 * Copyright 2020 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "SurroundViewServiceCallback.h"
18
19 #include <android-base/logging.h>
20 #include <math/mat4.h>
21 #include <ui/GraphicBuffer.h>
22 #include <utils/Log.h>
23
24 #include "shader_simpleTex.h"
25 #include "shader.h"
26
27 using android::GraphicBuffer;
28 using android::hardware::automotive::evs::V1_0::DisplayState;
29 using android::hardware::automotive::evs::V1_0::EvsResult;
30 using android::hardware::Return;
31 using android::sp;
32 using std::string;
33
34 EGLDisplay SurroundViewServiceCallback::sGLDisplay;
35 GLuint SurroundViewServiceCallback::sFrameBuffer;
36 GLuint SurroundViewServiceCallback::sColorBuffer;
37 GLuint SurroundViewServiceCallback::sDepthBuffer;
38 GLuint SurroundViewServiceCallback::sTextureId;
39 EGLImageKHR SurroundViewServiceCallback::sKHRimage;
40
getEGLError(void)41 const char* SurroundViewServiceCallback::getEGLError(void) {
42 switch (eglGetError()) {
43 case EGL_SUCCESS:
44 return "EGL_SUCCESS";
45 case EGL_NOT_INITIALIZED:
46 return "EGL_NOT_INITIALIZED";
47 case EGL_BAD_ACCESS:
48 return "EGL_BAD_ACCESS";
49 case EGL_BAD_ALLOC:
50 return "EGL_BAD_ALLOC";
51 case EGL_BAD_ATTRIBUTE:
52 return "EGL_BAD_ATTRIBUTE";
53 case EGL_BAD_CONTEXT:
54 return "EGL_BAD_CONTEXT";
55 case EGL_BAD_CONFIG:
56 return "EGL_BAD_CONFIG";
57 case EGL_BAD_CURRENT_SURFACE:
58 return "EGL_BAD_CURRENT_SURFACE";
59 case EGL_BAD_DISPLAY:
60 return "EGL_BAD_DISPLAY";
61 case EGL_BAD_SURFACE:
62 return "EGL_BAD_SURFACE";
63 case EGL_BAD_MATCH:
64 return "EGL_BAD_MATCH";
65 case EGL_BAD_PARAMETER:
66 return "EGL_BAD_PARAMETER";
67 case EGL_BAD_NATIVE_PIXMAP:
68 return "EGL_BAD_NATIVE_PIXMAP";
69 case EGL_BAD_NATIVE_WINDOW:
70 return "EGL_BAD_NATIVE_WINDOW";
71 case EGL_CONTEXT_LOST:
72 return "EGL_CONTEXT_LOST";
73 default:
74 return "Unknown error";
75 }
76 }
77
getGLFramebufferError(void)78 const string SurroundViewServiceCallback::getGLFramebufferError(void) {
79 switch (glCheckFramebufferStatus(GL_FRAMEBUFFER)) {
80 case GL_FRAMEBUFFER_COMPLETE:
81 return "GL_FRAMEBUFFER_COMPLETE";
82 case GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT:
83 return "GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT";
84 case GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT:
85 return "GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT";
86 case GL_FRAMEBUFFER_UNSUPPORTED:
87 return "GL_FRAMEBUFFER_UNSUPPORTED";
88 case GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS:
89 return "GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS";
90 default:
91 return std::to_string(glCheckFramebufferStatus(GL_FRAMEBUFFER));
92 }
93 }
94
prepareGL()95 bool SurroundViewServiceCallback::prepareGL() {
96 LOG(DEBUG) << __FUNCTION__;
97
98 // Just trivially return success if we're already prepared
99 if (sGLDisplay != EGL_NO_DISPLAY) {
100 return true;
101 }
102
103 // Hardcoded to RGBx output display
104 const EGLint config_attribs[] = {
105 // Tag Value
106 EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
107 EGL_RED_SIZE, 8,
108 EGL_GREEN_SIZE, 8,
109 EGL_BLUE_SIZE, 8,
110 EGL_NONE
111 };
112
113 // Select OpenGL ES v 3
114 const EGLint context_attribs[] = {EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE};
115
116 // Set up our OpenGL ES context associated with the default display
117 // (though we won't be visible)
118 EGLDisplay display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
119 if (display == EGL_NO_DISPLAY) {
120 LOG(ERROR) << "Failed to get egl display";
121 return false;
122 }
123
124 EGLint major = 0;
125 EGLint minor = 0;
126 if (!eglInitialize(display, &major, &minor)) {
127 LOG(ERROR) << "Failed to initialize EGL: "
128 << getEGLError();
129 return false;
130 } else {
131 LOG(INFO) << "Initialized EGL at "
132 << major
133 << "."
134 << minor;
135 }
136
137 // Select the configuration that "best" matches our desired characteristics
138 EGLConfig egl_config;
139 EGLint num_configs;
140 if (!eglChooseConfig(display, config_attribs, &egl_config, 1,
141 &num_configs)) {
142 LOG(ERROR) << "eglChooseConfig() failed with error: "
143 << getEGLError();
144 return false;
145 }
146
147 // Create a dummy pbuffer so we have a surface to bind -- we never intend
148 // to draw to this because attachRenderTarget will be called first.
149 EGLint surface_attribs[] = { EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE };
150 EGLSurface sDummySurface = eglCreatePbufferSurface(display, egl_config,
151 surface_attribs);
152 if (sDummySurface == EGL_NO_SURFACE) {
153 LOG(ERROR) << "Failed to create OpenGL ES Dummy surface: "
154 << getEGLError();
155 return false;
156 } else {
157 LOG(INFO) << "Dummy surface looks good! :)";
158 }
159
160 //
161 // Create the EGL context
162 //
163 EGLContext context = eglCreateContext(display, egl_config,
164 EGL_NO_CONTEXT, context_attribs);
165 if (context == EGL_NO_CONTEXT) {
166 LOG(ERROR) << "Failed to create OpenGL ES Context: "
167 << getEGLError();
168 return false;
169 }
170
171 // Activate our render target for drawing
172 if (!eglMakeCurrent(display, sDummySurface, sDummySurface, context)) {
173 LOG(ERROR) << "Failed to make the OpenGL ES Context current: "
174 << getEGLError();
175 return false;
176 } else {
177 LOG(INFO) << "We made our context current! :)";
178 }
179
180 // Report the extensions available on this implementation
181 const char* gl_extensions = (const char*) glGetString(GL_EXTENSIONS);
182 LOG(INFO) << "GL EXTENSIONS:\n "
183 << gl_extensions;
184
185 // Reserve handles for the color and depth targets we'll be setting up
186 glGenRenderbuffers(1, &sColorBuffer);
187 glGenRenderbuffers(1, &sDepthBuffer);
188
189 // Set up the frame buffer object we can modify and use for off screen
190 // rendering
191 glGenFramebuffers(1, &sFrameBuffer);
192 glBindFramebuffer(GL_FRAMEBUFFER, sFrameBuffer);
193
194 LOG(INFO) << "FrameBuffer is bound to "
195 << sFrameBuffer;
196
197 // New (from TextWrapper)
198 glGenTextures(1, &sTextureId);
199
200 // Now that we're assured success, store object handles we constructed
201 sGLDisplay = display;
202
203 GLuint mShaderProgram = 0;
204 // Load our shader program if we don't have it already
205 if (!mShaderProgram) {
206 mShaderProgram = buildShaderProgram(kVtxShaderSimpleTexture,
207 kPixShaderSimpleTexture,
208 "simpleTexture");
209 if (!mShaderProgram) {
210 LOG(ERROR) << "Error building shader program";
211 return false;
212 }
213 }
214
215 // Select our screen space simple texture shader
216 glUseProgram(mShaderProgram);
217
218 // Set up the model to clip space transform (identity matrix if we're
219 // modeling in screen space)
220 GLint loc = glGetUniformLocation(mShaderProgram, "cameraMat");
221 if (loc < 0) {
222 LOG(ERROR) << "Couldn't set shader parameter 'cameraMat'";
223 } else {
224 const android::mat4 identityMatrix;
225 glUniformMatrix4fv(loc, 1, false, identityMatrix.asArray());
226 }
227
228 GLint sampler = glGetUniformLocation(mShaderProgram, "tex");
229 if (sampler < 0) {
230 LOG(ERROR) << "Couldn't set shader parameter 'tex'";
231 } else {
232 // Tell the sampler we looked up from the shader to use texture slot 0
233 // as its source
234 glUniform1i(sampler, 0);
235 }
236
237 return true;
238 }
239
convertBufferDesc(const BufferDesc_1_0 & src)240 BufferDesc SurroundViewServiceCallback::convertBufferDesc(
241 const BufferDesc_1_0& src) {
242 BufferDesc dst = {};
243 AHardwareBuffer_Desc* pDesc =
244 reinterpret_cast<AHardwareBuffer_Desc *>(&dst.buffer.description);
245 pDesc->width = src.width;
246 pDesc->height = src.height;
247 pDesc->layers = 1;
248 pDesc->format = src.format;
249 pDesc->usage = static_cast<uint64_t>(src.usage);
250 pDesc->stride = src.stride;
251
252 dst.buffer.nativeHandle = src.memHandle;
253 dst.pixelSize = src.pixelSize;
254 dst.bufferId = src.bufferId;
255
256 return dst;
257 }
258
attachRenderTarget(const BufferDesc & tgtBuffer)259 bool SurroundViewServiceCallback::attachRenderTarget(
260 const BufferDesc& tgtBuffer) {
261 const AHardwareBuffer_Desc* pDesc =
262 reinterpret_cast<const AHardwareBuffer_Desc *>(
263 &tgtBuffer.buffer.description);
264 // Hardcoded to RGBx for now
265 if (pDesc->format != HAL_PIXEL_FORMAT_RGBA_8888) {
266 LOG(ERROR) << "Unsupported target buffer format";
267 return false;
268 }
269
270 // create a GraphicBuffer from the existing handle
271 sp<GraphicBuffer> pGfxBuffer =
272 new GraphicBuffer(tgtBuffer.buffer.nativeHandle,
273 GraphicBuffer::CLONE_HANDLE,
274 pDesc->width,
275 pDesc->height,
276 pDesc->format,
277 pDesc->layers,
278 GRALLOC_USAGE_HW_RENDER,
279 pDesc->stride);
280 if (pGfxBuffer == nullptr) {
281 LOG(ERROR) << "Failed to allocate GraphicBuffer to wrap image handle";
282 return false;
283 }
284
285 // Get a GL compatible reference to the graphics buffer we've been given
286 EGLint eglImageAttributes[] = {EGL_IMAGE_PRESERVED_KHR, EGL_TRUE, EGL_NONE};
287 EGLClientBuffer clientBuf = static_cast<EGLClientBuffer>(
288 pGfxBuffer->getNativeBuffer());
289
290 // Destroy current KHR image due to new request.
291 if (sKHRimage != EGL_NO_IMAGE_KHR) {
292 eglDestroyImageKHR(sGLDisplay, sKHRimage);
293 }
294
295 sKHRimage = eglCreateImageKHR(sGLDisplay, EGL_NO_CONTEXT,
296 EGL_NATIVE_BUFFER_ANDROID, clientBuf,
297 eglImageAttributes);
298 if (sKHRimage == EGL_NO_IMAGE_KHR) {
299 LOG(ERROR) << "error creating EGLImage for target buffer: "
300 << getEGLError();
301 return false;
302 }
303
304 glBindFramebuffer(GL_FRAMEBUFFER, sFrameBuffer);
305
306 // Construct a render buffer around the external buffer
307 glBindRenderbuffer(GL_RENDERBUFFER, sColorBuffer);
308 glEGLImageTargetRenderbufferStorageOES(
309 GL_RENDERBUFFER, static_cast<GLeglImageOES>(sKHRimage));
310 if (eglGetError() != EGL_SUCCESS) {
311 LOG(INFO) << "glEGLImageTargetRenderbufferStorageOES => %s"
312 << getEGLError();
313 return false;
314 }
315
316 glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0,
317 GL_RENDERBUFFER, sColorBuffer);
318 if (eglGetError() != EGL_SUCCESS) {
319 LOG(ERROR) << "glFramebufferRenderbuffer => %s", getEGLError();
320 return false;
321 }
322
323 GLenum checkResult = glCheckFramebufferStatus(GL_FRAMEBUFFER);
324 if (checkResult != GL_FRAMEBUFFER_COMPLETE) {
325 LOG(ERROR) << "Offscreen framebuffer not configured successfully ("
326 << checkResult
327 << ": "
328 << getGLFramebufferError().c_str()
329 << ")";
330 if (eglGetError() != EGL_SUCCESS) {
331 LOG(ERROR) << "glCheckFramebufferStatus => "
332 << getEGLError();
333 }
334 return false;
335 }
336
337 // Set the viewport
338 glViewport(0, 0, pDesc->width, pDesc->height);
339
340 // We don't actually need the clear if we're going to cover the whole
341 // screen anyway
342 // Clear the color buffer
343 glClearColor(0.8f, 0.1f, 0.2f, 1.0f);
344 glClear(GL_COLOR_BUFFER_BIT);
345
346 return true;
347 }
348
detachRenderTarget()349 void SurroundViewServiceCallback::detachRenderTarget() {
350 // Drop our external render target
351 if (sKHRimage != EGL_NO_IMAGE_KHR) {
352 eglDestroyImageKHR(sGLDisplay, sKHRimage);
353 sKHRimage = EGL_NO_IMAGE_KHR;
354 }
355 }
356
SurroundViewServiceCallback(sp<IEvsDisplay> pDisplay,sp<ISurroundViewSession> pSession)357 SurroundViewServiceCallback::SurroundViewServiceCallback(
358 sp<IEvsDisplay> pDisplay,
359 sp<ISurroundViewSession> pSession) :
360 mDisplay(pDisplay),
361 mSession(pSession) {
362 // Nothing but member initialization
363 }
364
notify(SvEvent svEvent)365 Return<void> SurroundViewServiceCallback::notify(SvEvent svEvent) {
366 // Waiting for STREAM_STARTED event.
367 if (svEvent == SvEvent::STREAM_STARTED) {
368 LOG(INFO) << "Received STREAM_STARTED event";
369
370 // Set the display state to VISIBLE_ON_NEXT_FRAME
371 if (mDisplay != nullptr) {
372 Return<EvsResult> result =
373 mDisplay->setDisplayState(DisplayState::VISIBLE_ON_NEXT_FRAME);
374 if (result != EvsResult::OK) {
375 LOG(ERROR) << "Failed to setDisplayState";
376 }
377 } else {
378 LOG(WARNING) << "setDisplayState is ignored since EVS display"
379 << " is null";
380 }
381
382 // Set up OpenGL (exit if fail)
383 if (!prepareGL()) {
384 LOG(ERROR) << "Error while setting up OpenGL!";
385 exit(EXIT_FAILURE);
386 }
387 } else if (svEvent == SvEvent::CONFIG_UPDATED) {
388 LOG(INFO) << "Received CONFIG_UPDATED event";
389 } else if (svEvent == SvEvent::STREAM_STOPPED) {
390 LOG(INFO) << "Received STREAM_STOPPED event";
391 } else if (svEvent == SvEvent::FRAME_DROPPED) {
392 LOG(INFO) << "Received FRAME_DROPPED event";
393 } else if (svEvent == SvEvent::TIMEOUT) {
394 LOG(INFO) << "Received TIMEOUT event";
395 } else {
396 LOG(INFO) << "Received unknown event";
397 }
398 return {};
399 }
400
receiveFrames(const SvFramesDesc & svFramesDesc)401 Return<void> SurroundViewServiceCallback::receiveFrames(
402 const SvFramesDesc& svFramesDesc) {
403 LOG(INFO) << "Incoming frames with svBuffers size: "
404 << svFramesDesc.svBuffers.size();
405 if (svFramesDesc.svBuffers.size() == 0) {
406 return {};
407 }
408
409 // Now we assume there is only one frame for both 2d and 3d.
410 auto handle =
411 svFramesDesc.svBuffers[0].hardwareBuffer.nativeHandle
412 .getNativeHandle();
413 const AHardwareBuffer_Desc* pDesc =
414 reinterpret_cast<const AHardwareBuffer_Desc *>(
415 &svFramesDesc.svBuffers[0].hardwareBuffer.description);
416
417 LOG(INFO) << "App received frames";
418 LOG(INFO) << "descData: "
419 << pDesc->width
420 << pDesc->height
421 << pDesc->layers
422 << pDesc->format
423 << pDesc->usage
424 << pDesc->stride;
425 LOG(INFO) << "nativeHandle: "
426 << handle;
427
428 // Only process the frame when EVS display is valid. If
429 // not, ignore the coming frame.
430 if (mDisplay == nullptr) {
431 LOG(WARNING) << "Display is not ready. Skip the frame";
432 } else {
433 // Get display buffer from EVS display
434 BufferDesc_1_0 tgtBuffer = {};
435 mDisplay->getTargetBuffer([&tgtBuffer](const BufferDesc_1_0& buff) {
436 tgtBuffer = buff;
437 });
438
439 if (!attachRenderTarget(convertBufferDesc(tgtBuffer))) {
440 LOG(ERROR) << "Failed to attach render target";
441 return {};
442 } else {
443 LOG(INFO) << "Successfully attached render target";
444 }
445
446 // Call HIDL API "doneWithFrames" to return the ownership
447 // back to SV service
448 if (mSession == nullptr) {
449 LOG(ERROR) << "SurroundViewSession in callback is invalid";
450 return {};
451 } else {
452 mSession->doneWithFrames(svFramesDesc);
453 }
454
455 // Render frame to EVS display
456 LOG(INFO) << "Rendering to display buffer";
457 sp<GraphicBuffer> graphicBuffer =
458 new GraphicBuffer(handle,
459 GraphicBuffer::CLONE_HANDLE,
460 pDesc->width,
461 pDesc->height,
462 pDesc->format,
463 pDesc->layers, // layer count
464 pDesc->usage,
465 pDesc->stride);
466
467 EGLImageKHR KHRimage = EGL_NO_IMAGE_KHR;
468
469 // Get a GL compatible reference to the graphics buffer we've been given
470 EGLint eglImageAttributes[] = {
471 EGL_IMAGE_PRESERVED_KHR,
472 EGL_TRUE,
473 EGL_NONE
474 };
475 EGLClientBuffer clientBuf = static_cast<EGLClientBuffer>(
476 graphicBuffer->getNativeBuffer());
477 KHRimage = eglCreateImageKHR(sGLDisplay, EGL_NO_CONTEXT,
478 EGL_NATIVE_BUFFER_ANDROID, clientBuf,
479 eglImageAttributes);
480 if (KHRimage == EGL_NO_IMAGE_KHR) {
481 const char *msg = getEGLError();
482 LOG(ERROR) << "error creating EGLImage: "
483 << msg;
484 return {};
485 } else {
486 LOG(INFO) << "Successfully created EGLImage";
487
488 // Update the texture handle we already created to refer to
489 // this gralloc buffer
490 glActiveTexture(GL_TEXTURE0);
491 glBindTexture(GL_TEXTURE_2D, sTextureId);
492 glEGLImageTargetTexture2DOES(GL_TEXTURE_2D,
493 static_cast<GLeglImageOES>(KHRimage));
494
495 // Initialize the sampling properties (it seems the sample may
496 // not work if this isn't done)
497 // The user of this texture may very well want to set their own
498 // filtering, but we're going to pay the (minor) price of
499 // setting this up for them to avoid the dreaded "black image"
500 // if they forget.
501 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER,
502 GL_LINEAR);
503 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER,
504 GL_NEAREST);
505 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S,
506 GL_CLAMP_TO_EDGE);
507 glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T,
508 GL_CLAMP_TO_EDGE);
509 }
510
511 // Bind the texture and assign it to the shader's sampler
512 glActiveTexture(GL_TEXTURE0);
513 glBindTexture(GL_TEXTURE_2D, sTextureId);
514
515 // We want our image to show up opaque regardless of alpha values
516 glDisable(GL_BLEND);
517
518 // Draw a rectangle on the screen
519 const GLfloat vertsCarPos[] = {
520 -1.0, 1.0, 0.0f, // left top in window space
521 1.0, 1.0, 0.0f, // right top
522 -1.0, -1.0, 0.0f, // left bottom
523 1.0, -1.0, 0.0f // right bottom
524 };
525 const GLfloat vertsCarTex[] = {
526 0.0f, 0.0f, // left top
527 1.0f, 0.0f, // right top
528 0.0f, 1.0f, // left bottom
529 1.0f, 1.0f // right bottom
530 };
531 glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, 0, vertsCarPos);
532 glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 0, vertsCarTex);
533 glEnableVertexAttribArray(0);
534 glEnableVertexAttribArray(1);
535
536 glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
537
538 glDisableVertexAttribArray(0);
539 glDisableVertexAttribArray(1);
540
541 // Now that everything is submitted, release our hold on the
542 // texture resource
543 detachRenderTarget();
544
545 // Wait for the rendering to finish
546 glFinish();
547 detachRenderTarget();
548
549 // Drop our external render target
550 if (KHRimage != EGL_NO_IMAGE_KHR) {
551 eglDestroyImageKHR(sGLDisplay, KHRimage);
552 KHRimage = EGL_NO_IMAGE_KHR;
553 }
554
555 LOG(DEBUG) << "Rendering finished. Going to return the buffer";
556
557 // Call HIDL API "doneWithFrames" to return the ownership
558 // back to SV service
559 if (mSession == nullptr) {
560 LOG(WARNING) << "SurroundViewSession in callback is invalid";
561 } else {
562 mSession->doneWithFrames(svFramesDesc);
563 }
564
565 // Return display buffer back to EVS display
566 mDisplay->returnTargetBufferForDisplay(tgtBuffer);
567 }
568 return {};
569 }
570