1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 #ifndef TENSORFLOW_LITE_DELEGATES_GPU_CL_GL_INTEROP_H_
17 #define TENSORFLOW_LITE_DELEGATES_GPU_CL_GL_INTEROP_H_
18 
19 #include <EGL/egl.h>
20 #include <EGL/eglext.h>
21 
22 #include <vector>
23 
24 #include "tensorflow/lite/delegates/gpu/cl/cl_command_queue.h"
25 #include "tensorflow/lite/delegates/gpu/cl/cl_context.h"
26 #include "tensorflow/lite/delegates/gpu/cl/cl_device.h"
27 #include "tensorflow/lite/delegates/gpu/cl/cl_event.h"
28 #include "tensorflow/lite/delegates/gpu/cl/cl_memory.h"
29 #include "tensorflow/lite/delegates/gpu/cl/egl_sync.h"
30 #include "tensorflow/lite/delegates/gpu/cl/environment.h"
31 #include "tensorflow/lite/delegates/gpu/cl/opencl_wrapper.h"
32 #include "tensorflow/lite/delegates/gpu/common/access_type.h"
33 #include "tensorflow/lite/delegates/gpu/common/status.h"
34 #include "tensorflow/lite/delegates/gpu/gl/portable_gl31.h"
35 #include "tensorflow/lite/delegates/gpu/spi.h"
36 
37 namespace tflite {
38 namespace gpu {
39 namespace cl {
40 
41 // Creates an EglSync from OpenCL event. Source event does not need to outlive
42 // returned sync and could be safely destroyed.
43 //
44 // Depends on EGL 1.5.
45 absl::Status CreateEglSyncFromClEvent(cl_event event, EGLDisplay display,
46                                       EglSync* sync);
47 
48 // Returns true if 'CreateEglSyncFromClEvent' is supported.
49 bool IsEglSyncFromClEventSupported();
50 
51 // Creates CL event from EGL sync.
52 // Created event could only be consumed by AcquiredGlObject::Acquire call as
53 // a 'wait_event'.
54 absl::Status CreateClEventFromEglSync(cl_context context,
55                                       const EglSync& egl_sync, CLEvent* event);
56 
57 // Returns true if 'CreateClEventFromEglSync' is supported.
58 bool IsClEventFromEglSyncSupported(const CLDevice& device);
59 
60 // Creates new CL memory object from OpenGL buffer.
61 absl::Status CreateClMemoryFromGlBuffer(GLuint gl_ssbo_id,
62                                         AccessType access_type,
63                                         CLContext* context, CLMemory* memory);
64 
65 // Creates new CL memory object from OpenGL texture.
66 absl::Status CreateClMemoryFromGlTexture(GLenum texture_target,
67                                          GLuint texture_id,
68                                          AccessType access_type,
69                                          CLContext* context, CLMemory* memory);
70 
71 // Returns true if GL objects could be shared with OpenCL context.
72 bool IsGlSharingSupported(const CLDevice& device);
73 
74 // RAII-wrapper for GL objects acquired into CL context.
75 class AcquiredGlObjects {
76  public:
77   static bool IsSupported(const CLDevice& device);
78 
AcquiredGlObjects()79   AcquiredGlObjects() : AcquiredGlObjects({}, nullptr) {}
80 
81   // Quitely releases OpenGL objects. It is recommended to call Release()
82   // explicitly to properly handle potential errors.
83   ~AcquiredGlObjects();
84 
85   // Acquires memory from the OpenGL context. Memory must be created by either
86   // CreateClMemoryFromGlBuffer or CreateClMemoryFromGlTexture calls.
87   // If 'acquire_event' is not nullptr, it will be signared once acquisition is
88   // complete.
89   static absl::Status Acquire(const std::vector<cl_mem>& memory,
90                               cl_command_queue queue,
91                               const std::vector<cl_event>& wait_events,
92                               CLEvent* acquire_event /* optional */,
93                               AcquiredGlObjects* objects);
94 
95   // Releases OpenCL memory back to OpenGL context. If 'release_event' is not
96   // nullptr, it will be signalled once release is complete.
97   absl::Status Release(const std::vector<cl_event>& wait_events,
98                        CLEvent* release_event /* optional */);
99 
100  private:
AcquiredGlObjects(const std::vector<cl_mem> & memory,cl_command_queue queue)101   AcquiredGlObjects(const std::vector<cl_mem>& memory, cl_command_queue queue)
102       : memory_(memory), queue_(queue) {}
103 
104   std::vector<cl_mem> memory_;
105   cl_command_queue queue_;
106 };
107 
108 // Incapsulates all complicated GL-CL synchronization. It manages life time of
109 // all appropriate events to ensure fast synchronization whenever possible.
110 class GlInteropFabric {
111  public:
112   GlInteropFabric(EGLDisplay egl_display, Environment* environment);
113 
114   // Ensures proper GL->CL synchronization is in place before
115   // GL objects that are mapped to CL objects are used.
116   absl::Status Start();
117 
118   // Puts appropriate CL->GL synchronization after all work is complete.
119   absl::Status Finish();
120 
121   // Registers memory to be used from GL context. Such CL memory object must
122   // be created with CreateClMemoryFromGlBuffer or CreateClMemoryFromGlTexture
123   // call.
124   void RegisterMemory(cl_mem memory);
125 
126   // Unregisters memory registered with RegisterMemory call.
127   void UnregisterMemory(cl_mem memory);
128 
129  private:
is_enabled()130   bool is_enabled() const { return egl_display_ && !memory_.empty(); }
131 
132   bool is_egl_sync_supported_;
133   bool is_egl_to_cl_mapping_supported_;
134   bool is_cl_to_egl_mapping_supported_;
135 
136   const EGLDisplay egl_display_;
137   cl_context context_;
138   cl_command_queue queue_;
139   std::vector<cl_mem> memory_;
140   AcquiredGlObjects gl_objects_;  // transient during Start/Finish calls.
141 };
142 
143 // Copies data from(to) GL buffer to(from) CL buffer using CPU.
144 class GlClBufferCopier : public TensorObjectConverter {
145  public:
IsSupported(const ObjectDef & input,const ObjectDef & output)146   static bool IsSupported(const ObjectDef& input, const ObjectDef& output) {
147     return input.data_type == output.data_type &&
148            input.data_layout == output.data_layout &&
149            ((input.object_type == ObjectType::OPENGL_SSBO &&
150              output.object_type == ObjectType::OPENCL_BUFFER) ||
151             (input.object_type == ObjectType::OPENCL_BUFFER &&
152              output.object_type == ObjectType::OPENGL_SSBO));
153   }
154 
155   GlClBufferCopier(const TensorObjectDef& input_def,
156                    const TensorObjectDef& output_def, Environment* environment);
157 
158   absl::Status Convert(const TensorObject& input_obj,
159                        const TensorObject& output_obj) override;
160 
161  private:
162   size_t size_in_bytes_;
163   CLCommandQueue* queue_ = nullptr;
164 };
165 
166 }  // namespace cl
167 }  // namespace gpu
168 }  // namespace tflite
169 
170 #endif  // TENSORFLOW_LITE_DELEGATES_GPU_CL_GL_INTEROP_H_
171