1 // Copyright (C) 2018 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "VulkanDispatch.h"
16 
17 #include "aemu/base/SharedLibrary.h"
18 #include "aemu/base/files/PathUtils.h"
19 #include "aemu/base/synchronization/Lock.h"
20 #include "aemu/base/system/System.h"
21 #include "host-common/misc.h"
22 
23 using android::base::AutoLock;
24 using android::base::Lock;
25 using android::base::pj;
26 
27 namespace gfxstream {
28 namespace vk {
29 
icdJsonNameToProgramAndLauncherPaths(const std::string & icdFilename)30 static std::string icdJsonNameToProgramAndLauncherPaths(const std::string& icdFilename) {
31     std::string suffix = pj({"lib64", "vulkan", icdFilename});
32 #if defined(_WIN32)
33     const char* sep = ";";
34 #else
35     const char* sep = ":";
36 #endif
37     return pj({android::base::getProgramDirectory(), suffix}) + sep +
38            pj({android::base::getLauncherDirectory(), suffix});
39 }
40 
setIcdPaths(const std::string & icdFilename)41 static void setIcdPaths(const std::string& icdFilename) {
42     const std::string paths = icdJsonNameToProgramAndLauncherPaths(icdFilename);
43     INFO("Setting ICD filenames for the loader = %s", paths.c_str());
44     android::base::setEnvironmentVariable("VK_ICD_FILENAMES", paths);
45 }
46 
getTestIcdFilename()47 static const char* getTestIcdFilename() {
48 #if defined(__APPLE__)
49     return "libvk_swiftshader.dylib";
50 #elif defined(__linux__) || defined(__QNX__)
51     return "libvk_swiftshader.so";
52 #elif defined(_WIN32) || defined(_MSC_VER)
53     return "vk_swiftshader.dll";
54 #else
55 #error Host operating system not supported
56 #endif
57 }
58 
initIcdPaths(bool forTesting)59 static void initIcdPaths(bool forTesting) {
60     auto androidIcd = android::base::getEnvironmentVariable("ANDROID_EMU_VK_ICD");
61     if (androidIcd == "") {
62         // Rely on user to set VK_ICD_FILENAMES
63         return;
64     }
65 
66     if (forTesting) {
67         const char* testingICD = "swiftshader";
68         INFO("%s: In test environment, enforcing %s ICD.", __func__, testingICD);
69         android::base::setEnvironmentVariable("ANDROID_EMU_VK_ICD", testingICD);
70         androidIcd = testingICD;
71     }
72 
73     if (androidIcd == "swiftshader") {
74         INFO("%s: ICD set to 'swiftshader', using Swiftshader ICD", __func__);
75         setIcdPaths("vk_swiftshader_icd.json");
76     } else {
77 #ifdef __APPLE__
78         // Mac: Use MoltenVK by default unless GPU mode is set to swiftshader
79         if (androidIcd != "moltenvk") {
80             WARN("%s: Unknown ICD, resetting to MoltenVK", __func__);
81             android::base::setEnvironmentVariable("ANDROID_EMU_VK_ICD", "moltenvk");
82         }
83         setIcdPaths("MoltenVK_icd.json");
84 
85         // Configure MoltenVK library with environment variables
86         // 0: No logging.
87         // 1: Log errors only.
88         // 2: Log errors and warning messages.
89         // 3: Log errors, warnings and informational messages.
90         // 4: Log errors, warnings, infos and debug messages.
91         const bool verboseLogs =
92             (android::base::getEnvironmentVariable("ANDROID_EMUGL_VERBOSE") == "1");
93         const char* logLevelValue = verboseLogs ? "4" : "2";
94         android::base::setEnvironmentVariable("MVK_CONFIG_LOG_LEVEL", logLevelValue);
95 
96         //  Limit MoltenVK to use single queue, as some older ANGLE versions
97         //  expect this for -guest-angle to work.
98         //  0: Limit Vulkan to a single queue, with no explicit semaphore
99         //  synchronization, and use Metal's implicit guarantees that all operations
100         //  submitted to a queue will give the same result as if they had been run in
101         //  submission order.
102         android::base::setEnvironmentVariable("MVK_CONFIG_VK_SEMAPHORE_SUPPORT_STYLE", "0");
103 #else
104         // By default, on other platforms, just use whatever the system
105         // is packing.
106 #endif
107     }
108 }
109 
110 class SharedLibraries {
111    public:
SharedLibraries(size_t sizeLimit=1)112     explicit SharedLibraries(size_t sizeLimit = 1) : mSizeLimit(sizeLimit) {}
113 
size() const114     size_t size() const { return mLibs.size(); }
115 
addLibrary(const std::string & path)116     bool addLibrary(const std::string& path) {
117         if (size() >= mSizeLimit) {
118             fprintf(stderr, "cannot add library %s: full\n", path.c_str());
119             return false;
120         }
121 
122         auto library = android::base::SharedLibrary::open(path.c_str());
123         if (library) {
124             mLibs.push_back(library);
125             fprintf(stderr, "added library %s\n", path.c_str());
126             return true;
127         } else {
128             fprintf(stderr, "cannot add library %s: failed\n", path.c_str());
129             return false;
130         }
131     }
132 
addFirstAvailableLibrary(const std::vector<std::string> & possiblePaths)133     bool addFirstAvailableLibrary(const std::vector<std::string>& possiblePaths) {
134         for (const std::string& possiblePath : possiblePaths) {
135             if (addLibrary(possiblePath)) {
136                 return true;
137             }
138         }
139         return false;
140     }
141 
142     ~SharedLibraries() = default;
143 
dlsym(const char * name)144     void* dlsym(const char* name) {
145         for (const auto& lib : mLibs) {
146             void* funcPtr = reinterpret_cast<void*>(lib->findSymbol(name));
147             if (funcPtr) {
148                 return funcPtr;
149             }
150         }
151         return nullptr;
152     }
153 
154    private:
155     size_t mSizeLimit;
156     std::vector<android::base::SharedLibrary*> mLibs;
157 };
158 
getVulkanLibraryNumLimits()159 static constexpr size_t getVulkanLibraryNumLimits() {
160     // macOS may have both Vulkan loader (for non MoltenVK-specific functions) and
161     // MoltenVK library (only for MoltenVK-specific vk...MVK functions) loaded at
162     // the same time. So there could be at most 2 libraries loaded. On other systems
163     // only one Vulkan loader is allowed.
164 #ifdef __APPLE__
165     return 2;
166 #else
167     return 1;
168 #endif
169 }
170 
171 class VulkanDispatchImpl {
172    public:
VulkanDispatchImpl()173     VulkanDispatchImpl() : mVulkanLibs(getVulkanLibraryNumLimits()) {}
174 
175     void initialize(bool forTesting);
176 
getPossibleLoaderPathBasenames()177     static std::vector<std::string> getPossibleLoaderPathBasenames() {
178 #if defined(__APPLE__)
179         return std::vector<std::string>{"libvulkan.dylib"};
180 #elif defined(__linux__)
181         // When running applications with Gfxstream as the Vulkan ICD, i.e. with
182         //
183         //   App -> Vulkan Loader -> Gfxstream ICD -> Vulkan Loader -> Driver ICD
184         //
185         // Gfxstream needs to use a different nested loader library to avoid issues with
186         // conflating/deadlock with the first level loader. Detect that here and look for
187         // a "libvulkan_gfxstream" which can be generated with the provided
188         // scripts/build-nested-vulkan-loader.sh script.
189         const std::vector<std::string> nestedVulkanLoaderVars = {
190             "GFXSTREAM_VK_ADD_DRIVER_FILES",
191             "GFXSTREAM_VK_ADD_LAYER_PATH",
192             "GFXSTREAM_VK_DRIVER_FILES",
193             "GFXSTREAM_VK_ICD_FILENAMES",
194             "GFXSTREAM_VK_INSTANCE_LAYERS",
195             "GFXSTREAM_VK_LAYER_PATH",
196             "GFXSTREAM_VK_LAYER_PATH",
197             "GFXSTREAM_VK_LOADER_DEBUG",
198             "GFXSTREAM_VK_LOADER_DRIVERS_DISABLE",
199             "GFXSTREAM_VK_LOADER_DRIVERS_SELECT",
200             "GFXSTREAM_VK_LOADER_LAYERS_ALLOW",
201             "GFXSTREAM_VK_LOADER_LAYERS_DISABLE",
202             "GFXSTREAM_VK_LOADER_LAYERS_ENABLE",
203         };
204         bool usesNestedVulkanLoader = false;
205         for (const std::string& var : nestedVulkanLoaderVars) {
206             if (android::base::getEnvironmentVariable(var) != "") {
207                 usesNestedVulkanLoader = true;
208                 break;
209             }
210         }
211         if (usesNestedVulkanLoader) {
212             return std::vector<std::string>{
213                 "libvulkan_gfxstream.so",
214                 "libvulkan_gfxstream.so.1",
215             };
216         } else {
217             return std::vector<std::string>{
218                 "libvulkan.so",
219                 "libvulkan.so.1",
220             };
221         }
222 
223 #elif defined(_WIN32)
224         return std::vector<std::string>{"vulkan-1.dll"};
225 #elif defined(__QNX__)
226         return std::vector<std::string>{
227             "libvulkan.so",
228             "libvulkan.so.1",
229         };
230 #else
231 #error "Unhandled platform in VulkanDispatchImpl."
232 #endif
233     }
234 
getPossibleLoaderPaths()235     std::vector<std::string> getPossibleLoaderPaths() {
236         const std::string explicitPath =
237             android::base::getEnvironmentVariable("ANDROID_EMU_VK_LOADER_PATH");
238         if (!explicitPath.empty()) {
239             return {
240                 explicitPath,
241             };
242         }
243 
244         const std::vector<std::string> possibleBasenames = getPossibleLoaderPathBasenames();
245 
246         const std::string explicitIcd = android::base::getEnvironmentVariable("ANDROID_EMU_VK_ICD");
247 
248 #ifdef _WIN32
249         constexpr const bool isWindows = true;
250 #else
251         constexpr const bool isWindows = false;
252 #endif
253         if (explicitIcd.empty() || isWindows) {
254             return possibleBasenames;
255         }
256 
257         std::vector<std::string> possibleDirectories;
258 
259         if (mForTesting || explicitIcd == "mock") {
260             possibleDirectories = {
261                 pj({android::base::getProgramDirectory(), "testlib64"}),
262                 pj({android::base::getLauncherDirectory(), "testlib64"}),
263             };
264         }
265 
266         possibleDirectories.push_back(
267             pj({android::base::getProgramDirectory(), "lib64", "vulkan"}));
268         possibleDirectories.push_back(
269             pj({android::base::getLauncherDirectory(), "lib64", "vulkan"}));
270 
271         std::vector<std::string> possiblePaths;
272         for (const std::string& possibleDirectory : possibleDirectories) {
273             for (const std::string& possibleBasename : possibleBasenames) {
274                 possiblePaths.push_back(pj({possibleDirectory, possibleBasename}));
275             }
276         }
277         return possiblePaths;
278     }
279 
280 #ifdef __APPLE__
getPossibleMoltenVkPaths()281     std::vector<std::string> getPossibleMoltenVkPaths() {
282         const std::string explicitPath =
283             android::base::getEnvironmentVariable("ANDROID_EMU_VK_LOADER_PATH");
284         if (!explicitPath.empty()) {
285             return {
286                 explicitPath,
287             };
288         }
289 
290         const std::string& customIcd = android::base::getEnvironmentVariable("ANDROID_EMU_VK_ICD");
291 
292         // Skip loader when using MoltenVK as this gives us access to
293         // VK_MVK_moltenvk, which is required for external memory support.
294         if (!mForTesting && customIcd == "moltenvk") {
295             return {
296                 pj({android::base::getProgramDirectory(), "lib64", "vulkan", "libMoltenVK.dylib"}),
297                 pj({android::base::getLauncherDirectory(), "lib64", "vulkan", "libMoltenVK.dylib"}),
298             };
299         }
300 
301         return {};
302     }
303 #endif
304 
dlopen()305     void* dlopen() {
306         if (mVulkanLibs.size() == 0) {
307             mVulkanLibs.addFirstAvailableLibrary(getPossibleLoaderPaths());
308 
309 #ifdef __APPLE__
310             // On macOS it is possible that we are using MoltenVK as the
311             // ICD. In that case we need to add MoltenVK libraries to
312             // mSharedLibs to use MoltenVK-specific functions.
313             mVulkanLibs.addFirstAvailableLibrary(getPossibleMoltenVkPaths());
314 #endif
315         }
316         return static_cast<void*>(&mVulkanLibs);
317     }
318 
dlsym(void * lib,const char * name)319     void* dlsym(void* lib, const char* name) {
320         return (void*)((SharedLibraries*)(lib))->dlsym(name);
321     }
322 
dispatch()323     VulkanDispatch* dispatch() { return &mDispatch; }
324 
325    private:
326     Lock mLock;
327     bool mForTesting = false;
328     bool mInitialized = false;
329     VulkanDispatch mDispatch;
330     SharedLibraries mVulkanLibs;
331 };
332 
sVulkanDispatchImpl()333 VulkanDispatchImpl* sVulkanDispatchImpl() {
334     static VulkanDispatchImpl* impl = new VulkanDispatchImpl;
335     return impl;
336 }
337 
sVulkanDispatchDlOpen()338 static void* sVulkanDispatchDlOpen() { return sVulkanDispatchImpl()->dlopen(); }
339 
sVulkanDispatchDlSym(void * lib,const char * sym)340 static void* sVulkanDispatchDlSym(void* lib, const char* sym) {
341     return sVulkanDispatchImpl()->dlsym(lib, sym);
342 }
343 
initialize(bool forTesting)344 void VulkanDispatchImpl::initialize(bool forTesting) {
345     AutoLock lock(mLock);
346 
347     if (mInitialized) {
348         return;
349     }
350 
351     mForTesting = forTesting;
352     initIcdPaths(mForTesting);
353 
354     init_vulkan_dispatch_from_system_loader(sVulkanDispatchDlOpen, sVulkanDispatchDlSym,
355                                             &mDispatch);
356 
357     mInitialized = true;
358 }
359 
vkDispatch(bool forTesting)360 VulkanDispatch* vkDispatch(bool forTesting) {
361     sVulkanDispatchImpl()->initialize(forTesting);
362     return sVulkanDispatchImpl()->dispatch();
363 }
364 
vkDispatchValid(const VulkanDispatch * vk)365 bool vkDispatchValid(const VulkanDispatch* vk) {
366     return vk->vkEnumerateInstanceExtensionProperties != nullptr ||
367            vk->vkGetInstanceProcAddr != nullptr || vk->vkGetDeviceProcAddr != nullptr;
368 }
369 
370 }  // namespace vk
371 }  // namespace gfxstream
372