1 //===----------- rtl.cpp - Target independent OpenMP target RTL -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Functionality for handling RTL plugins.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "device.h"
14 #include "private.h"
15 #include "rtl.h"
16 
17 #include <cassert>
18 #include <cstdlib>
19 #include <cstring>
20 #include <dlfcn.h>
21 #include <mutex>
22 #include <string>
23 
24 // List of all plugins that can support offloading.
25 static const char *RTLNames[] = {
26     /* PowerPC target       */ "libomptarget.rtl.ppc64.so",
27     /* x86_64 target        */ "libomptarget.rtl.x86_64.so",
28     /* CUDA target          */ "libomptarget.rtl.cuda.so",
29     /* AArch64 target       */ "libomptarget.rtl.aarch64.so",
30     /* SX-Aurora VE target  */ "libomptarget.rtl.ve.so",
31     /* AMDGPU target        */ "libomptarget.rtl.amdgpu.so",
32 };
33 
34 PluginManager *PM;
35 
init()36 __attribute__((constructor(101))) void init() {
37   DP("Init target library!\n");
38   PM = new PluginManager();
39 }
40 
deinit()41 __attribute__((destructor(101))) void deinit() {
42   DP("Deinit target library!\n");
43   delete PM;
44 }
45 
LoadRTLs()46 void RTLsTy::LoadRTLs() {
47   // Parse environment variable OMP_TARGET_OFFLOAD (if set)
48   PM->TargetOffloadPolicy =
49       (kmp_target_offload_kind_t)__kmpc_get_target_offload();
50   if (PM->TargetOffloadPolicy == tgt_disabled) {
51     return;
52   }
53 
54   DP("Loading RTLs...\n");
55 
56   // Attempt to open all the plugins and, if they exist, check if the interface
57   // is correct and if they are supporting any devices.
58   for (auto *Name : RTLNames) {
59     DP("Loading library '%s'...\n", Name);
60     void *dynlib_handle = dlopen(Name, RTLD_NOW);
61 
62     if (!dynlib_handle) {
63       // Library does not exist or cannot be found.
64       DP("Unable to load library '%s': %s!\n", Name, dlerror());
65       continue;
66     }
67 
68     DP("Successfully loaded library '%s'!\n", Name);
69 
70     // Retrieve the RTL information from the runtime library.
71     RTLInfoTy R;
72 
73     R.LibraryHandler = dynlib_handle;
74     R.isUsed = false;
75 
76 #ifdef OMPTARGET_DEBUG
77     R.RTLName = Name;
78 #endif
79 
80     if (!(*((void **)&R.is_valid_binary) =
81               dlsym(dynlib_handle, "__tgt_rtl_is_valid_binary")))
82       continue;
83     if (!(*((void **)&R.number_of_devices) =
84               dlsym(dynlib_handle, "__tgt_rtl_number_of_devices")))
85       continue;
86     if (!(*((void **)&R.init_device) =
87               dlsym(dynlib_handle, "__tgt_rtl_init_device")))
88       continue;
89     if (!(*((void **)&R.load_binary) =
90               dlsym(dynlib_handle, "__tgt_rtl_load_binary")))
91       continue;
92     if (!(*((void **)&R.data_alloc) =
93               dlsym(dynlib_handle, "__tgt_rtl_data_alloc")))
94       continue;
95     if (!(*((void **)&R.data_submit) =
96               dlsym(dynlib_handle, "__tgt_rtl_data_submit")))
97       continue;
98     if (!(*((void **)&R.data_retrieve) =
99               dlsym(dynlib_handle, "__tgt_rtl_data_retrieve")))
100       continue;
101     if (!(*((void **)&R.data_delete) =
102               dlsym(dynlib_handle, "__tgt_rtl_data_delete")))
103       continue;
104     if (!(*((void **)&R.run_region) =
105               dlsym(dynlib_handle, "__tgt_rtl_run_target_region")))
106       continue;
107     if (!(*((void **)&R.run_team_region) =
108               dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region")))
109       continue;
110 
111     // Optional functions
112     *((void **)&R.init_requires) =
113         dlsym(dynlib_handle, "__tgt_rtl_init_requires");
114     *((void **)&R.data_submit_async) =
115         dlsym(dynlib_handle, "__tgt_rtl_data_submit_async");
116     *((void **)&R.data_retrieve_async) =
117         dlsym(dynlib_handle, "__tgt_rtl_data_retrieve_async");
118     *((void **)&R.run_region_async) =
119         dlsym(dynlib_handle, "__tgt_rtl_run_target_region_async");
120     *((void **)&R.run_team_region_async) =
121         dlsym(dynlib_handle, "__tgt_rtl_run_target_team_region_async");
122     *((void **)&R.synchronize) = dlsym(dynlib_handle, "__tgt_rtl_synchronize");
123     *((void **)&R.data_exchange) =
124         dlsym(dynlib_handle, "__tgt_rtl_data_exchange");
125     *((void **)&R.data_exchange_async) =
126         dlsym(dynlib_handle, "__tgt_rtl_data_exchange_async");
127     *((void **)&R.is_data_exchangable) =
128         dlsym(dynlib_handle, "__tgt_rtl_is_data_exchangable");
129 
130     // No devices are supported by this RTL?
131     if (!(R.NumberOfDevices = R.number_of_devices())) {
132       DP("No devices supported in this RTL\n");
133       continue;
134     }
135 
136     DP("Registering RTL %s supporting %d devices!\n", R.RTLName.c_str(),
137        R.NumberOfDevices);
138 
139     // The RTL is valid! Will save the information in the RTLs list.
140     AllRTLs.push_back(R);
141   }
142 
143   DP("RTLs loaded!\n");
144 
145   return;
146 }
147 
148 ////////////////////////////////////////////////////////////////////////////////
149 // Functionality for registering libs
150 
RegisterImageIntoTranslationTable(TranslationTable & TT,RTLInfoTy & RTL,__tgt_device_image * image)151 static void RegisterImageIntoTranslationTable(TranslationTable &TT,
152     RTLInfoTy &RTL, __tgt_device_image *image) {
153 
154   // same size, as when we increase one, we also increase the other.
155   assert(TT.TargetsTable.size() == TT.TargetsImages.size() &&
156          "We should have as many images as we have tables!");
157 
158   // Resize the Targets Table and Images to accommodate the new targets if
159   // required
160   unsigned TargetsTableMinimumSize = RTL.Idx + RTL.NumberOfDevices;
161 
162   if (TT.TargetsTable.size() < TargetsTableMinimumSize) {
163     TT.TargetsImages.resize(TargetsTableMinimumSize, 0);
164     TT.TargetsTable.resize(TargetsTableMinimumSize, 0);
165   }
166 
167   // Register the image in all devices for this target type.
168   for (int32_t i = 0; i < RTL.NumberOfDevices; ++i) {
169     // If we are changing the image we are also invalidating the target table.
170     if (TT.TargetsImages[RTL.Idx + i] != image) {
171       TT.TargetsImages[RTL.Idx + i] = image;
172       TT.TargetsTable[RTL.Idx + i] = 0; // lazy initialization of target table.
173     }
174   }
175 }
176 
177 ////////////////////////////////////////////////////////////////////////////////
178 // Functionality for registering Ctors/Dtors
179 
RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc * desc,__tgt_device_image * img,RTLInfoTy * RTL)180 static void RegisterGlobalCtorsDtorsForImage(__tgt_bin_desc *desc,
181     __tgt_device_image *img, RTLInfoTy *RTL) {
182 
183   for (int32_t i = 0; i < RTL->NumberOfDevices; ++i) {
184     DeviceTy &Device = PM->Devices[RTL->Idx + i];
185     Device.PendingGlobalsMtx.lock();
186     Device.HasPendingGlobals = true;
187     for (__tgt_offload_entry *entry = img->EntriesBegin;
188         entry != img->EntriesEnd; ++entry) {
189       if (entry->flags & OMP_DECLARE_TARGET_CTOR) {
190         DP("Adding ctor " DPxMOD " to the pending list.\n",
191             DPxPTR(entry->addr));
192         Device.PendingCtorsDtors[desc].PendingCtors.push_back(entry->addr);
193       } else if (entry->flags & OMP_DECLARE_TARGET_DTOR) {
194         // Dtors are pushed in reverse order so they are executed from end
195         // to beginning when unregistering the library!
196         DP("Adding dtor " DPxMOD " to the pending list.\n",
197             DPxPTR(entry->addr));
198         Device.PendingCtorsDtors[desc].PendingDtors.push_front(entry->addr);
199       }
200 
201       if (entry->flags & OMP_DECLARE_TARGET_LINK) {
202         DP("The \"link\" attribute is not yet supported!\n");
203       }
204     }
205     Device.PendingGlobalsMtx.unlock();
206   }
207 }
208 
RegisterRequires(int64_t flags)209 void RTLsTy::RegisterRequires(int64_t flags) {
210   // TODO: add more elaborate check.
211   // Minimal check: only set requires flags if previous value
212   // is undefined. This ensures that only the first call to this
213   // function will set the requires flags. All subsequent calls
214   // will be checked for compatibility.
215   assert(flags != OMP_REQ_UNDEFINED &&
216          "illegal undefined flag for requires directive!");
217   if (RequiresFlags == OMP_REQ_UNDEFINED) {
218     RequiresFlags = flags;
219     return;
220   }
221 
222   // If multiple compilation units are present enforce
223   // consistency across all of them for require clauses:
224   //  - reverse_offload
225   //  - unified_address
226   //  - unified_shared_memory
227   if ((RequiresFlags & OMP_REQ_REVERSE_OFFLOAD) !=
228       (flags & OMP_REQ_REVERSE_OFFLOAD)) {
229     FATAL_MESSAGE0(1,
230         "'#pragma omp requires reverse_offload' not used consistently!");
231   }
232   if ((RequiresFlags & OMP_REQ_UNIFIED_ADDRESS) !=
233           (flags & OMP_REQ_UNIFIED_ADDRESS)) {
234     FATAL_MESSAGE0(1,
235         "'#pragma omp requires unified_address' not used consistently!");
236   }
237   if ((RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY) !=
238           (flags & OMP_REQ_UNIFIED_SHARED_MEMORY)) {
239     FATAL_MESSAGE0(1,
240         "'#pragma omp requires unified_shared_memory' not used consistently!");
241   }
242 
243   // TODO: insert any other missing checks
244 
245   DP("New requires flags %" PRId64 " compatible with existing %" PRId64 "!\n",
246      flags, RequiresFlags);
247 }
248 
RegisterLib(__tgt_bin_desc * desc)249 void RTLsTy::RegisterLib(__tgt_bin_desc *desc) {
250   // Attempt to load all plugins available in the system.
251   std::call_once(initFlag, &RTLsTy::LoadRTLs, this);
252 
253   PM->RTLsMtx.lock();
254   // Register the images with the RTLs that understand them, if any.
255   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
256     // Obtain the image.
257     __tgt_device_image *img = &desc->DeviceImages[i];
258 
259     RTLInfoTy *FoundRTL = NULL;
260 
261     // Scan the RTLs that have associated images until we find one that supports
262     // the current image.
263     for (auto &R : AllRTLs) {
264       if (!R.is_valid_binary(img)) {
265         DP("Image " DPxMOD " is NOT compatible with RTL %s!\n",
266             DPxPTR(img->ImageStart), R.RTLName.c_str());
267         continue;
268       }
269 
270       DP("Image " DPxMOD " is compatible with RTL %s!\n",
271           DPxPTR(img->ImageStart), R.RTLName.c_str());
272 
273       // If this RTL is not already in use, initialize it.
274       if (!R.isUsed) {
275         // Initialize the device information for the RTL we are about to use.
276         DeviceTy device(&R);
277         size_t Start = PM->Devices.size();
278         PM->Devices.resize(Start + R.NumberOfDevices, device);
279         for (int32_t device_id = 0; device_id < R.NumberOfDevices;
280             device_id++) {
281           // global device ID
282           PM->Devices[Start + device_id].DeviceID = Start + device_id;
283           // RTL local device ID
284           PM->Devices[Start + device_id].RTLDeviceID = device_id;
285         }
286 
287         // Initialize the index of this RTL and save it in the used RTLs.
288         R.Idx = (UsedRTLs.empty())
289                     ? 0
290                     : UsedRTLs.back()->Idx + UsedRTLs.back()->NumberOfDevices;
291         assert((size_t) R.Idx == Start &&
292             "RTL index should equal the number of devices used so far.");
293         R.isUsed = true;
294         UsedRTLs.push_back(&R);
295 
296         DP("RTL " DPxMOD " has index %d!\n", DPxPTR(R.LibraryHandler), R.Idx);
297       }
298 
299       // Initialize (if necessary) translation table for this library.
300       PM->TrlTblMtx.lock();
301       if (!PM->HostEntriesBeginToTransTable.count(desc->HostEntriesBegin)) {
302         TranslationTable &TransTable =
303             (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
304         TransTable.HostTable.EntriesBegin = desc->HostEntriesBegin;
305         TransTable.HostTable.EntriesEnd = desc->HostEntriesEnd;
306       }
307 
308       // Retrieve translation table for this library.
309       TranslationTable &TransTable =
310           (PM->HostEntriesBeginToTransTable)[desc->HostEntriesBegin];
311 
312       DP("Registering image " DPxMOD " with RTL %s!\n",
313           DPxPTR(img->ImageStart), R.RTLName.c_str());
314       RegisterImageIntoTranslationTable(TransTable, R, img);
315       PM->TrlTblMtx.unlock();
316       FoundRTL = &R;
317 
318       // Load ctors/dtors for static objects
319       RegisterGlobalCtorsDtorsForImage(desc, img, FoundRTL);
320 
321       // if an RTL was found we are done - proceed to register the next image
322       break;
323     }
324 
325     if (!FoundRTL) {
326       DP("No RTL found for image " DPxMOD "!\n", DPxPTR(img->ImageStart));
327     }
328   }
329   PM->RTLsMtx.unlock();
330 
331   DP("Done registering entries!\n");
332 }
333 
UnregisterLib(__tgt_bin_desc * desc)334 void RTLsTy::UnregisterLib(__tgt_bin_desc *desc) {
335   DP("Unloading target library!\n");
336 
337   PM->RTLsMtx.lock();
338   // Find which RTL understands each image, if any.
339   for (int32_t i = 0; i < desc->NumDeviceImages; ++i) {
340     // Obtain the image.
341     __tgt_device_image *img = &desc->DeviceImages[i];
342 
343     RTLInfoTy *FoundRTL = NULL;
344 
345     // Scan the RTLs that have associated images until we find one that supports
346     // the current image. We only need to scan RTLs that are already being used.
347     for (auto *R : UsedRTLs) {
348 
349       assert(R->isUsed && "Expecting used RTLs.");
350 
351       if (!R->is_valid_binary(img)) {
352         DP("Image " DPxMOD " is NOT compatible with RTL " DPxMOD "!\n",
353             DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
354         continue;
355       }
356 
357       DP("Image " DPxMOD " is compatible with RTL " DPxMOD "!\n",
358           DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
359 
360       FoundRTL = R;
361 
362       // Execute dtors for static objects if the device has been used, i.e.
363       // if its PendingCtors list has been emptied.
364       for (int32_t i = 0; i < FoundRTL->NumberOfDevices; ++i) {
365         DeviceTy &Device = PM->Devices[FoundRTL->Idx + i];
366         Device.PendingGlobalsMtx.lock();
367         if (Device.PendingCtorsDtors[desc].PendingCtors.empty()) {
368           for (auto &dtor : Device.PendingCtorsDtors[desc].PendingDtors) {
369             int rc = target(Device.DeviceID, dtor, 0, nullptr, nullptr, nullptr,
370                             nullptr, nullptr, nullptr, 1, 1, true /*team*/);
371             if (rc != OFFLOAD_SUCCESS) {
372               DP("Running destructor " DPxMOD " failed.\n", DPxPTR(dtor));
373             }
374           }
375           // Remove this library's entry from PendingCtorsDtors
376           Device.PendingCtorsDtors.erase(desc);
377         }
378         Device.PendingGlobalsMtx.unlock();
379       }
380 
381       DP("Unregistered image " DPxMOD " from RTL " DPxMOD "!\n",
382           DPxPTR(img->ImageStart), DPxPTR(R->LibraryHandler));
383 
384       break;
385     }
386 
387     // if no RTL was found proceed to unregister the next image
388     if (!FoundRTL){
389       DP("No RTLs in use support the image " DPxMOD "!\n",
390           DPxPTR(img->ImageStart));
391     }
392   }
393   PM->RTLsMtx.unlock();
394   DP("Done unregistering images!\n");
395 
396   // Remove entries from PM->HostPtrToTableMap
397   PM->TblMapMtx.lock();
398   for (__tgt_offload_entry *cur = desc->HostEntriesBegin;
399       cur < desc->HostEntriesEnd; ++cur) {
400     PM->HostPtrToTableMap.erase(cur->addr);
401   }
402 
403   // Remove translation table for this descriptor.
404   auto TransTable = PM->HostEntriesBeginToTransTable.find(desc->HostEntriesBegin);
405   if (TransTable != PM->HostEntriesBeginToTransTable.end()) {
406     DP("Removing translation table for descriptor " DPxMOD "\n",
407         DPxPTR(desc->HostEntriesBegin));
408     PM->HostEntriesBeginToTransTable.erase(TransTable);
409   } else {
410     DP("Translation table for descriptor " DPxMOD " cannot be found, probably "
411         "it has been already removed.\n", DPxPTR(desc->HostEntriesBegin));
412   }
413 
414   PM->TblMapMtx.unlock();
415 
416   // TODO: Remove RTL and the devices it manages if it's not used anymore?
417   // TODO: Write some RTL->unload_image(...) function?
418 
419   DP("Done unregistering library!\n");
420 }
421