1import os 2import platform 3import re 4import subprocess 5import sys 6 7import lit.util 8from lit.llvm.subst import FindTool 9from lit.llvm.subst import ToolSubst 10 11 12class LLVMConfig(object): 13 14 def __init__(self, lit_config, config): 15 self.lit_config = lit_config 16 self.config = config 17 18 features = config.available_features 19 20 self.use_lit_shell = False 21 # Tweak PATH for Win32 to decide to use bash.exe or not. 22 if sys.platform == 'win32': 23 # For tests that require Windows to run. 24 features.add('system-windows') 25 26 # Seek sane tools in directories and set to $PATH. 27 path = self.lit_config.getToolsPath(config.lit_tools_dir, 28 config.environment['PATH'], 29 ['cmp.exe', 'grep.exe', 'sed.exe']) 30 if path is not None: 31 self.with_environment('PATH', path, append_path=True) 32 # Many tools behave strangely if these environment variables aren't set. 33 self.with_system_environment(['SystemDrive', 'SystemRoot', 'TEMP', 'TMP']) 34 self.use_lit_shell = True 35 36 # Choose between lit's internal shell pipeline runner and a real shell. If 37 # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override. 38 lit_shell_env = os.environ.get('LIT_USE_INTERNAL_SHELL') 39 if lit_shell_env: 40 self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env) 41 42 if not self.use_lit_shell: 43 features.add('shell') 44 45 # Running on Darwin OS 46 if platform.system() == 'Darwin': 47 # FIXME: lld uses the first, other projects use the second. 48 # We should standardize on the former. 49 features.add('system-linker-mach-o') 50 features.add('system-darwin') 51 elif platform.system() == 'Windows': 52 # For tests that require Windows to run. 53 features.add('system-windows') 54 elif platform.system() == 'Linux': 55 features.add('system-linux') 56 elif platform.system() in ['FreeBSD']: 57 features.add('system-freebsd') 58 elif platform.system() == 'NetBSD': 59 features.add('system-netbsd') 60 elif platform.system() == 'AIX': 61 features.add('system-aix') 62 elif platform.system() == 'SunOS': 63 features.add('system-solaris') 64 65 # Native compilation: host arch == default triple arch 66 # Both of these values should probably be in every site config (e.g. as 67 # part of the standard header. But currently they aren't) 68 host_triple = getattr(config, 'host_triple', None) 69 target_triple = getattr(config, 'target_triple', None) 70 if host_triple and host_triple == target_triple: 71 features.add('native') 72 73 # Sanitizers. 74 sanitizers = getattr(config, 'llvm_use_sanitizer', '') 75 sanitizers = frozenset(x.lower() for x in sanitizers.split(';')) 76 if 'address' in sanitizers: 77 features.add('asan') 78 if 'memory' in sanitizers or 'memorywithorigins' in sanitizers: 79 features.add('msan') 80 if 'undefined' in sanitizers: 81 features.add('ubsan') 82 83 have_zlib = getattr(config, 'have_zlib', None) 84 if have_zlib: 85 features.add('zlib') 86 87 # Check if we should run long running tests. 88 long_tests = lit_config.params.get('run_long_tests', None) 89 if lit.util.pythonize_bool(long_tests): 90 features.add('long_tests') 91 92 if target_triple: 93 if re.match(r'^x86_64.*-apple', target_triple): 94 features.add('x86_64-apple') 95 host_cxx = getattr(config, 'host_cxx', None) 96 if 'address' in sanitizers and self.get_clang_has_lsan(host_cxx, target_triple): 97 self.with_environment( 98 'ASAN_OPTIONS', 'detect_leaks=1', append_path=True) 99 if re.match(r'^x86_64.*-linux', target_triple): 100 features.add('x86_64-linux') 101 if re.match(r'^i.86.*', target_triple): 102 features.add('target-x86') 103 elif re.match(r'^x86_64.*', target_triple): 104 features.add('target-x86_64') 105 elif re.match(r'^aarch64.*', target_triple): 106 features.add('target-aarch64') 107 elif re.match(r'^arm.*', target_triple): 108 features.add('target-arm') 109 110 use_gmalloc = lit_config.params.get('use_gmalloc', None) 111 if lit.util.pythonize_bool(use_gmalloc): 112 # Allow use of an explicit path for gmalloc library. 113 # Will default to '/usr/lib/libgmalloc.dylib' if not set. 114 gmalloc_path_str = lit_config.params.get('gmalloc_path', 115 '/usr/lib/libgmalloc.dylib') 116 if gmalloc_path_str is not None: 117 self.with_environment( 118 'DYLD_INSERT_LIBRARIES', gmalloc_path_str) 119 120 def with_environment(self, variable, value, append_path=False): 121 if append_path: 122 # For paths, we should be able to take a list of them and process all 123 # of them. 124 paths_to_add = value 125 if lit.util.is_string(paths_to_add): 126 paths_to_add = [paths_to_add] 127 128 def norm(x): 129 return os.path.normcase(os.path.normpath(x)) 130 131 current_paths = self.config.environment.get(variable, None) 132 if current_paths: 133 current_paths = current_paths.split(os.path.pathsep) 134 paths = [norm(p) for p in current_paths] 135 else: 136 paths = [] 137 138 # If we are passed a list [a b c], then iterating this list forwards 139 # and adding each to the beginning would result in b c a. So we 140 # need to iterate in reverse to end up with the original ordering. 141 for p in reversed(paths_to_add): 142 # Move it to the front if it already exists, otherwise insert it at the 143 # beginning. 144 p = norm(p) 145 try: 146 paths.remove(p) 147 except ValueError: 148 pass 149 paths = [p] + paths 150 value = os.pathsep.join(paths) 151 self.config.environment[variable] = value 152 153 def with_system_environment(self, variables, append_path=False): 154 if lit.util.is_string(variables): 155 variables = [variables] 156 for v in variables: 157 value = os.environ.get(v) 158 if value: 159 self.with_environment(v, value, append_path) 160 161 def clear_environment(self, variables): 162 for name in variables: 163 if name in self.config.environment: 164 del self.config.environment[name] 165 166 def get_process_output(self, command): 167 try: 168 cmd = subprocess.Popen( 169 command, stdout=subprocess.PIPE, 170 stderr=subprocess.PIPE, env=self.config.environment) 171 stdout, stderr = cmd.communicate() 172 stdout = lit.util.to_string(stdout) 173 stderr = lit.util.to_string(stderr) 174 return (stdout, stderr) 175 except OSError: 176 self.lit_config.fatal('Could not run process %s' % command) 177 178 def feature_config(self, features): 179 # Ask llvm-config about the specified feature. 180 arguments = [x for (x, _) in features] 181 config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config') 182 183 output, _ = self.get_process_output([config_path] + arguments) 184 lines = output.split('\n') 185 186 for (feature_line, (_, patterns)) in zip(lines, features): 187 # We should have either a callable or a dictionary. If it's a 188 # dictionary, grep each key against the output and use the value if 189 # it matches. If it's a callable, it does the entire translation. 190 if callable(patterns): 191 features_to_add = patterns(feature_line) 192 self.config.available_features.update(features_to_add) 193 else: 194 for (re_pattern, feature) in patterns.items(): 195 if re.search(re_pattern, feature_line): 196 self.config.available_features.add(feature) 197 198 # Note that when substituting %clang_cc1 also fill in the include directory of 199 # the builtin headers. Those are part of even a freestanding environment, but 200 # Clang relies on the driver to locate them. 201 def get_clang_builtin_include_dir(self, clang): 202 # FIXME: Rather than just getting the version, we should have clang print 203 # out its resource dir here in an easy to scrape form. 204 clang_dir, _ = self.get_process_output( 205 [clang, '-print-file-name=include']) 206 207 if not clang_dir: 208 self.lit_config.fatal( 209 "Couldn't find the include dir for Clang ('%s')" % clang) 210 211 clang_dir = clang_dir.strip() 212 if sys.platform in ['win32'] and not self.use_lit_shell: 213 # Don't pass dosish path separator to msys bash.exe. 214 clang_dir = clang_dir.replace('\\', '/') 215 # Ensure the result is an ascii string, across Python2.5+ - Python3. 216 return clang_dir 217 218 # On macOS, LSan is only supported on clang versions 5 and higher 219 def get_clang_has_lsan(self, clang, triple): 220 if not clang: 221 self.lit_config.warning( 222 'config.host_cxx is unset but test suite is configured to use sanitizers.') 223 return False 224 225 clang_binary = clang.split()[0] 226 version_string, _ = self.get_process_output( 227 [clang_binary, '--version']) 228 if not 'clang' in version_string: 229 self.lit_config.warning( 230 "compiler '%s' does not appear to be clang, " % clang_binary + 231 'but test suite is configured to use sanitizers.') 232 return False 233 234 if re.match(r'.*-linux', triple): 235 return True 236 237 if re.match(r'^x86_64.*-apple', triple): 238 version_regex = re.search(r'version ([0-9]+)\.([0-9]+).([0-9]+)', version_string) 239 major_version_number = int(version_regex.group(1)) 240 minor_version_number = int(version_regex.group(2)) 241 patch_version_number = int(version_regex.group(3)) 242 if ('Apple LLVM' in version_string) or ('Apple clang' in version_string): 243 # Apple clang doesn't yet support LSan 244 return False 245 else: 246 return major_version_number >= 5 247 248 return False 249 250 def make_itanium_abi_triple(self, triple): 251 m = re.match(r'(\w+)-(\w+)-(\w+)', triple) 252 if not m: 253 self.lit_config.fatal( 254 "Could not turn '%s' into Itanium ABI triple" % triple) 255 if m.group(3).lower() != 'windows': 256 # All non-windows triples use the Itanium ABI. 257 return triple 258 return m.group(1) + '-' + m.group(2) + '-' + m.group(3) + '-gnu' 259 260 def make_msabi_triple(self, triple): 261 m = re.match(r'(\w+)-(\w+)-(\w+)', triple) 262 if not m: 263 self.lit_config.fatal( 264 "Could not turn '%s' into MS ABI triple" % triple) 265 isa = m.group(1).lower() 266 vendor = m.group(2).lower() 267 os = m.group(3).lower() 268 if os == 'windows' and re.match(r'.*-msvc$', triple): 269 # If the OS is windows and environment is msvc, we're done. 270 return triple 271 if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa): 272 # For x86 ISAs, adjust the OS. 273 return isa + '-' + vendor + '-windows-msvc' 274 # -msvc is not supported for non-x86 targets; use a default. 275 return 'i686-pc-windows-msvc' 276 277 def add_tool_substitutions(self, tools, search_dirs=None): 278 if not search_dirs: 279 search_dirs = [self.config.llvm_tools_dir] 280 281 if lit.util.is_string(search_dirs): 282 search_dirs = [search_dirs] 283 284 tools = [x if isinstance(x, ToolSubst) else ToolSubst(x) 285 for x in tools] 286 287 search_dirs = os.pathsep.join(search_dirs) 288 substitutions = [] 289 290 for tool in tools: 291 match = tool.resolve(self, search_dirs) 292 293 # Either no match occurred, or there was an unresolved match that 294 # is ignored. 295 if not match: 296 continue 297 298 subst_key, tool_pipe, command = match 299 300 # An unresolved match occurred that can't be ignored. Fail without 301 # adding any of the previously-discovered substitutions. 302 if not command: 303 return False 304 305 substitutions.append((subst_key, tool_pipe + command)) 306 307 self.config.substitutions.extend(substitutions) 308 return True 309 310 def use_default_substitutions(self): 311 tool_patterns = [ 312 ToolSubst('FileCheck', unresolved='fatal'), 313 # Handle these specially as they are strings searched for during testing. 314 ToolSubst(r'\| \bcount\b', command=FindTool( 315 'count'), verbatim=True, unresolved='fatal'), 316 ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')] 317 318 self.config.substitutions.append(('%python', '"%s"' % (sys.executable))) 319 320 self.add_tool_substitutions( 321 tool_patterns, [self.config.llvm_tools_dir]) 322 323 def use_llvm_tool(self, name, search_env=None, required=False, quiet=False): 324 """Find the executable program 'name', optionally using the specified 325 environment variable as an override before searching the 326 configuration's PATH.""" 327 # If the override is specified in the environment, use it without 328 # validation. 329 if search_env: 330 tool = self.config.environment.get(search_env) 331 if tool: 332 return tool 333 334 # Otherwise look in the path. 335 tool = lit.util.which(name, self.config.environment['PATH']) 336 337 if required and not tool: 338 message = "couldn't find '{}' program".format(name) 339 if search_env: 340 message = message + \ 341 ', try setting {} in your environment'.format(search_env) 342 self.lit_config.fatal(message) 343 344 if tool: 345 tool = os.path.normpath(tool) 346 if not self.lit_config.quiet and not quiet: 347 self.lit_config.note('using {}: {}'.format(name, tool)) 348 return tool 349 350 def use_clang(self, additional_tool_dirs=[], additional_flags=[], required=True): 351 """Configure the test suite to be able to invoke clang. 352 353 Sets up some environment variables important to clang, locates a 354 just-built or installed clang, and add a set of standard 355 substitutions useful to any test suite that makes use of clang. 356 357 """ 358 # Clear some environment variables that might affect Clang. 359 # 360 # This first set of vars are read by Clang, but shouldn't affect tests 361 # that aren't specifically looking for these features, or are required 362 # simply to run the tests at all. 363 # 364 # FIXME: Should we have a tool that enforces this? 365 366 # safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD', 367 # 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET', 368 # 'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS', 369 # 'VC80COMNTOOLS') 370 possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS', 371 'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH', 372 'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 373 'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH', 374 'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING', 375 'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX', 376 'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS', 377 'LIBCLANG_RESOURCE_USAGE', 378 'LIBCLANG_CODE_COMPLETION_LOGGING'] 379 # Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it. 380 if platform.system() != 'Windows': 381 possibly_dangerous_env_vars.append('INCLUDE') 382 383 self.clear_environment(possibly_dangerous_env_vars) 384 385 # Tweak the PATH to include the tools dir and the scripts dir. 386 # Put Clang first to avoid LLVM from overriding out-of-tree clang builds. 387 exe_dir_props = [self.config.name.lower() + '_tools_dir', 'clang_tools_dir', 'llvm_tools_dir'] 388 paths = [getattr(self.config, pp) for pp in exe_dir_props 389 if getattr(self.config, pp, None)] 390 paths = additional_tool_dirs + paths 391 self.with_environment('PATH', paths, append_path=True) 392 393 lib_dir_props = [self.config.name.lower() + '_libs_dir', 'clang_libs_dir', 'llvm_shlib_dir', 'llvm_libs_dir'] 394 paths = [getattr(self.config, pp) for pp in lib_dir_props 395 if getattr(self.config, pp, None)] 396 397 self.with_environment('LD_LIBRARY_PATH', paths, append_path=True) 398 399 shl = getattr(self.config, 'llvm_shlib_dir', None) 400 pext = getattr(self.config, 'llvm_plugin_ext', None) 401 if shl: 402 self.config.substitutions.append(('%llvmshlibdir', shl)) 403 if pext: 404 self.config.substitutions.append(('%pluginext', pext)) 405 406 # Discover the 'clang' and 'clangcc' to use. 407 self.config.clang = self.use_llvm_tool( 408 'clang', search_env='CLANG', required=required) 409 if self.config.clang: 410 self.config.available_features.add('clang') 411 builtin_include_dir = self.get_clang_builtin_include_dir(self.config.clang) 412 tool_substitutions = [ 413 ToolSubst('%clang', command=self.config.clang, extra_args=additional_flags), 414 ToolSubst('%clang_analyze_cc1', command='%clang_cc1', extra_args=['-analyze', '%analyze', '-setup-static-analyzer']+additional_flags), 415 ToolSubst('%clang_cc1', command=self.config.clang, extra_args=['-cc1', '-internal-isystem', builtin_include_dir, '-nostdsysteminc']+additional_flags), 416 ToolSubst('%clang_cpp', command=self.config.clang, extra_args=['--driver-mode=cpp']+additional_flags), 417 ToolSubst('%clang_cl', command=self.config.clang, extra_args=['--driver-mode=cl']+additional_flags), 418 ToolSubst('%clangxx', command=self.config.clang, extra_args=['--driver-mode=g++']+additional_flags), 419 ] 420 self.add_tool_substitutions(tool_substitutions) 421 self.config.substitutions.append( 422 ('%resource_dir', builtin_include_dir)) 423 424 self.config.substitutions.append(('%itanium_abi_triple', 425 self.make_itanium_abi_triple(self.config.target_triple))) 426 self.config.substitutions.append(('%ms_abi_triple', 427 self.make_msabi_triple(self.config.target_triple))) 428 429 # The host triple might not be set, at least if we're compiling clang from 430 # an already installed llvm. 431 if self.config.host_triple and self.config.host_triple != '@LLVM_HOST_TRIPLE@': 432 self.config.substitutions.append(('%target_itanium_abi_host_triple', 433 '--target=%s' % self.make_itanium_abi_triple(self.config.host_triple))) 434 else: 435 self.config.substitutions.append( 436 ('%target_itanium_abi_host_triple', '')) 437 438 # FIXME: Find nicer way to prohibit this. 439 self.config.substitutions.append( 440 (' clang ', """\"*** Do not use 'clang' in tests, use '%clang'. ***\"""")) 441 self.config.substitutions.append( 442 (r' clang\+\+ ', """\"*** Do not use 'clang++' in tests, use '%clangxx'. ***\"""")) 443 self.config.substitutions.append( 444 (' clang-cc ', 445 """\"*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***\"""")) 446 self.config.substitutions.append( 447 (' clang-cl ', 448 """\"*** Do not use 'clang-cl' in tests, use '%clang_cl'. ***\"""")) 449 self.config.substitutions.append( 450 (' clang -cc1 -analyze ', 451 """\"*** Do not use 'clang -cc1 -analyze' in tests, use '%clang_analyze_cc1'. ***\"""")) 452 self.config.substitutions.append( 453 (' clang -cc1 ', 454 """\"*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***\"""")) 455 self.config.substitutions.append( 456 (' %clang-cc1 ', 457 """\"*** invalid substitution, use '%clang_cc1'. ***\"""")) 458 self.config.substitutions.append( 459 (' %clang-cpp ', 460 """\"*** invalid substitution, use '%clang_cpp'. ***\"""")) 461 self.config.substitutions.append( 462 (' %clang-cl ', 463 """\"*** invalid substitution, use '%clang_cl'. ***\"""")) 464 465 def use_lld(self, additional_tool_dirs=[], required=True): 466 """Configure the test suite to be able to invoke lld. 467 468 Sets up some environment variables important to lld, locates a 469 just-built or installed lld, and add a set of standard 470 substitutions useful to any test suite that makes use of lld. 471 472 """ 473 474 # Tweak the PATH to include the tools dir and the scripts dir. 475 exe_dir_props = [self.config.name.lower() + '_tools_dir', 'lld_tools_dir', 'llvm_tools_dir'] 476 paths = [getattr(self.config, pp) for pp in exe_dir_props 477 if getattr(self.config, pp, None)] 478 paths = additional_tool_dirs + paths 479 self.with_environment('PATH', paths, append_path=True) 480 481 lib_dir_props = [self.config.name.lower() + '_libs_dir', 'lld_libs_dir', 'llvm_libs_dir'] 482 paths = [getattr(self.config, pp) for pp in lib_dir_props 483 if getattr(self.config, pp, None)] 484 485 self.with_environment('LD_LIBRARY_PATH', paths, append_path=True) 486 487 # Discover the 'clang' and 'clangcc' to use. 488 489 ld_lld = self.use_llvm_tool('ld.lld', required=required) 490 lld_link = self.use_llvm_tool('lld-link', required=required) 491 ld64_lld = self.use_llvm_tool('ld64.lld', required=required) 492 wasm_ld = self.use_llvm_tool('wasm-ld', required=required) 493 494 was_found = ld_lld and lld_link and ld64_lld and wasm_ld 495 tool_substitutions = [] 496 if ld_lld: 497 tool_substitutions.append(ToolSubst(r'ld\.lld', command=ld_lld)) 498 if lld_link: 499 tool_substitutions.append(ToolSubst('lld-link', command=lld_link)) 500 if ld64_lld: 501 tool_substitutions.append(ToolSubst(r'ld64\.lld', command=ld64_lld)) 502 if wasm_ld: 503 tool_substitutions.append(ToolSubst('wasm-ld', command=wasm_ld)) 504 self.add_tool_substitutions(tool_substitutions) 505 return was_found 506