1import os 2import platform 3import re 4import subprocess 5import sys 6 7import lit.util 8from lit.llvm.subst import FindTool 9from lit.llvm.subst import ToolSubst 10 11 12def binary_feature(on, feature, off_prefix): 13 return feature if on else off_prefix + feature 14 15 16class LLVMConfig(object): 17 18 def __init__(self, lit_config, config): 19 self.lit_config = lit_config 20 self.config = config 21 22 features = config.available_features 23 24 self.use_lit_shell = False 25 # Tweak PATH for Win32 to decide to use bash.exe or not. 26 if sys.platform == 'win32': 27 # For tests that require Windows to run. 28 features.add('system-windows') 29 30 # Seek sane tools in directories and set to $PATH. 31 path = self.lit_config.getToolsPath(config.lit_tools_dir, 32 config.environment['PATH'], 33 ['cmp.exe', 'grep.exe', 'sed.exe']) 34 if path is not None: 35 self.with_environment('PATH', path, append_path=True) 36 self.use_lit_shell = True 37 38 # Choose between lit's internal shell pipeline runner and a real shell. If 39 # LIT_USE_INTERNAL_SHELL is in the environment, we use that as an override. 40 lit_shell_env = os.environ.get('LIT_USE_INTERNAL_SHELL') 41 if lit_shell_env: 42 self.use_lit_shell = lit.util.pythonize_bool(lit_shell_env) 43 44 if not self.use_lit_shell: 45 features.add('shell') 46 47 # Running on Darwin OS 48 if platform.system() == 'Darwin': 49 # FIXME: lld uses the first, other projects use the second. 50 # We should standardize on the former. 51 features.add('system-linker-mach-o') 52 features.add('system-darwin') 53 elif platform.system() == 'Windows': 54 # For tests that require Windows to run. 55 features.add('system-windows') 56 elif platform.system() == "Linux": 57 features.add('system-linux') 58 59 # Native compilation: host arch == default triple arch 60 # Both of these values should probably be in every site config (e.g. as 61 # part of the standard header. But currently they aren't) 62 host_triple = getattr(config, 'host_triple', None) 63 target_triple = getattr(config, 'target_triple', None) 64 if host_triple and host_triple == target_triple: 65 features.add('native') 66 67 # Sanitizers. 68 sanitizers = getattr(config, 'llvm_use_sanitizer', '') 69 sanitizers = frozenset(x.lower() for x in sanitizers.split(';')) 70 features.add(binary_feature('address' in sanitizers, 'asan', 'not_')) 71 features.add(binary_feature('memory' in sanitizers, 'msan', 'not_')) 72 features.add(binary_feature( 73 'undefined' in sanitizers, 'ubsan', 'not_')) 74 75 have_zlib = getattr(config, 'have_zlib', None) 76 features.add(binary_feature(have_zlib, 'zlib', 'no')) 77 78 # Check if we should run long running tests. 79 long_tests = lit_config.params.get('run_long_tests', None) 80 if lit.util.pythonize_bool(long_tests): 81 features.add('long_tests') 82 83 if target_triple: 84 if re.match(r'^x86_64.*-apple', target_triple): 85 host_cxx = getattr(config, 'host_cxx', None) 86 if 'address' in sanitizers and self.get_clang_has_lsan(host_cxx, target_triple): 87 self.with_environment( 88 'ASAN_OPTIONS', 'detect_leaks=1', append_path=True) 89 if re.match(r'^x86_64.*-linux', target_triple): 90 features.add('x86_64-linux') 91 if re.match(r'.*-win32$', target_triple): 92 features.add('target-windows') 93 94 use_gmalloc = lit_config.params.get('use_gmalloc', None) 95 if lit.util.pythonize_bool(use_gmalloc): 96 # Allow use of an explicit path for gmalloc library. 97 # Will default to '/usr/lib/libgmalloc.dylib' if not set. 98 gmalloc_path_str = lit_config.params.get('gmalloc_path', 99 '/usr/lib/libgmalloc.dylib') 100 if gmalloc_path_str is not None: 101 self.with_environment( 102 'DYLD_INSERT_LIBRARIES', gmalloc_path_str) 103 104 def with_environment(self, variable, value, append_path=False): 105 if append_path: 106 # For paths, we should be able to take a list of them and process all 107 # of them. 108 paths_to_add = value 109 if lit.util.is_string(paths_to_add): 110 paths_to_add = [paths_to_add] 111 112 def norm(x): 113 return os.path.normcase(os.path.normpath(x)) 114 115 current_paths = self.config.environment.get(variable, None) 116 if current_paths: 117 current_paths = current_paths.split(os.path.pathsep) 118 paths = [norm(p) for p in current_paths] 119 else: 120 paths = [] 121 122 # If we are passed a list [a b c], then iterating this list forwards 123 # and adding each to the beginning would result in b c a. So we 124 # need to iterate in reverse to end up with the original ordering. 125 for p in reversed(paths_to_add): 126 # Move it to the front if it already exists, otherwise insert it at the 127 # beginning. 128 p = norm(p) 129 try: 130 paths.remove(p) 131 except ValueError: 132 pass 133 paths = [p] + paths 134 value = os.pathsep.join(paths) 135 self.config.environment[variable] = value 136 137 def with_system_environment(self, variables, append_path=False): 138 if lit.util.is_string(variables): 139 variables = [variables] 140 for v in variables: 141 value = os.environ.get(v) 142 if value: 143 self.with_environment(v, value, append_path) 144 145 def clear_environment(self, variables): 146 for name in variables: 147 if name in self.config.environment: 148 del self.config.environment[name] 149 150 def get_process_output(self, command): 151 try: 152 cmd = subprocess.Popen( 153 command, stdout=subprocess.PIPE, 154 stderr=subprocess.PIPE, env=self.config.environment) 155 stdout, stderr = cmd.communicate() 156 stdout = lit.util.to_string(stdout) 157 stderr = lit.util.to_string(stderr) 158 return (stdout, stderr) 159 except OSError: 160 self.lit_config.fatal('Could not run process %s' % command) 161 162 def feature_config(self, features): 163 # Ask llvm-config about the specified feature. 164 arguments = [x for (x, _) in features] 165 config_path = os.path.join(self.config.llvm_tools_dir, 'llvm-config') 166 167 output, _ = self.get_process_output([config_path] + arguments) 168 lines = output.split('\n') 169 170 for (feature_line, (_, patterns)) in zip(lines, features): 171 # We should have either a callable or a dictionary. If it's a 172 # dictionary, grep each key against the output and use the value if 173 # it matches. If it's a callable, it does the entire translation. 174 if callable(patterns): 175 features_to_add = patterns(feature_line) 176 self.config.available_features.update(features_to_add) 177 else: 178 for (re_pattern, feature) in patterns.items(): 179 if re.search(re_pattern, feature_line): 180 self.config.available_features.add(feature) 181 182 # Note that when substituting %clang_cc1 also fill in the include directory of 183 # the builtin headers. Those are part of even a freestanding environment, but 184 # Clang relies on the driver to locate them. 185 def get_clang_builtin_include_dir(self, clang): 186 # FIXME: Rather than just getting the version, we should have clang print 187 # out its resource dir here in an easy to scrape form. 188 clang_dir, _ = self.get_process_output( 189 [clang, '-print-file-name=include']) 190 191 if not clang_dir: 192 self.lit_config.fatal( 193 "Couldn't find the include dir for Clang ('%s')" % clang) 194 195 clang_dir = clang_dir.strip() 196 if sys.platform in ['win32'] and not self.use_lit_shell: 197 # Don't pass dosish path separator to msys bash.exe. 198 clang_dir = clang_dir.replace('\\', '/') 199 # Ensure the result is an ascii string, across Python2.5+ - Python3. 200 return clang_dir 201 202 # On macOS, LSan is only supported on clang versions 5 and higher 203 def get_clang_has_lsan(self, clang, triple): 204 if not clang: 205 self.lit_config.warning( 206 'config.host_cxx is unset but test suite is configured to use sanitizers.') 207 return False 208 209 clang_binary = clang.split()[0] 210 version_string, _ = self.get_process_output( 211 [clang_binary, '--version']) 212 if not 'clang' in version_string: 213 self.lit_config.warning( 214 "compiler '%s' does not appear to be clang, " % clang_binary + 215 'but test suite is configured to use sanitizers.') 216 return False 217 218 if re.match(r'.*-linux', triple): 219 return True 220 221 if re.match(r'^x86_64.*-apple', triple): 222 version_regex = re.search(r'version ([0-9]+)\.([0-9]+).([0-9]+)', version_string) 223 major_version_number = int(version_regex.group(1)) 224 minor_version_number = int(version_regex.group(2)) 225 patch_version_number = int(version_regex.group(3)) 226 if 'Apple LLVM' in version_string: 227 # Apple LLVM doesn't yet support LSan 228 return False 229 else: 230 return major_version_number >= 5 231 232 return False 233 234 def make_itanium_abi_triple(self, triple): 235 m = re.match(r'(\w+)-(\w+)-(\w+)', triple) 236 if not m: 237 self.lit_config.fatal( 238 "Could not turn '%s' into Itanium ABI triple" % triple) 239 if m.group(3).lower() != 'win32': 240 # All non-win32 triples use the Itanium ABI. 241 return triple 242 return m.group(1) + '-' + m.group(2) + '-mingw32' 243 244 def make_msabi_triple(self, triple): 245 m = re.match(r'(\w+)-(\w+)-(\w+)', triple) 246 if not m: 247 self.lit_config.fatal( 248 "Could not turn '%s' into MS ABI triple" % triple) 249 isa = m.group(1).lower() 250 vendor = m.group(2).lower() 251 os = m.group(3).lower() 252 if os == 'win32': 253 # If the OS is win32, we're done. 254 return triple 255 if isa.startswith('x86') or isa == 'amd64' or re.match(r'i\d86', isa): 256 # For x86 ISAs, adjust the OS. 257 return isa + '-' + vendor + '-win32' 258 # -win32 is not supported for non-x86 targets; use a default. 259 return 'i686-pc-win32' 260 261 def add_tool_substitutions(self, tools, search_dirs=None): 262 if not search_dirs: 263 search_dirs = [self.config.llvm_tools_dir] 264 265 if lit.util.is_string(search_dirs): 266 search_dirs = [search_dirs] 267 268 tools = [x if isinstance(x, ToolSubst) else ToolSubst(x) 269 for x in tools] 270 271 search_dirs = os.pathsep.join(search_dirs) 272 substitutions = [] 273 274 for tool in tools: 275 match = tool.resolve(self, search_dirs) 276 277 # Either no match occurred, or there was an unresolved match that 278 # is ignored. 279 if not match: 280 continue 281 282 subst_key, tool_pipe, command = match 283 284 # An unresolved match occurred that can't be ignored. Fail without 285 # adding any of the previously-discovered substitutions. 286 if not command: 287 return False 288 289 substitutions.append((subst_key, tool_pipe + command)) 290 291 self.config.substitutions.extend(substitutions) 292 return True 293 294 def use_default_substitutions(self): 295 tool_patterns = [ 296 ToolSubst('FileCheck', unresolved='fatal'), 297 # Handle these specially as they are strings searched for during testing. 298 ToolSubst(r'\| \bcount\b', command=FindTool( 299 'count'), verbatim=True, unresolved='fatal'), 300 ToolSubst(r'\| \bnot\b', command=FindTool('not'), verbatim=True, unresolved='fatal')] 301 302 self.config.substitutions.append(('%python', '"%s"' % (sys.executable))) 303 304 self.add_tool_substitutions( 305 tool_patterns, [self.config.llvm_tools_dir]) 306 307 def use_llvm_tool(self, name, search_env=None, required=False, quiet=False): 308 """Find the executable program 'name', optionally using the specified 309 environment variable as an override before searching the 310 configuration's PATH.""" 311 # If the override is specified in the environment, use it without 312 # validation. 313 if search_env: 314 tool = self.config.environment.get(search_env) 315 if tool: 316 return tool 317 318 # Otherwise look in the path. 319 tool = lit.util.which(name, self.config.environment['PATH']) 320 321 if required and not tool: 322 message = "couldn't find '{}' program".format(name) 323 if search_env: 324 message = message + \ 325 ', try setting {} in your environment'.format(search_env) 326 self.lit_config.fatal(message) 327 328 if tool: 329 tool = os.path.normpath(tool) 330 if not self.lit_config.quiet and not quiet: 331 self.lit_config.note('using {}: {}'.format(name, tool)) 332 return tool 333 334 def use_clang(self, required=True): 335 """Configure the test suite to be able to invoke clang. 336 337 Sets up some environment variables important to clang, locates a 338 just-built or installed clang, and add a set of standard 339 substitutions useful to any test suite that makes use of clang. 340 341 """ 342 # Clear some environment variables that might affect Clang. 343 # 344 # This first set of vars are read by Clang, but shouldn't affect tests 345 # that aren't specifically looking for these features, or are required 346 # simply to run the tests at all. 347 # 348 # FIXME: Should we have a tool that enforces this? 349 350 # safe_env_vars = ('TMPDIR', 'TEMP', 'TMP', 'USERPROFILE', 'PWD', 351 # 'MACOSX_DEPLOYMENT_TARGET', 'IPHONEOS_DEPLOYMENT_TARGET', 352 # 'VCINSTALLDIR', 'VC100COMNTOOLS', 'VC90COMNTOOLS', 353 # 'VC80COMNTOOLS') 354 possibly_dangerous_env_vars = ['COMPILER_PATH', 'RC_DEBUG_OPTIONS', 355 'CINDEXTEST_PREAMBLE_FILE', 'LIBRARY_PATH', 356 'CPATH', 'C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 357 'OBJC_INCLUDE_PATH', 'OBJCPLUS_INCLUDE_PATH', 358 'LIBCLANG_TIMING', 'LIBCLANG_OBJTRACKING', 359 'LIBCLANG_LOGGING', 'LIBCLANG_BGPRIO_INDEX', 360 'LIBCLANG_BGPRIO_EDIT', 'LIBCLANG_NOTHREADS', 361 'LIBCLANG_RESOURCE_USAGE', 362 'LIBCLANG_CODE_COMPLETION_LOGGING'] 363 # Clang/Win32 may refer to %INCLUDE%. vsvarsall.bat sets it. 364 if platform.system() != 'Windows': 365 possibly_dangerous_env_vars.append('INCLUDE') 366 367 self.clear_environment(possibly_dangerous_env_vars) 368 369 # Tweak the PATH to include the tools dir and the scripts dir. 370 # Put Clang first to avoid LLVM from overriding out-of-tree clang builds. 371 possible_paths = ['clang_tools_dir', 'llvm_tools_dir'] 372 paths = [getattr(self.config, pp) for pp in possible_paths 373 if getattr(self.config, pp, None)] 374 self.with_environment('PATH', paths, append_path=True) 375 376 paths = [self.config.llvm_shlib_dir, self.config.llvm_libs_dir] 377 self.with_environment('LD_LIBRARY_PATH', paths, append_path=True) 378 379 # Discover the 'clang' and 'clangcc' to use. 380 381 self.config.clang = self.use_llvm_tool( 382 'clang', search_env='CLANG', required=required) 383 384 self.config.substitutions.append( 385 ('%llvmshlibdir', self.config.llvm_shlib_dir)) 386 self.config.substitutions.append( 387 ('%pluginext', self.config.llvm_plugin_ext)) 388 389 builtin_include_dir = self.get_clang_builtin_include_dir(self.config.clang) 390 tool_substitutions = [ 391 ToolSubst('%clang', command=self.config.clang), 392 ToolSubst('%clang_analyze_cc1', command='%clang_cc1', extra_args=['-analyze', '%analyze']), 393 ToolSubst('%clang_cc1', command=self.config.clang, extra_args=['-cc1', '-internal-isystem', builtin_include_dir, '-nostdsysteminc']), 394 ToolSubst('%clang_cpp', command=self.config.clang, extra_args=['--driver-mode=cpp']), 395 ToolSubst('%clang_cl', command=self.config.clang, extra_args=['--driver-mode=cl']), 396 ToolSubst('%clangxx', command=self.config.clang, extra_args=['--driver-mode=g++']), 397 ] 398 self.add_tool_substitutions(tool_substitutions) 399 400 self.config.substitutions.append(('%itanium_abi_triple', 401 self.make_itanium_abi_triple(self.config.target_triple))) 402 self.config.substitutions.append(('%ms_abi_triple', 403 self.make_msabi_triple(self.config.target_triple))) 404 self.config.substitutions.append( 405 ('%resource_dir', builtin_include_dir)) 406 407 # The host triple might not be set, at least if we're compiling clang from 408 # an already installed llvm. 409 if self.config.host_triple and self.config.host_triple != '@LLVM_HOST_TRIPLE@': 410 self.config.substitutions.append(('%target_itanium_abi_host_triple', 411 '--target=%s' % self.make_itanium_abi_triple(self.config.host_triple))) 412 else: 413 self.config.substitutions.append( 414 ('%target_itanium_abi_host_triple', '')) 415 416 self.config.substitutions.append( 417 ('%src_include_dir', self.config.clang_src_dir + '/include')) 418 419 # FIXME: Find nicer way to prohibit this. 420 self.config.substitutions.append( 421 (' clang ', """*** Do not use 'clang' in tests, use '%clang'. ***""")) 422 self.config.substitutions.append( 423 (' clang\+\+ ', """*** Do not use 'clang++' in tests, use '%clangxx'. ***""")) 424 self.config.substitutions.append( 425 (' clang-cc ', 426 """*** Do not use 'clang-cc' in tests, use '%clang_cc1'. ***""")) 427 self.config.substitutions.append( 428 (' clang -cc1 -analyze ', 429 """*** Do not use 'clang -cc1 -analyze' in tests, use '%clang_analyze_cc1'. ***""")) 430 self.config.substitutions.append( 431 (' clang -cc1 ', 432 """*** Do not use 'clang -cc1' in tests, use '%clang_cc1'. ***""")) 433 self.config.substitutions.append( 434 (' %clang-cc1 ', 435 """*** invalid substitution, use '%clang_cc1'. ***""")) 436 self.config.substitutions.append( 437 (' %clang-cpp ', 438 """*** invalid substitution, use '%clang_cpp'. ***""")) 439 self.config.substitutions.append( 440 (' %clang-cl ', 441 """*** invalid substitution, use '%clang_cl'. ***""")) 442 443 def use_lld(self, required=True): 444 """Configure the test suite to be able to invoke lld. 445 446 Sets up some environment variables important to lld, locates a 447 just-built or installed lld, and add a set of standard 448 substitutions useful to any test suite that makes use of lld. 449 450 """ 451 # Tweak the PATH to include the tools dir 452 tool_dirs = [self.config.llvm_tools_dir] 453 lib_dirs = [self.config.llvm_libs_dir] 454 lld_tools_dir = getattr(self.config, 'lld_tools_dir', None) 455 lld_libs_dir = getattr(self.config, 'lld_libs_dir', None) 456 457 if lld_tools_dir: 458 tool_dirs = tool_dirs + [lld_tools_dir] 459 if lld_libs_dir: 460 lib_dirs = lib_dirs + [lld_libs_dir] 461 462 self.with_environment('PATH', tool_dirs, append_path=True) 463 self.with_environment('LD_LIBRARY_PATH', lib_dirs, append_path=True) 464 465 tool_patterns = ['lld', 'ld.lld', 'lld-link', 'ld64.lld', 'wasm-ld'] 466 467 self.add_tool_substitutions(tool_patterns, tool_dirs) 468