1# Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14# ============================================================================== 15"""configure script to get build parameters from user.""" 16 17from __future__ import absolute_import 18from __future__ import division 19from __future__ import print_function 20 21import argparse 22import errno 23import os 24import platform 25import re 26import subprocess 27import sys 28 29# pylint: disable=g-import-not-at-top 30try: 31 from shutil import which 32except ImportError: 33 from distutils.spawn import find_executable as which 34# pylint: enable=g-import-not-at-top 35 36_DEFAULT_CUDA_VERSION = '10.0' 37_DEFAULT_CUDNN_VERSION = '7' 38_DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,7.0' 39_DEFAULT_CUDA_PATH = '/usr/local/cuda' 40_DEFAULT_CUDA_PATH_LINUX = '/opt/cuda' 41_DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing ' 42 'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION) 43_TF_OPENCL_VERSION = '1.2' 44_DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp' 45_DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include' 46_SUPPORTED_ANDROID_NDK_VERSIONS = [10, 11, 12, 13, 14, 15, 16, 17, 18] 47 48_DEFAULT_PROMPT_ASK_ATTEMPTS = 10 49 50_TF_BAZELRC_FILENAME = '.tf_configure.bazelrc' 51_TF_WORKSPACE_ROOT = '' 52_TF_BAZELRC = '' 53_TF_CURRENT_BAZEL_VERSION = None 54 55NCCL_LIB_PATHS = [ 56 'lib64/', 'lib/powerpc64le-linux-gnu/', 'lib/x86_64-linux-gnu/', '' 57] 58 59# List of files to be configured for using Bazel on Apple platforms. 60APPLE_BAZEL_FILES = [ 61 'tensorflow/lite/experimental/objc/BUILD', 62 'tensorflow/lite/experimental/swift/BUILD' 63] 64 65if platform.machine() == 'ppc64le': 66 _DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/powerpc64le-linux-gnu/' 67else: 68 _DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/%s-linux-gnu' % platform.machine() 69 70 71class UserInputError(Exception): 72 pass 73 74 75def is_windows(): 76 return platform.system() == 'Windows' 77 78 79def is_linux(): 80 return platform.system() == 'Linux' 81 82 83def is_macos(): 84 return platform.system() == 'Darwin' 85 86 87def is_ppc64le(): 88 return platform.machine() == 'ppc64le' 89 90 91def is_cygwin(): 92 return platform.system().startswith('CYGWIN_NT') 93 94 95def get_input(question): 96 try: 97 try: 98 answer = raw_input(question) 99 except NameError: 100 answer = input(question) # pylint: disable=bad-builtin 101 except EOFError: 102 answer = '' 103 return answer 104 105 106def symlink_force(target, link_name): 107 """Force symlink, equivalent of 'ln -sf'. 108 109 Args: 110 target: items to link to. 111 link_name: name of the link. 112 """ 113 try: 114 os.symlink(target, link_name) 115 except OSError as e: 116 if e.errno == errno.EEXIST: 117 os.remove(link_name) 118 os.symlink(target, link_name) 119 else: 120 raise e 121 122 123def sed_in_place(filename, old, new): 124 """Replace old string with new string in file. 125 126 Args: 127 filename: string for filename. 128 old: string to replace. 129 new: new string to replace to. 130 """ 131 with open(filename, 'r') as f: 132 filedata = f.read() 133 newdata = filedata.replace(old, new) 134 with open(filename, 'w') as f: 135 f.write(newdata) 136 137 138def write_to_bazelrc(line): 139 with open(_TF_BAZELRC, 'a') as f: 140 f.write(line + '\n') 141 142 143def write_action_env_to_bazelrc(var_name, var): 144 write_to_bazelrc('build --action_env %s="%s"' % (var_name, str(var))) 145 146 147def run_shell(cmd, allow_non_zero=False): 148 if allow_non_zero: 149 try: 150 output = subprocess.check_output(cmd) 151 except subprocess.CalledProcessError as e: 152 output = e.output 153 else: 154 output = subprocess.check_output(cmd) 155 return output.decode('UTF-8').strip() 156 157 158def cygpath(path): 159 """Convert path from posix to windows.""" 160 return os.path.abspath(path).replace('\\', '/') 161 162 163def get_python_path(environ_cp, python_bin_path): 164 """Get the python site package paths.""" 165 python_paths = [] 166 if environ_cp.get('PYTHONPATH'): 167 python_paths = environ_cp.get('PYTHONPATH').split(':') 168 try: 169 library_paths = run_shell([ 170 python_bin_path, '-c', 171 'import site; print("\\n".join(site.getsitepackages()))' 172 ]).split('\n') 173 except subprocess.CalledProcessError: 174 library_paths = [ 175 run_shell([ 176 python_bin_path, '-c', 177 'from distutils.sysconfig import get_python_lib;' 178 'print(get_python_lib())' 179 ]) 180 ] 181 182 all_paths = set(python_paths + library_paths) 183 184 paths = [] 185 for path in all_paths: 186 if os.path.isdir(path): 187 paths.append(path) 188 return paths 189 190 191def get_python_major_version(python_bin_path): 192 """Get the python major version.""" 193 return run_shell([python_bin_path, '-c', 'import sys; print(sys.version[0])']) 194 195 196def setup_python(environ_cp): 197 """Setup python related env variables.""" 198 # Get PYTHON_BIN_PATH, default is the current running python. 199 default_python_bin_path = sys.executable 200 ask_python_bin_path = ('Please specify the location of python. [Default is ' 201 '%s]: ') % default_python_bin_path 202 while True: 203 python_bin_path = get_from_env_or_user_or_default( 204 environ_cp, 'PYTHON_BIN_PATH', ask_python_bin_path, 205 default_python_bin_path) 206 # Check if the path is valid 207 if os.path.isfile(python_bin_path) and os.access(python_bin_path, os.X_OK): 208 break 209 elif not os.path.exists(python_bin_path): 210 print('Invalid python path: %s cannot be found.' % python_bin_path) 211 else: 212 print('%s is not executable. Is it the python binary?' % python_bin_path) 213 environ_cp['PYTHON_BIN_PATH'] = '' 214 215 # Convert python path to Windows style before checking lib and version 216 if is_windows() or is_cygwin(): 217 python_bin_path = cygpath(python_bin_path) 218 219 # Get PYTHON_LIB_PATH 220 python_lib_path = environ_cp.get('PYTHON_LIB_PATH') 221 if not python_lib_path: 222 python_lib_paths = get_python_path(environ_cp, python_bin_path) 223 if environ_cp.get('USE_DEFAULT_PYTHON_LIB_PATH') == '1': 224 python_lib_path = python_lib_paths[0] 225 else: 226 print('Found possible Python library paths:\n %s' % 227 '\n '.join(python_lib_paths)) 228 default_python_lib_path = python_lib_paths[0] 229 python_lib_path = get_input( 230 'Please input the desired Python library path to use. ' 231 'Default is [%s]\n' % python_lib_paths[0]) 232 if not python_lib_path: 233 python_lib_path = default_python_lib_path 234 environ_cp['PYTHON_LIB_PATH'] = python_lib_path 235 236 _ = get_python_major_version(python_bin_path) 237 238 # Convert python path to Windows style before writing into bazel.rc 239 if is_windows() or is_cygwin(): 240 python_lib_path = cygpath(python_lib_path) 241 242 # Set-up env variables used by python_configure.bzl 243 write_action_env_to_bazelrc('PYTHON_BIN_PATH', python_bin_path) 244 write_action_env_to_bazelrc('PYTHON_LIB_PATH', python_lib_path) 245 write_to_bazelrc('build --python_path=\"%s"' % python_bin_path) 246 environ_cp['PYTHON_BIN_PATH'] = python_bin_path 247 248 # If choosen python_lib_path is from a path specified in the PYTHONPATH 249 # variable, need to tell bazel to include PYTHONPATH 250 if environ_cp.get('PYTHONPATH'): 251 python_paths = environ_cp.get('PYTHONPATH').split(':') 252 if python_lib_path in python_paths: 253 write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) 254 255 # Write tools/python_bin_path.sh 256 with open( 257 os.path.join(_TF_WORKSPACE_ROOT, 'tools', 'python_bin_path.sh'), 258 'w') as f: 259 f.write('export PYTHON_BIN_PATH="%s"' % python_bin_path) 260 261 262def reset_tf_configure_bazelrc(): 263 """Reset file that contains customized config settings.""" 264 open(_TF_BAZELRC, 'w').close() 265 266 267def cleanup_makefile(): 268 """Delete any leftover BUILD files from the Makefile build. 269 270 These files could interfere with Bazel parsing. 271 """ 272 makefile_download_dir = os.path.join(_TF_WORKSPACE_ROOT, 'tensorflow', 273 'contrib', 'makefile', 'downloads') 274 if os.path.isdir(makefile_download_dir): 275 for root, _, filenames in os.walk(makefile_download_dir): 276 for f in filenames: 277 if f.endswith('BUILD'): 278 os.remove(os.path.join(root, f)) 279 280 281def get_var(environ_cp, 282 var_name, 283 query_item, 284 enabled_by_default, 285 question=None, 286 yes_reply=None, 287 no_reply=None): 288 """Get boolean input from user. 289 290 If var_name is not set in env, ask user to enable query_item or not. If the 291 response is empty, use the default. 292 293 Args: 294 environ_cp: copy of the os.environ. 295 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". 296 query_item: string for feature related to the variable, e.g. "Hadoop File 297 System". 298 enabled_by_default: boolean for default behavior. 299 question: optional string for how to ask for user input. 300 yes_reply: optional string for reply when feature is enabled. 301 no_reply: optional string for reply when feature is disabled. 302 303 Returns: 304 boolean value of the variable. 305 306 Raises: 307 UserInputError: if an environment variable is set, but it cannot be 308 interpreted as a boolean indicator, assume that the user has made a 309 scripting error, and will continue to provide invalid input. 310 Raise the error to avoid infinitely looping. 311 """ 312 if not question: 313 question = 'Do you wish to build TensorFlow with %s support?' % query_item 314 if not yes_reply: 315 yes_reply = '%s support will be enabled for TensorFlow.' % query_item 316 if not no_reply: 317 no_reply = 'No %s' % yes_reply 318 319 yes_reply += '\n' 320 no_reply += '\n' 321 322 if enabled_by_default: 323 question += ' [Y/n]: ' 324 else: 325 question += ' [y/N]: ' 326 327 var = environ_cp.get(var_name) 328 if var is not None: 329 var_content = var.strip().lower() 330 true_strings = ('1', 't', 'true', 'y', 'yes') 331 false_strings = ('0', 'f', 'false', 'n', 'no') 332 if var_content in true_strings: 333 var = True 334 elif var_content in false_strings: 335 var = False 336 else: 337 raise UserInputError( 338 'Environment variable %s must be set as a boolean indicator.\n' 339 'The following are accepted as TRUE : %s.\n' 340 'The following are accepted as FALSE: %s.\n' 341 'Current value is %s.' % 342 (var_name, ', '.join(true_strings), ', '.join(false_strings), var)) 343 344 while var is None: 345 user_input_origin = get_input(question) 346 user_input = user_input_origin.strip().lower() 347 if user_input == 'y': 348 print(yes_reply) 349 var = True 350 elif user_input == 'n': 351 print(no_reply) 352 var = False 353 elif not user_input: 354 if enabled_by_default: 355 print(yes_reply) 356 var = True 357 else: 358 print(no_reply) 359 var = False 360 else: 361 print('Invalid selection: %s' % user_input_origin) 362 return var 363 364 365def set_build_var(environ_cp, 366 var_name, 367 query_item, 368 option_name, 369 enabled_by_default, 370 bazel_config_name=None): 371 """Set if query_item will be enabled for the build. 372 373 Ask user if query_item will be enabled. Default is used if no input is given. 374 Set subprocess environment variable and write to .bazelrc if enabled. 375 376 Args: 377 environ_cp: copy of the os.environ. 378 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". 379 query_item: string for feature related to the variable, e.g. "Hadoop File 380 System". 381 option_name: string for option to define in .bazelrc. 382 enabled_by_default: boolean for default behavior. 383 bazel_config_name: Name for Bazel --config argument to enable build feature. 384 """ 385 386 var = str(int(get_var(environ_cp, var_name, query_item, enabled_by_default))) 387 environ_cp[var_name] = var 388 if var == '1': 389 write_to_bazelrc( 390 'build:%s --define %s=true' % (bazel_config_name, option_name)) 391 write_to_bazelrc('build --config=%s' % bazel_config_name) 392 elif bazel_config_name is not None: 393 # TODO(mikecase): Migrate all users of configure.py to use --config Bazel 394 # options and not to set build configs through environment variables. 395 write_to_bazelrc( 396 'build:%s --define %s=true' % (bazel_config_name, option_name)) 397 398 399def set_action_env_var(environ_cp, 400 var_name, 401 query_item, 402 enabled_by_default, 403 question=None, 404 yes_reply=None, 405 no_reply=None): 406 """Set boolean action_env variable. 407 408 Ask user if query_item will be enabled. Default is used if no input is given. 409 Set environment variable and write to .bazelrc. 410 411 Args: 412 environ_cp: copy of the os.environ. 413 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". 414 query_item: string for feature related to the variable, e.g. "Hadoop File 415 System". 416 enabled_by_default: boolean for default behavior. 417 question: optional string for how to ask for user input. 418 yes_reply: optional string for reply when feature is enabled. 419 no_reply: optional string for reply when feature is disabled. 420 """ 421 var = int( 422 get_var(environ_cp, var_name, query_item, enabled_by_default, question, 423 yes_reply, no_reply)) 424 425 write_action_env_to_bazelrc(var_name, var) 426 environ_cp[var_name] = str(var) 427 428 429def convert_version_to_int(version): 430 """Convert a version number to a integer that can be used to compare. 431 432 Version strings of the form X.YZ and X.Y.Z-xxxxx are supported. The 433 'xxxxx' part, for instance 'homebrew' on OS/X, is ignored. 434 435 Args: 436 version: a version to be converted 437 438 Returns: 439 An integer if converted successfully, otherwise return None. 440 """ 441 version = version.split('-')[0] 442 version_segments = version.split('.') 443 for seg in version_segments: 444 if not seg.isdigit(): 445 return None 446 447 version_str = ''.join(['%03d' % int(seg) for seg in version_segments]) 448 return int(version_str) 449 450 451def check_bazel_version(min_version, max_version): 452 """Check installed bazel version is between min_version and max_version. 453 454 Args: 455 min_version: string for minimum bazel version. 456 max_version: string for maximum bazel version. 457 458 Returns: 459 The bazel version detected. 460 """ 461 if which('bazel') is None: 462 print('Cannot find bazel. Please install bazel.') 463 sys.exit(0) 464 curr_version = run_shell( 465 ['bazel', '--batch', '--bazelrc=/dev/null', 'version']) 466 467 for line in curr_version.split('\n'): 468 if 'Build label: ' in line: 469 curr_version = line.split('Build label: ')[1] 470 break 471 472 min_version_int = convert_version_to_int(min_version) 473 curr_version_int = convert_version_to_int(curr_version) 474 max_version_int = convert_version_to_int(max_version) 475 476 # Check if current bazel version can be detected properly. 477 if not curr_version_int: 478 print('WARNING: current bazel installation is not a release version.') 479 print('Make sure you are running at least bazel %s' % min_version) 480 return curr_version 481 482 print('You have bazel %s installed.' % curr_version) 483 484 if curr_version_int < min_version_int: 485 print('Please upgrade your bazel installation to version %s or higher to ' 486 'build TensorFlow!' % min_version) 487 sys.exit(1) 488 if (curr_version_int > max_version_int and 489 'TF_IGNORE_MAX_BAZEL_VERSION' not in os.environ): 490 print('Please downgrade your bazel installation to version %s or lower to ' 491 'build TensorFlow! To downgrade: download the installer for the old ' 492 'version (from https://github.com/bazelbuild/bazel/releases) then ' 493 'run the installer.' % max_version) 494 sys.exit(1) 495 return curr_version 496 497 498def set_cc_opt_flags(environ_cp): 499 """Set up architecture-dependent optimization flags. 500 501 Also append CC optimization flags to bazel.rc.. 502 503 Args: 504 environ_cp: copy of the os.environ. 505 """ 506 if is_ppc64le(): 507 # gcc on ppc64le does not support -march, use mcpu instead 508 default_cc_opt_flags = '-mcpu=native' 509 elif is_windows(): 510 default_cc_opt_flags = '/arch:AVX' 511 else: 512 default_cc_opt_flags = '-march=native -Wno-sign-compare' 513 question = ('Please specify optimization flags to use during compilation when' 514 ' bazel option "--config=opt" is specified [Default is %s]: ' 515 ) % default_cc_opt_flags 516 cc_opt_flags = get_from_env_or_user_or_default(environ_cp, 'CC_OPT_FLAGS', 517 question, default_cc_opt_flags) 518 for opt in cc_opt_flags.split(): 519 write_to_bazelrc('build:opt --copt=%s' % opt) 520 # It should be safe on the same build host. 521 if not is_ppc64le() and not is_windows(): 522 write_to_bazelrc('build:opt --host_copt=-march=native') 523 write_to_bazelrc('build:opt --define with_default_optimizations=true') 524 525 526def set_tf_cuda_clang(environ_cp): 527 """set TF_CUDA_CLANG action_env. 528 529 Args: 530 environ_cp: copy of the os.environ. 531 """ 532 question = 'Do you want to use clang as CUDA compiler?' 533 yes_reply = 'Clang will be used as CUDA compiler.' 534 no_reply = 'nvcc will be used as CUDA compiler.' 535 set_action_env_var( 536 environ_cp, 537 'TF_CUDA_CLANG', 538 None, 539 False, 540 question=question, 541 yes_reply=yes_reply, 542 no_reply=no_reply) 543 544 545def set_tf_download_clang(environ_cp): 546 """Set TF_DOWNLOAD_CLANG action_env.""" 547 question = 'Do you wish to download a fresh release of clang? (Experimental)' 548 yes_reply = 'Clang will be downloaded and used to compile tensorflow.' 549 no_reply = 'Clang will not be downloaded.' 550 set_action_env_var( 551 environ_cp, 552 'TF_DOWNLOAD_CLANG', 553 None, 554 False, 555 question=question, 556 yes_reply=yes_reply, 557 no_reply=no_reply) 558 559 560def get_from_env_or_user_or_default(environ_cp, var_name, ask_for_var, 561 var_default): 562 """Get var_name either from env, or user or default. 563 564 If var_name has been set as environment variable, use the preset value, else 565 ask for user input. If no input is provided, the default is used. 566 567 Args: 568 environ_cp: copy of the os.environ. 569 var_name: string for name of environment variable, e.g. "TF_NEED_HDFS". 570 ask_for_var: string for how to ask for user input. 571 var_default: default value string. 572 573 Returns: 574 string value for var_name 575 """ 576 var = environ_cp.get(var_name) 577 if not var: 578 var = get_input(ask_for_var) 579 print('\n') 580 if not var: 581 var = var_default 582 return var 583 584 585def set_clang_cuda_compiler_path(environ_cp): 586 """Set CLANG_CUDA_COMPILER_PATH.""" 587 default_clang_path = which('clang') or '' 588 ask_clang_path = ('Please specify which clang should be used as device and ' 589 'host compiler. [Default is %s]: ') % default_clang_path 590 591 while True: 592 clang_cuda_compiler_path = get_from_env_or_user_or_default( 593 environ_cp, 'CLANG_CUDA_COMPILER_PATH', ask_clang_path, 594 default_clang_path) 595 if os.path.exists(clang_cuda_compiler_path): 596 break 597 598 # Reset and retry 599 print('Invalid clang path: %s cannot be found.' % clang_cuda_compiler_path) 600 environ_cp['CLANG_CUDA_COMPILER_PATH'] = '' 601 602 # Set CLANG_CUDA_COMPILER_PATH 603 environ_cp['CLANG_CUDA_COMPILER_PATH'] = clang_cuda_compiler_path 604 write_action_env_to_bazelrc('CLANG_CUDA_COMPILER_PATH', 605 clang_cuda_compiler_path) 606 607 608def prompt_loop_or_load_from_env(environ_cp, 609 var_name, 610 var_default, 611 ask_for_var, 612 check_success, 613 error_msg, 614 suppress_default_error=False, 615 n_ask_attempts=_DEFAULT_PROMPT_ASK_ATTEMPTS): 616 """Loop over user prompts for an ENV param until receiving a valid response. 617 618 For the env param var_name, read from the environment or verify user input 619 until receiving valid input. When done, set var_name in the environ_cp to its 620 new value. 621 622 Args: 623 environ_cp: (Dict) copy of the os.environ. 624 var_name: (String) string for name of environment variable, e.g. "TF_MYVAR". 625 var_default: (String) default value string. 626 ask_for_var: (String) string for how to ask for user input. 627 check_success: (Function) function that takes one argument and returns a 628 boolean. Should return True if the value provided is considered valid. May 629 contain a complex error message if error_msg does not provide enough 630 information. In that case, set suppress_default_error to True. 631 error_msg: (String) String with one and only one '%s'. Formatted with each 632 invalid response upon check_success(input) failure. 633 suppress_default_error: (Bool) Suppress the above error message in favor of 634 one from the check_success function. 635 n_ask_attempts: (Integer) Number of times to query for valid input before 636 raising an error and quitting. 637 638 Returns: 639 [String] The value of var_name after querying for input. 640 641 Raises: 642 UserInputError: if a query has been attempted n_ask_attempts times without 643 success, assume that the user has made a scripting error, and will 644 continue to provide invalid input. Raise the error to avoid infinitely 645 looping. 646 """ 647 default = environ_cp.get(var_name) or var_default 648 full_query = '%s [Default is %s]: ' % ( 649 ask_for_var, 650 default, 651 ) 652 653 for _ in range(n_ask_attempts): 654 val = get_from_env_or_user_or_default(environ_cp, var_name, full_query, 655 default) 656 if check_success(val): 657 break 658 if not suppress_default_error: 659 print(error_msg % val) 660 environ_cp[var_name] = '' 661 else: 662 raise UserInputError( 663 'Invalid %s setting was provided %d times in a row. ' 664 'Assuming to be a scripting mistake.' % (var_name, n_ask_attempts)) 665 666 environ_cp[var_name] = val 667 return val 668 669 670def create_android_ndk_rule(environ_cp): 671 """Set ANDROID_NDK_HOME and write Android NDK WORKSPACE rule.""" 672 if is_windows() or is_cygwin(): 673 default_ndk_path = cygpath( 674 '%s/Android/Sdk/ndk-bundle' % environ_cp['APPDATA']) 675 elif is_macos(): 676 default_ndk_path = '%s/library/Android/Sdk/ndk-bundle' % environ_cp['HOME'] 677 else: 678 default_ndk_path = '%s/Android/Sdk/ndk-bundle' % environ_cp['HOME'] 679 680 def valid_ndk_path(path): 681 return (os.path.exists(path) and 682 os.path.exists(os.path.join(path, 'source.properties'))) 683 684 android_ndk_home_path = prompt_loop_or_load_from_env( 685 environ_cp, 686 var_name='ANDROID_NDK_HOME', 687 var_default=default_ndk_path, 688 ask_for_var='Please specify the home path of the Android NDK to use.', 689 check_success=valid_ndk_path, 690 error_msg=('The path %s or its child file "source.properties" ' 691 'does not exist.')) 692 write_action_env_to_bazelrc('ANDROID_NDK_HOME', android_ndk_home_path) 693 write_action_env_to_bazelrc('ANDROID_NDK_API_LEVEL', 694 check_ndk_level(android_ndk_home_path)) 695 696 697def create_android_sdk_rule(environ_cp): 698 """Set Android variables and write Android SDK WORKSPACE rule.""" 699 if is_windows() or is_cygwin(): 700 default_sdk_path = cygpath('%s/Android/Sdk' % environ_cp['APPDATA']) 701 elif is_macos(): 702 default_sdk_path = '%s/library/Android/Sdk' % environ_cp['HOME'] 703 else: 704 default_sdk_path = '%s/Android/Sdk' % environ_cp['HOME'] 705 706 def valid_sdk_path(path): 707 return (os.path.exists(path) and 708 os.path.exists(os.path.join(path, 'platforms')) and 709 os.path.exists(os.path.join(path, 'build-tools'))) 710 711 android_sdk_home_path = prompt_loop_or_load_from_env( 712 environ_cp, 713 var_name='ANDROID_SDK_HOME', 714 var_default=default_sdk_path, 715 ask_for_var='Please specify the home path of the Android SDK to use.', 716 check_success=valid_sdk_path, 717 error_msg=('Either %s does not exist, or it does not contain the ' 718 'subdirectories "platforms" and "build-tools".')) 719 720 platforms = os.path.join(android_sdk_home_path, 'platforms') 721 api_levels = sorted(os.listdir(platforms)) 722 api_levels = [x.replace('android-', '') for x in api_levels] 723 724 def valid_api_level(api_level): 725 return os.path.exists( 726 os.path.join(android_sdk_home_path, 'platforms', 727 'android-' + api_level)) 728 729 android_api_level = prompt_loop_or_load_from_env( 730 environ_cp, 731 var_name='ANDROID_API_LEVEL', 732 var_default=api_levels[-1], 733 ask_for_var=('Please specify the Android SDK API level to use. ' 734 '[Available levels: %s]') % api_levels, 735 check_success=valid_api_level, 736 error_msg='Android-%s is not present in the SDK path.') 737 738 build_tools = os.path.join(android_sdk_home_path, 'build-tools') 739 versions = sorted(os.listdir(build_tools)) 740 741 def valid_build_tools(version): 742 return os.path.exists( 743 os.path.join(android_sdk_home_path, 'build-tools', version)) 744 745 android_build_tools_version = prompt_loop_or_load_from_env( 746 environ_cp, 747 var_name='ANDROID_BUILD_TOOLS_VERSION', 748 var_default=versions[-1], 749 ask_for_var=('Please specify an Android build tools version to use. ' 750 '[Available versions: %s]') % versions, 751 check_success=valid_build_tools, 752 error_msg=('The selected SDK does not have build-tools version %s ' 753 'available.')) 754 755 write_action_env_to_bazelrc('ANDROID_BUILD_TOOLS_VERSION', 756 android_build_tools_version) 757 write_action_env_to_bazelrc('ANDROID_SDK_API_LEVEL', android_api_level) 758 write_action_env_to_bazelrc('ANDROID_SDK_HOME', android_sdk_home_path) 759 760 761def check_ndk_level(android_ndk_home_path): 762 """Check the revision number of an Android NDK path.""" 763 properties_path = '%s/source.properties' % android_ndk_home_path 764 if is_windows() or is_cygwin(): 765 properties_path = cygpath(properties_path) 766 with open(properties_path, 'r') as f: 767 filedata = f.read() 768 769 revision = re.search(r'Pkg.Revision = (\d+)', filedata) 770 if revision: 771 ndk_api_level = revision.group(1) 772 else: 773 raise Exception('Unable to parse NDK revision.') 774 if int(ndk_api_level) not in _SUPPORTED_ANDROID_NDK_VERSIONS: 775 print( 776 'WARNING: The API level of the NDK in %s is %s, which is not ' 777 'supported by Bazel (officially supported versions: %s). Please use ' 778 'another version. Compiling Android targets may result in confusing ' 779 'errors.\n' % 780 (android_ndk_home_path, ndk_api_level, _SUPPORTED_ANDROID_NDK_VERSIONS)) 781 return ndk_api_level 782 783 784def set_gcc_host_compiler_path(environ_cp): 785 """Set GCC_HOST_COMPILER_PATH.""" 786 default_gcc_host_compiler_path = which('gcc') or '' 787 cuda_bin_symlink = '%s/bin/gcc' % environ_cp.get('CUDA_TOOLKIT_PATH') 788 789 if os.path.islink(cuda_bin_symlink): 790 # os.readlink is only available in linux 791 default_gcc_host_compiler_path = os.path.realpath(cuda_bin_symlink) 792 793 gcc_host_compiler_path = prompt_loop_or_load_from_env( 794 environ_cp, 795 var_name='GCC_HOST_COMPILER_PATH', 796 var_default=default_gcc_host_compiler_path, 797 ask_for_var='Please specify which gcc should be used by nvcc as the host compiler.', 798 check_success=os.path.exists, 799 error_msg='Invalid gcc path. %s cannot be found.', 800 ) 801 802 write_action_env_to_bazelrc('GCC_HOST_COMPILER_PATH', gcc_host_compiler_path) 803 804 805def reformat_version_sequence(version_str, sequence_count): 806 """Reformat the version string to have the given number of sequences. 807 808 For example: 809 Given (7, 2) -> 7.0 810 (7.0.1, 2) -> 7.0 811 (5, 1) -> 5 812 (5.0.3.2, 1) -> 5 813 814 Args: 815 version_str: String, the version string. 816 sequence_count: int, an integer. 817 818 Returns: 819 string, reformatted version string. 820 """ 821 v = version_str.split('.') 822 if len(v) < sequence_count: 823 v = v + (['0'] * (sequence_count - len(v))) 824 825 return '.'.join(v[:sequence_count]) 826 827 828def set_tf_cuda_version(environ_cp): 829 """Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION.""" 830 ask_cuda_version = ( 831 'Please specify the CUDA SDK version you want to use. ' 832 '[Leave empty to default to CUDA %s]: ') % _DEFAULT_CUDA_VERSION 833 834 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): 835 # Configure the Cuda SDK version to use. 836 tf_cuda_version = get_from_env_or_user_or_default( 837 environ_cp, 'TF_CUDA_VERSION', ask_cuda_version, _DEFAULT_CUDA_VERSION) 838 tf_cuda_version = reformat_version_sequence(str(tf_cuda_version), 2) 839 840 # Find out where the CUDA toolkit is installed 841 default_cuda_path = _DEFAULT_CUDA_PATH 842 if is_windows() or is_cygwin(): 843 default_cuda_path = cygpath( 844 environ_cp.get('CUDA_PATH', _DEFAULT_CUDA_PATH_WIN)) 845 elif is_linux(): 846 # If the default doesn't exist, try an alternative default. 847 if (not os.path.exists(default_cuda_path) 848 ) and os.path.exists(_DEFAULT_CUDA_PATH_LINUX): 849 default_cuda_path = _DEFAULT_CUDA_PATH_LINUX 850 ask_cuda_path = ('Please specify the location where CUDA %s toolkit is' 851 ' installed. Refer to README.md for more details. ' 852 '[Default is %s]: ') % (tf_cuda_version, default_cuda_path) 853 cuda_toolkit_path = get_from_env_or_user_or_default( 854 environ_cp, 'CUDA_TOOLKIT_PATH', ask_cuda_path, default_cuda_path) 855 if is_windows() or is_cygwin(): 856 cuda_toolkit_path = cygpath(cuda_toolkit_path) 857 858 if is_windows(): 859 cuda_rt_lib_paths = ['lib/x64/cudart.lib'] 860 elif is_linux(): 861 cuda_rt_lib_paths = [ 862 '%s/libcudart.so.%s' % (x, tf_cuda_version) for x in [ 863 'lib64', 864 'lib/powerpc64le-linux-gnu', 865 'lib/x86_64-linux-gnu', 866 ] 867 ] 868 elif is_macos(): 869 cuda_rt_lib_paths = ['lib/libcudart.%s.dylib' % tf_cuda_version] 870 871 cuda_toolkit_paths_full = [ 872 os.path.join(cuda_toolkit_path, x) for x in cuda_rt_lib_paths 873 ] 874 if any(os.path.exists(x) for x in cuda_toolkit_paths_full): 875 break 876 877 # Reset and retry 878 print('Invalid path to CUDA %s toolkit. %s cannot be found' % 879 (tf_cuda_version, cuda_toolkit_paths_full)) 880 environ_cp['TF_CUDA_VERSION'] = '' 881 environ_cp['CUDA_TOOLKIT_PATH'] = '' 882 883 else: 884 raise UserInputError('Invalid TF_CUDA_SETTING setting was provided %d ' 885 'times in a row. Assuming to be a scripting mistake.' % 886 _DEFAULT_PROMPT_ASK_ATTEMPTS) 887 888 # Set CUDA_TOOLKIT_PATH and TF_CUDA_VERSION 889 environ_cp['CUDA_TOOLKIT_PATH'] = cuda_toolkit_path 890 write_action_env_to_bazelrc('CUDA_TOOLKIT_PATH', cuda_toolkit_path) 891 environ_cp['TF_CUDA_VERSION'] = tf_cuda_version 892 write_action_env_to_bazelrc('TF_CUDA_VERSION', tf_cuda_version) 893 894 895def set_tf_cudnn_version(environ_cp): 896 """Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION.""" 897 ask_cudnn_version = ( 898 'Please specify the cuDNN version you want to use. ' 899 '[Leave empty to default to cuDNN %s]: ') % _DEFAULT_CUDNN_VERSION 900 901 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): 902 tf_cudnn_version = get_from_env_or_user_or_default( 903 environ_cp, 'TF_CUDNN_VERSION', ask_cudnn_version, 904 _DEFAULT_CUDNN_VERSION) 905 tf_cudnn_version = reformat_version_sequence(str(tf_cudnn_version), 1) 906 907 default_cudnn_path = environ_cp.get('CUDA_TOOLKIT_PATH') 908 ask_cudnn_path = (r'Please specify the location where cuDNN %s library is ' 909 'installed. Refer to README.md for more details. [Default' 910 ' is %s]: ') % (tf_cudnn_version, default_cudnn_path) 911 cudnn_install_path = get_from_env_or_user_or_default( 912 environ_cp, 'CUDNN_INSTALL_PATH', ask_cudnn_path, default_cudnn_path) 913 914 # Result returned from "read" will be used unexpanded. That make "~" 915 # unusable. Going through one more level of expansion to handle that. 916 cudnn_install_path = os.path.realpath( 917 os.path.expanduser(cudnn_install_path)) 918 if is_windows() or is_cygwin(): 919 cudnn_install_path = cygpath(cudnn_install_path) 920 921 if is_windows(): 922 cuda_dnn_lib_path = 'lib/x64/cudnn.lib' 923 cuda_dnn_lib_alt_path = 'lib/x64/cudnn.lib' 924 elif is_linux(): 925 cuda_dnn_lib_path = 'lib64/libcudnn.so.%s' % tf_cudnn_version 926 cuda_dnn_lib_alt_path = 'libcudnn.so.%s' % tf_cudnn_version 927 elif is_macos(): 928 cuda_dnn_lib_path = 'lib/libcudnn.%s.dylib' % tf_cudnn_version 929 cuda_dnn_lib_alt_path = 'libcudnn.%s.dylib' % tf_cudnn_version 930 931 cuda_dnn_lib_path_full = os.path.join(cudnn_install_path, cuda_dnn_lib_path) 932 cuda_dnn_lib_alt_path_full = os.path.join(cudnn_install_path, 933 cuda_dnn_lib_alt_path) 934 if os.path.exists(cuda_dnn_lib_path_full) or os.path.exists( 935 cuda_dnn_lib_alt_path_full): 936 break 937 938 # Try another alternative for Linux 939 if is_linux(): 940 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' 941 cudnn_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) 942 cudnn_path_from_ldconfig = re.search('.*libcudnn.so .* => (.*)', 943 cudnn_path_from_ldconfig) 944 if cudnn_path_from_ldconfig: 945 cudnn_path_from_ldconfig = cudnn_path_from_ldconfig.group(1) 946 if os.path.exists( 947 '%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)): 948 cudnn_install_path = os.path.dirname(cudnn_path_from_ldconfig) 949 break 950 951 # Reset and Retry 952 print( 953 'Invalid path to cuDNN %s toolkit. None of the following files can be ' 954 'found:' % tf_cudnn_version) 955 print(cuda_dnn_lib_path_full) 956 print(cuda_dnn_lib_alt_path_full) 957 if is_linux(): 958 print('%s.%s' % (cudnn_path_from_ldconfig, tf_cudnn_version)) 959 960 environ_cp['TF_CUDNN_VERSION'] = '' 961 else: 962 raise UserInputError('Invalid TF_CUDNN setting was provided %d ' 963 'times in a row. Assuming to be a scripting mistake.' % 964 _DEFAULT_PROMPT_ASK_ATTEMPTS) 965 966 # Set CUDNN_INSTALL_PATH and TF_CUDNN_VERSION 967 environ_cp['CUDNN_INSTALL_PATH'] = cudnn_install_path 968 write_action_env_to_bazelrc('CUDNN_INSTALL_PATH', cudnn_install_path) 969 environ_cp['TF_CUDNN_VERSION'] = tf_cudnn_version 970 write_action_env_to_bazelrc('TF_CUDNN_VERSION', tf_cudnn_version) 971 972 973def is_cuda_compatible(lib, cuda_ver, cudnn_ver): 974 """Check compatibility between given library and cudnn/cudart libraries.""" 975 ldd_bin = which('ldd') or '/usr/bin/ldd' 976 ldd_out = run_shell([ldd_bin, lib], True) 977 ldd_out = ldd_out.split(os.linesep) 978 cudnn_pattern = re.compile('.*libcudnn.so\\.?(.*) =>.*$') 979 cuda_pattern = re.compile('.*libcudart.so\\.?(.*) =>.*$') 980 cudnn = None 981 cudart = None 982 cudnn_ok = True # assume no cudnn dependency by default 983 cuda_ok = True # assume no cuda dependency by default 984 for line in ldd_out: 985 if 'libcudnn.so' in line: 986 cudnn = cudnn_pattern.search(line) 987 cudnn_ok = False 988 elif 'libcudart.so' in line: 989 cudart = cuda_pattern.search(line) 990 cuda_ok = False 991 if cudnn and len(cudnn.group(1)): 992 cudnn = convert_version_to_int(cudnn.group(1)) 993 if cudart and len(cudart.group(1)): 994 cudart = convert_version_to_int(cudart.group(1)) 995 if cudnn is not None: 996 cudnn_ok = (cudnn == cudnn_ver) 997 if cudart is not None: 998 cuda_ok = (cudart == cuda_ver) 999 return cudnn_ok and cuda_ok 1000 1001 1002def set_tf_tensorrt_install_path(environ_cp): 1003 """Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION. 1004 1005 Adapted from code contributed by Sami Kama (https://github.com/samikama). 1006 1007 Args: 1008 environ_cp: copy of the os.environ. 1009 1010 Raises: 1011 ValueError: if this method was called under non-Linux platform. 1012 UserInputError: if user has provided invalid input multiple times. 1013 """ 1014 if not is_linux(): 1015 raise ValueError('Currently TensorRT is only supported on Linux platform.') 1016 1017 # Ask user whether to add TensorRT support. 1018 if str(int(get_var(environ_cp, 'TF_NEED_TENSORRT', 'TensorRT', 1019 False))) != '1': 1020 return 1021 1022 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): 1023 ask_tensorrt_path = (r'Please specify the location where TensorRT is ' 1024 'installed. [Default is %s]:') % ( 1025 _DEFAULT_TENSORRT_PATH_LINUX) 1026 trt_install_path = get_from_env_or_user_or_default( 1027 environ_cp, 'TENSORRT_INSTALL_PATH', ask_tensorrt_path, 1028 _DEFAULT_TENSORRT_PATH_LINUX) 1029 1030 # Result returned from "read" will be used unexpanded. That make "~" 1031 # unusable. Going through one more level of expansion to handle that. 1032 trt_install_path = os.path.realpath(os.path.expanduser(trt_install_path)) 1033 1034 def find_libs(search_path): 1035 """Search for libnvinfer.so in "search_path".""" 1036 fl = set() 1037 if os.path.exists(search_path) and os.path.isdir(search_path): 1038 fl.update([ 1039 os.path.realpath(os.path.join(search_path, x)) 1040 for x in os.listdir(search_path) 1041 if 'libnvinfer.so' in x 1042 ]) 1043 return fl 1044 1045 possible_files = find_libs(trt_install_path) 1046 possible_files.update(find_libs(os.path.join(trt_install_path, 'lib'))) 1047 possible_files.update(find_libs(os.path.join(trt_install_path, 'lib64'))) 1048 cuda_ver = convert_version_to_int(environ_cp['TF_CUDA_VERSION']) 1049 cudnn_ver = convert_version_to_int(environ_cp['TF_CUDNN_VERSION']) 1050 nvinfer_pattern = re.compile('.*libnvinfer.so.?(.*)$') 1051 highest_ver = [0, None, None] 1052 1053 for lib_file in possible_files: 1054 if is_cuda_compatible(lib_file, cuda_ver, cudnn_ver): 1055 matches = nvinfer_pattern.search(lib_file) 1056 if not matches.groups(): 1057 continue 1058 ver_str = matches.group(1) 1059 ver = convert_version_to_int(ver_str) if len(ver_str) else 0 1060 if ver > highest_ver[0]: 1061 highest_ver = [ver, ver_str, lib_file] 1062 if highest_ver[1] is not None: 1063 trt_install_path = os.path.dirname(highest_ver[2]) 1064 tf_tensorrt_version = highest_ver[1] 1065 break 1066 1067 # Try another alternative from ldconfig. 1068 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' 1069 ldconfig_output = run_shell([ldconfig_bin, '-p']) 1070 search_result = re.search('.*libnvinfer.so\\.?([0-9.]*).* => (.*)', 1071 ldconfig_output) 1072 if search_result: 1073 libnvinfer_path_from_ldconfig = search_result.group(2) 1074 if os.path.exists(libnvinfer_path_from_ldconfig): 1075 if is_cuda_compatible(libnvinfer_path_from_ldconfig, cuda_ver, 1076 cudnn_ver): 1077 trt_install_path = os.path.dirname(libnvinfer_path_from_ldconfig) 1078 tf_tensorrt_version = search_result.group(1) 1079 break 1080 1081 # Reset and Retry 1082 if possible_files: 1083 print('TensorRT libraries found in one the following directories', 1084 'are not compatible with selected cuda and cudnn installations') 1085 print(trt_install_path) 1086 print(os.path.join(trt_install_path, 'lib')) 1087 print(os.path.join(trt_install_path, 'lib64')) 1088 if search_result: 1089 print(libnvinfer_path_from_ldconfig) 1090 else: 1091 print( 1092 'Invalid path to TensorRT. None of the following files can be found:') 1093 print(trt_install_path) 1094 print(os.path.join(trt_install_path, 'lib')) 1095 print(os.path.join(trt_install_path, 'lib64')) 1096 if search_result: 1097 print(libnvinfer_path_from_ldconfig) 1098 1099 else: 1100 raise UserInputError('Invalid TF_TENSORRT setting was provided %d ' 1101 'times in a row. Assuming to be a scripting mistake.' % 1102 _DEFAULT_PROMPT_ASK_ATTEMPTS) 1103 1104 # Set TENSORRT_INSTALL_PATH and TF_TENSORRT_VERSION 1105 environ_cp['TENSORRT_INSTALL_PATH'] = trt_install_path 1106 write_action_env_to_bazelrc('TENSORRT_INSTALL_PATH', trt_install_path) 1107 environ_cp['TF_TENSORRT_VERSION'] = tf_tensorrt_version 1108 write_action_env_to_bazelrc('TF_TENSORRT_VERSION', tf_tensorrt_version) 1109 1110 1111def set_tf_nccl_install_path(environ_cp): 1112 """Set NCCL_INSTALL_PATH, NCCL_HDR_PATH and TF_NCCL_VERSION. 1113 1114 Args: 1115 environ_cp: copy of the os.environ. 1116 1117 Raises: 1118 ValueError: if this method was called under non-Linux platform. 1119 UserInputError: if user has provided invalid input multiple times. 1120 """ 1121 if not is_linux(): 1122 raise ValueError('Currently NCCL is only supported on Linux platforms.') 1123 1124 ask_nccl_version = ( 1125 'Please specify the locally installed NCCL version you want to use. ' 1126 '[Default is to use https://github.com/nvidia/nccl]: ') 1127 1128 for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS): 1129 tf_nccl_version = get_from_env_or_user_or_default( 1130 environ_cp, 'TF_NCCL_VERSION', ask_nccl_version, '') 1131 1132 if not tf_nccl_version: 1133 break # No need to get install path, building the open source code. 1134 1135 tf_nccl_version = reformat_version_sequence(str(tf_nccl_version), 1) 1136 1137 # Look with ldconfig first if we can find the library in paths 1138 # like /usr/lib/x86_64-linux-gnu and the header file in the corresponding 1139 # include directory. This is where the NCCL .deb packages install them. 1140 1141 # First check to see if NCCL is in the ldconfig. 1142 # If its found, use that location. 1143 if is_linux(): 1144 ldconfig_bin = which('ldconfig') or '/sbin/ldconfig' 1145 nccl2_path_from_ldconfig = run_shell([ldconfig_bin, '-p']) 1146 nccl2_path_from_ldconfig = re.search('.*libnccl.so .* => (.*)', 1147 nccl2_path_from_ldconfig) 1148 if nccl2_path_from_ldconfig: 1149 nccl2_path_from_ldconfig = nccl2_path_from_ldconfig.group(1) 1150 if os.path.exists('%s.%s' % (nccl2_path_from_ldconfig, tf_nccl_version)): 1151 nccl_install_path = os.path.dirname(nccl2_path_from_ldconfig) 1152 print('NCCL libraries found in ' + nccl2_path_from_ldconfig) 1153 1154 # Check if this is the main system lib location 1155 if re.search('.*linux-gnu', nccl_install_path): 1156 trunc_nccl_install_path = '/usr' 1157 print('This looks like a system path.') 1158 else: 1159 trunc_nccl_install_path = nccl_install_path + '/..' 1160 1161 # Look for header 1162 nccl_hdr_path = trunc_nccl_install_path + '/include' 1163 print('Assuming NCCL header path is ' + nccl_hdr_path) 1164 if os.path.exists(nccl_hdr_path + '/nccl.h'): 1165 # Set NCCL_INSTALL_PATH 1166 environ_cp['NCCL_INSTALL_PATH'] = nccl_install_path 1167 write_action_env_to_bazelrc('NCCL_INSTALL_PATH', nccl_install_path) 1168 1169 # Set NCCL_HDR_PATH 1170 environ_cp['NCCL_HDR_PATH'] = nccl_hdr_path 1171 write_action_env_to_bazelrc('NCCL_HDR_PATH', nccl_hdr_path) 1172 break 1173 else: 1174 print( 1175 'The header for NCCL2 cannot be found. Please install the libnccl-dev package.' 1176 ) 1177 else: 1178 print('NCCL2 is listed by ldconfig but the library is not found. ' 1179 'Your ldconfig is out of date. Please run sudo ldconfig.') 1180 else: 1181 # NCCL is not found in ldconfig. Ask the user for the location. 1182 default_nccl_path = environ_cp.get('CUDA_TOOLKIT_PATH') 1183 ask_nccl_path = ( 1184 r'Please specify the location where NCCL %s library is ' 1185 'installed. Refer to README.md for more details. [Default ' 1186 'is %s]:') % (tf_nccl_version, default_nccl_path) 1187 nccl_install_path = get_from_env_or_user_or_default( 1188 environ_cp, 'NCCL_INSTALL_PATH', ask_nccl_path, default_nccl_path) 1189 1190 # Result returned from "read" will be used unexpanded. That make "~" 1191 # unusable. Going through one more level of expansion to handle that. 1192 nccl_install_path = os.path.realpath( 1193 os.path.expanduser(nccl_install_path)) 1194 if is_windows() or is_cygwin(): 1195 nccl_install_path = cygpath(nccl_install_path) 1196 1197 nccl_lib_path = '' 1198 if is_windows(): 1199 nccl_lib_path = 'lib/x64/nccl.lib' 1200 elif is_linux(): 1201 nccl_lib_filename = 'libnccl.so.%s' % tf_nccl_version 1202 nccl_lpath = '%s/lib/%s' % (nccl_install_path, nccl_lib_filename) 1203 if not os.path.exists(nccl_lpath): 1204 for relative_path in NCCL_LIB_PATHS: 1205 path = '%s/%s%s' % (nccl_install_path, relative_path, 1206 nccl_lib_filename) 1207 if os.path.exists(path): 1208 print('NCCL found at ' + path) 1209 nccl_lib_path = path 1210 break 1211 else: 1212 nccl_lib_path = nccl_lpath 1213 elif is_macos(): 1214 nccl_lib_path = 'lib/libnccl.%s.dylib' % tf_nccl_version 1215 1216 nccl_lib_path = os.path.join(nccl_install_path, nccl_lib_path) 1217 nccl_hdr_path = os.path.join( 1218 os.path.dirname(nccl_lib_path), '../include/nccl.h') 1219 print('Assuming NCCL header path is ' + nccl_hdr_path) 1220 if os.path.exists(nccl_lib_path) and os.path.exists(nccl_hdr_path): 1221 # Set NCCL_INSTALL_PATH 1222 environ_cp['NCCL_INSTALL_PATH'] = os.path.dirname(nccl_lib_path) 1223 write_action_env_to_bazelrc('NCCL_INSTALL_PATH', 1224 os.path.dirname(nccl_lib_path)) 1225 1226 # Set NCCL_HDR_PATH 1227 environ_cp['NCCL_HDR_PATH'] = os.path.dirname(nccl_hdr_path) 1228 write_action_env_to_bazelrc('NCCL_HDR_PATH', 1229 os.path.dirname(nccl_hdr_path)) 1230 break 1231 1232 # Reset and Retry 1233 print( 1234 'Invalid path to NCCL %s toolkit, %s or %s not found. Please use the ' 1235 'O/S agnostic package of NCCL 2' % 1236 (tf_nccl_version, nccl_lib_path, nccl_hdr_path)) 1237 1238 environ_cp['TF_NCCL_VERSION'] = '' 1239 else: 1240 raise UserInputError('Invalid TF_NCCL setting was provided %d ' 1241 'times in a row. Assuming to be a scripting mistake.' % 1242 _DEFAULT_PROMPT_ASK_ATTEMPTS) 1243 1244 # Set TF_NCCL_VERSION 1245 environ_cp['TF_NCCL_VERSION'] = tf_nccl_version 1246 write_action_env_to_bazelrc('TF_NCCL_VERSION', tf_nccl_version) 1247 1248 1249def get_native_cuda_compute_capabilities(environ_cp): 1250 """Get native cuda compute capabilities. 1251 1252 Args: 1253 environ_cp: copy of the os.environ. 1254 1255 Returns: 1256 string of native cuda compute capabilities, separated by comma. 1257 """ 1258 device_query_bin = os.path.join( 1259 environ_cp.get('CUDA_TOOLKIT_PATH'), 'extras/demo_suite/deviceQuery') 1260 if os.path.isfile(device_query_bin) and os.access(device_query_bin, os.X_OK): 1261 try: 1262 output = run_shell(device_query_bin).split('\n') 1263 pattern = re.compile('[0-9]*\\.[0-9]*') 1264 output = [pattern.search(x) for x in output if 'Capability' in x] 1265 output = ','.join(x.group() for x in output if x is not None) 1266 except subprocess.CalledProcessError: 1267 output = '' 1268 else: 1269 output = '' 1270 return output 1271 1272 1273def set_tf_cuda_compute_capabilities(environ_cp): 1274 """Set TF_CUDA_COMPUTE_CAPABILITIES.""" 1275 while True: 1276 native_cuda_compute_capabilities = get_native_cuda_compute_capabilities( 1277 environ_cp) 1278 if not native_cuda_compute_capabilities: 1279 default_cuda_compute_capabilities = _DEFAULT_CUDA_COMPUTE_CAPABILITIES 1280 else: 1281 default_cuda_compute_capabilities = native_cuda_compute_capabilities 1282 1283 ask_cuda_compute_capabilities = ( 1284 'Please specify a list of comma-separated ' 1285 'CUDA compute capabilities you want to ' 1286 'build with.\nYou can find the compute ' 1287 'capability of your device at: ' 1288 'https://developer.nvidia.com/cuda-gpus.\nPlease' 1289 ' note that each additional compute ' 1290 'capability significantly increases your ' 1291 'build time and binary size, and that ' 1292 'TensorFlow only supports compute ' 1293 'capabilities >= 3.5 [Default is: %s]: ' % 1294 default_cuda_compute_capabilities) 1295 tf_cuda_compute_capabilities = get_from_env_or_user_or_default( 1296 environ_cp, 'TF_CUDA_COMPUTE_CAPABILITIES', 1297 ask_cuda_compute_capabilities, default_cuda_compute_capabilities) 1298 # Check whether all capabilities from the input is valid 1299 all_valid = True 1300 # Remove all whitespace characters before splitting the string 1301 # that users may insert by accident, as this will result in error 1302 tf_cuda_compute_capabilities = ''.join(tf_cuda_compute_capabilities.split()) 1303 for compute_capability in tf_cuda_compute_capabilities.split(','): 1304 m = re.match('[0-9]+.[0-9]+', compute_capability) 1305 if not m: 1306 print('Invalid compute capability: %s' % compute_capability) 1307 all_valid = False 1308 else: 1309 ver = float(m.group(0)) 1310 if ver < 3.0: 1311 print('ERROR: TensorFlow only supports CUDA compute capabilities 3.0 ' 1312 'and higher. Please re-specify the list of compute ' 1313 'capabilities excluding version %s.' % ver) 1314 all_valid = False 1315 if ver < 3.5: 1316 print('WARNING: XLA does not support CUDA compute capabilities ' 1317 'lower than 3.5. Disable XLA when running on older GPUs.') 1318 1319 if all_valid: 1320 break 1321 1322 # Reset and Retry 1323 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = '' 1324 1325 # Set TF_CUDA_COMPUTE_CAPABILITIES 1326 environ_cp['TF_CUDA_COMPUTE_CAPABILITIES'] = tf_cuda_compute_capabilities 1327 write_action_env_to_bazelrc('TF_CUDA_COMPUTE_CAPABILITIES', 1328 tf_cuda_compute_capabilities) 1329 1330 1331def set_other_cuda_vars(environ_cp): 1332 """Set other CUDA related variables.""" 1333 # If CUDA is enabled, always use GPU during build and test. 1334 if environ_cp.get('TF_CUDA_CLANG') == '1': 1335 write_to_bazelrc('build --config=cuda_clang') 1336 write_to_bazelrc('test --config=cuda_clang') 1337 else: 1338 write_to_bazelrc('build --config=cuda') 1339 write_to_bazelrc('test --config=cuda') 1340 1341 1342def set_host_cxx_compiler(environ_cp): 1343 """Set HOST_CXX_COMPILER.""" 1344 default_cxx_host_compiler = which('g++') or '' 1345 1346 host_cxx_compiler = prompt_loop_or_load_from_env( 1347 environ_cp, 1348 var_name='HOST_CXX_COMPILER', 1349 var_default=default_cxx_host_compiler, 1350 ask_for_var=('Please specify which C++ compiler should be used as the ' 1351 'host C++ compiler.'), 1352 check_success=os.path.exists, 1353 error_msg='Invalid C++ compiler path. %s cannot be found.', 1354 ) 1355 1356 write_action_env_to_bazelrc('HOST_CXX_COMPILER', host_cxx_compiler) 1357 1358 1359def set_host_c_compiler(environ_cp): 1360 """Set HOST_C_COMPILER.""" 1361 default_c_host_compiler = which('gcc') or '' 1362 1363 host_c_compiler = prompt_loop_or_load_from_env( 1364 environ_cp, 1365 var_name='HOST_C_COMPILER', 1366 var_default=default_c_host_compiler, 1367 ask_for_var=('Please specify which C compiler should be used as the host ' 1368 'C compiler.'), 1369 check_success=os.path.exists, 1370 error_msg='Invalid C compiler path. %s cannot be found.', 1371 ) 1372 1373 write_action_env_to_bazelrc('HOST_C_COMPILER', host_c_compiler) 1374 1375 1376def set_computecpp_toolkit_path(environ_cp): 1377 """Set COMPUTECPP_TOOLKIT_PATH.""" 1378 1379 def toolkit_exists(toolkit_path): 1380 """Check if a computecpp toolkit path is valid.""" 1381 if is_linux(): 1382 sycl_rt_lib_path = 'lib/libComputeCpp.so' 1383 else: 1384 sycl_rt_lib_path = '' 1385 1386 sycl_rt_lib_path_full = os.path.join(toolkit_path, sycl_rt_lib_path) 1387 exists = os.path.exists(sycl_rt_lib_path_full) 1388 if not exists: 1389 print('Invalid SYCL %s library path. %s cannot be found' % 1390 (_TF_OPENCL_VERSION, sycl_rt_lib_path_full)) 1391 return exists 1392 1393 computecpp_toolkit_path = prompt_loop_or_load_from_env( 1394 environ_cp, 1395 var_name='COMPUTECPP_TOOLKIT_PATH', 1396 var_default=_DEFAULT_COMPUTECPP_TOOLKIT_PATH, 1397 ask_for_var=( 1398 'Please specify the location where ComputeCpp for SYCL %s is ' 1399 'installed.' % _TF_OPENCL_VERSION), 1400 check_success=toolkit_exists, 1401 error_msg='Invalid SYCL compiler path. %s cannot be found.', 1402 suppress_default_error=True) 1403 1404 write_action_env_to_bazelrc('COMPUTECPP_TOOLKIT_PATH', 1405 computecpp_toolkit_path) 1406 1407 1408def set_trisycl_include_dir(environ_cp): 1409 """Set TRISYCL_INCLUDE_DIR.""" 1410 1411 ask_trisycl_include_dir = ('Please specify the location of the triSYCL ' 1412 'include directory. (Use --config=sycl_trisycl ' 1413 'when building with Bazel) ' 1414 '[Default is %s]: ') % ( 1415 _DEFAULT_TRISYCL_INCLUDE_DIR) 1416 1417 while True: 1418 trisycl_include_dir = get_from_env_or_user_or_default( 1419 environ_cp, 'TRISYCL_INCLUDE_DIR', ask_trisycl_include_dir, 1420 _DEFAULT_TRISYCL_INCLUDE_DIR) 1421 if os.path.exists(trisycl_include_dir): 1422 break 1423 1424 print('Invalid triSYCL include directory, %s cannot be found' % 1425 (trisycl_include_dir)) 1426 1427 # Set TRISYCL_INCLUDE_DIR 1428 environ_cp['TRISYCL_INCLUDE_DIR'] = trisycl_include_dir 1429 write_action_env_to_bazelrc('TRISYCL_INCLUDE_DIR', trisycl_include_dir) 1430 1431 1432def set_mpi_home(environ_cp): 1433 """Set MPI_HOME.""" 1434 1435 default_mpi_home = which('mpirun') or which('mpiexec') or '' 1436 default_mpi_home = os.path.dirname(os.path.dirname(default_mpi_home)) 1437 1438 def valid_mpi_path(mpi_home): 1439 exists = ( 1440 os.path.exists(os.path.join(mpi_home, 'include')) and 1441 (os.path.exists(os.path.join(mpi_home, 'lib')) or 1442 os.path.exists(os.path.join(mpi_home, 'lib64')) or 1443 os.path.exists(os.path.join(mpi_home, 'lib32')))) 1444 if not exists: 1445 print( 1446 'Invalid path to the MPI Toolkit. %s or %s or %s or %s cannot be found' 1447 % (os.path.join(mpi_home, 'include'), 1448 os.path.exists(os.path.join(mpi_home, 'lib')), 1449 os.path.exists(os.path.join(mpi_home, 'lib64')), 1450 os.path.exists(os.path.join(mpi_home, 'lib32')))) 1451 return exists 1452 1453 _ = prompt_loop_or_load_from_env( 1454 environ_cp, 1455 var_name='MPI_HOME', 1456 var_default=default_mpi_home, 1457 ask_for_var='Please specify the MPI toolkit folder.', 1458 check_success=valid_mpi_path, 1459 error_msg='', 1460 suppress_default_error=True) 1461 1462 1463def set_other_mpi_vars(environ_cp): 1464 """Set other MPI related variables.""" 1465 # Link the MPI header files 1466 mpi_home = environ_cp.get('MPI_HOME') 1467 symlink_force('%s/include/mpi.h' % mpi_home, 'third_party/mpi/mpi.h') 1468 1469 # Determine if we use OpenMPI or MVAPICH, these require different header files 1470 # to be included here to make bazel dependency checker happy 1471 if os.path.exists(os.path.join(mpi_home, 'include/mpi_portable_platform.h')): 1472 symlink_force( 1473 os.path.join(mpi_home, 'include/mpi_portable_platform.h'), 1474 'third_party/mpi/mpi_portable_platform.h') 1475 # TODO(gunan): avoid editing files in configure 1476 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=False', 1477 'MPI_LIB_IS_OPENMPI=True') 1478 else: 1479 # MVAPICH / MPICH 1480 symlink_force( 1481 os.path.join(mpi_home, 'include/mpio.h'), 'third_party/mpi/mpio.h') 1482 symlink_force( 1483 os.path.join(mpi_home, 'include/mpicxx.h'), 'third_party/mpi/mpicxx.h') 1484 # TODO(gunan): avoid editing files in configure 1485 sed_in_place('third_party/mpi/mpi.bzl', 'MPI_LIB_IS_OPENMPI=True', 1486 'MPI_LIB_IS_OPENMPI=False') 1487 1488 if os.path.exists(os.path.join(mpi_home, 'lib/libmpi.so')): 1489 symlink_force( 1490 os.path.join(mpi_home, 'lib/libmpi.so'), 'third_party/mpi/libmpi.so') 1491 elif os.path.exists(os.path.join(mpi_home, 'lib64/libmpi.so')): 1492 symlink_force( 1493 os.path.join(mpi_home, 'lib64/libmpi.so'), 'third_party/mpi/libmpi.so') 1494 elif os.path.exists(os.path.join(mpi_home, 'lib32/libmpi.so')): 1495 symlink_force( 1496 os.path.join(mpi_home, 'lib32/libmpi.so'), 'third_party/mpi/libmpi.so') 1497 1498 else: 1499 raise ValueError( 1500 'Cannot find the MPI library file in %s/lib or %s/lib64 or %s/lib32' % 1501 (mpi_home, mpi_home, mpi_home)) 1502 1503 1504def system_specific_test_config(env): 1505 """Add default test flags required for TF tests to bazelrc.""" 1506 write_to_bazelrc('test --flaky_test_attempts=3') 1507 write_to_bazelrc('test --test_size_filters=small,medium') 1508 write_to_bazelrc( 1509 'test --test_tag_filters=-benchmark-test,-no_oss,-oss_serial') 1510 write_to_bazelrc('test --build_tag_filters=-benchmark-test,-no_oss') 1511 if is_windows(): 1512 if env.get('TF_NEED_CUDA', None) == '1': 1513 write_to_bazelrc( 1514 'test --test_tag_filters=-no_windows,-no_windows_gpu,-no_gpu') 1515 write_to_bazelrc( 1516 'test --build_tag_filters=-no_windows,-no_windows_gpu,-no_gpu') 1517 else: 1518 write_to_bazelrc('test --test_tag_filters=-no_windows,-gpu') 1519 write_to_bazelrc('test --build_tag_filters=-no_windows,-gpu') 1520 elif is_macos(): 1521 write_to_bazelrc('test --test_tag_filters=-gpu,-nomac,-no_mac') 1522 write_to_bazelrc('test --build_tag_filters=-gpu,-nomac,-no_mac') 1523 elif is_linux(): 1524 if env.get('TF_NEED_CUDA', None) == '1': 1525 write_to_bazelrc('test --test_tag_filters=-no_gpu') 1526 write_to_bazelrc('test --build_tag_filters=-no_gpu') 1527 write_to_bazelrc('test --test_env=LD_LIBRARY_PATH') 1528 else: 1529 write_to_bazelrc('test --test_tag_filters=-gpu') 1530 write_to_bazelrc('test --build_tag_filters=-gpu') 1531 1532 1533def set_system_libs_flag(environ_cp): 1534 syslibs = environ_cp.get('TF_SYSTEM_LIBS', '') 1535 if syslibs: 1536 if ',' in syslibs: 1537 syslibs = ','.join(sorted(syslibs.split(','))) 1538 else: 1539 syslibs = ','.join(sorted(syslibs.split())) 1540 write_action_env_to_bazelrc('TF_SYSTEM_LIBS', syslibs) 1541 1542 if 'PREFIX' in environ_cp: 1543 write_to_bazelrc('build --define=PREFIX=%s' % environ_cp['PREFIX']) 1544 if 'LIBDIR' in environ_cp: 1545 write_to_bazelrc('build --define=LIBDIR=%s' % environ_cp['LIBDIR']) 1546 if 'INCLUDEDIR' in environ_cp: 1547 write_to_bazelrc('build --define=INCLUDEDIR=%s' % environ_cp['INCLUDEDIR']) 1548 1549 1550def set_windows_build_flags(environ_cp): 1551 """Set Windows specific build options.""" 1552 # The non-monolithic build is not supported yet 1553 write_to_bazelrc('build --config monolithic') 1554 # Suppress warning messages 1555 write_to_bazelrc('build --copt=-w --host_copt=-w') 1556 # Fix winsock2.h conflicts 1557 write_to_bazelrc( 1558 'build --copt=-DWIN32_LEAN_AND_MEAN --host_copt=-DWIN32_LEAN_AND_MEAN') 1559 # Output more verbose information when something goes wrong 1560 write_to_bazelrc('build --verbose_failures') 1561 # The host and target platforms are the same in Windows build. So we don't 1562 # have to distinct them. This avoids building the same targets twice. 1563 write_to_bazelrc('build --distinct_host_configuration=false') 1564 1565 if get_var( 1566 environ_cp, 'TF_OVERRIDE_EIGEN_STRONG_INLINE', 'Eigen strong inline', 1567 True, ('Would you like to override eigen strong inline for some C++ ' 1568 'compilation to reduce the compilation time?'), 1569 'Eigen strong inline overridden.', 'Not overriding eigen strong inline, ' 1570 'some compilations could take more than 20 mins.'): 1571 # Due to a known MSVC compiler issue 1572 # https://github.com/tensorflow/tensorflow/issues/10521 1573 # Overriding eigen strong inline speeds up the compiling of 1574 # conv_grad_ops_3d.cc and conv_ops_3d.cc by 20 minutes, 1575 # but this also hurts the performance. Let users decide what they want. 1576 write_to_bazelrc('build --define=override_eigen_strong_inline=true') 1577 1578 1579def config_info_line(name, help_text): 1580 """Helper function to print formatted help text for Bazel config options.""" 1581 print('\t--config=%-12s\t# %s' % (name, help_text)) 1582 1583 1584def configure_apple_bazel_rules(): 1585 """Configures Bazel rules for building on Apple platforms. 1586 1587 Enables analyzing and building Apple Bazel rules on Apple platforms. This 1588 function will only be executed if `is_macos()` is true. 1589 """ 1590 if not is_macos(): 1591 return 1592 for filepath in APPLE_BAZEL_FILES: 1593 print( 1594 'Configuring %s file to analyze and build Bazel rules on Apple platforms.' 1595 % filepath) 1596 existing_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath + '.apple') 1597 renamed_filepath = os.path.join(_TF_WORKSPACE_ROOT, filepath) 1598 os.rename(existing_filepath, renamed_filepath) 1599 if _TF_CURRENT_BAZEL_VERSION is None or _TF_CURRENT_BAZEL_VERSION < 23000: 1600 print( 1601 'Building Bazel rules on Apple platforms requires Bazel 0.23 or later.') 1602 1603 1604def main(): 1605 global _TF_WORKSPACE_ROOT 1606 global _TF_BAZELRC 1607 global _TF_CURRENT_BAZEL_VERSION 1608 1609 parser = argparse.ArgumentParser() 1610 parser.add_argument( 1611 '--workspace', 1612 type=str, 1613 default=os.path.abspath(os.path.dirname(__file__)), 1614 help='The absolute path to your active Bazel workspace.') 1615 args = parser.parse_args() 1616 1617 _TF_WORKSPACE_ROOT = args.workspace 1618 _TF_BAZELRC = os.path.join(_TF_WORKSPACE_ROOT, _TF_BAZELRC_FILENAME) 1619 1620 # Make a copy of os.environ to be clear when functions and getting and setting 1621 # environment variables. 1622 environ_cp = dict(os.environ) 1623 1624 current_bazel_version = check_bazel_version('0.19.0', '0.23.2') 1625 _TF_CURRENT_BAZEL_VERSION = convert_version_to_int(current_bazel_version) 1626 1627 reset_tf_configure_bazelrc() 1628 1629 cleanup_makefile() 1630 setup_python(environ_cp) 1631 1632 if is_windows(): 1633 environ_cp['TF_NEED_OPENCL_SYCL'] = '0' 1634 environ_cp['TF_NEED_COMPUTECPP'] = '0' 1635 environ_cp['TF_NEED_OPENCL'] = '0' 1636 environ_cp['TF_CUDA_CLANG'] = '0' 1637 environ_cp['TF_NEED_TENSORRT'] = '0' 1638 # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on 1639 # Windows. 1640 environ_cp['TF_DOWNLOAD_CLANG'] = '0' 1641 environ_cp['TF_NEED_MPI'] = '0' 1642 environ_cp['TF_SET_ANDROID_WORKSPACE'] = '0' 1643 1644 if is_macos(): 1645 environ_cp['TF_NEED_TENSORRT'] = '0' 1646 else: 1647 environ_cp['TF_CONFIGURE_APPLE_BAZEL_RULES'] = '0' 1648 1649 # The numpy package on ppc64le uses OpenBLAS which has multi-threading 1650 # issues that lead to incorrect answers. Set OMP_NUM_THREADS=1 at 1651 # runtime to allow the Tensorflow testcases which compare numpy 1652 # results to Tensorflow results to succeed. 1653 if is_ppc64le(): 1654 write_action_env_to_bazelrc('OMP_NUM_THREADS', 1) 1655 1656 xla_enabled_by_default = is_linux() 1657 set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support', 1658 xla_enabled_by_default, 'xla') 1659 1660 set_action_env_var(environ_cp, 'TF_NEED_OPENCL_SYCL', 'OpenCL SYCL', False) 1661 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': 1662 set_host_cxx_compiler(environ_cp) 1663 set_host_c_compiler(environ_cp) 1664 set_action_env_var(environ_cp, 'TF_NEED_COMPUTECPP', 'ComputeCPP', True) 1665 if environ_cp.get('TF_NEED_COMPUTECPP') == '1': 1666 set_computecpp_toolkit_path(environ_cp) 1667 else: 1668 set_trisycl_include_dir(environ_cp) 1669 1670 set_action_env_var(environ_cp, 'TF_NEED_ROCM', 'ROCm', False) 1671 if (environ_cp.get('TF_NEED_ROCM') == '1' and 1672 'LD_LIBRARY_PATH' in environ_cp and 1673 environ_cp.get('LD_LIBRARY_PATH') != '1'): 1674 write_action_env_to_bazelrc('LD_LIBRARY_PATH', 1675 environ_cp.get('LD_LIBRARY_PATH')) 1676 1677 set_action_env_var(environ_cp, 'TF_NEED_CUDA', 'CUDA', False) 1678 if (environ_cp.get('TF_NEED_CUDA') == '1' and 1679 'TF_CUDA_CONFIG_REPO' not in environ_cp): 1680 set_tf_cuda_version(environ_cp) 1681 set_tf_cudnn_version(environ_cp) 1682 if is_linux(): 1683 set_tf_tensorrt_install_path(environ_cp) 1684 set_tf_nccl_install_path(environ_cp) 1685 1686 set_tf_cuda_compute_capabilities(environ_cp) 1687 if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get( 1688 'LD_LIBRARY_PATH') != '1': 1689 write_action_env_to_bazelrc('LD_LIBRARY_PATH', 1690 environ_cp.get('LD_LIBRARY_PATH')) 1691 1692 set_tf_cuda_clang(environ_cp) 1693 if environ_cp.get('TF_CUDA_CLANG') == '1': 1694 # Ask whether we should download the clang toolchain. 1695 set_tf_download_clang(environ_cp) 1696 if environ_cp.get('TF_DOWNLOAD_CLANG') != '1': 1697 # Set up which clang we should use as the cuda / host compiler. 1698 set_clang_cuda_compiler_path(environ_cp) 1699 else: 1700 # Use downloaded LLD for linking. 1701 write_to_bazelrc('build:cuda_clang --config=download_clang_use_lld') 1702 write_to_bazelrc('test:cuda_clang --config=download_clang_use_lld') 1703 else: 1704 # Set up which gcc nvcc should use as the host compiler 1705 # No need to set this on Windows 1706 if not is_windows(): 1707 set_gcc_host_compiler_path(environ_cp) 1708 set_other_cuda_vars(environ_cp) 1709 else: 1710 # CUDA not required. Ask whether we should download the clang toolchain and 1711 # use it for the CPU build. 1712 set_tf_download_clang(environ_cp) 1713 if environ_cp.get('TF_DOWNLOAD_CLANG') == '1': 1714 write_to_bazelrc('build --config=download_clang') 1715 write_to_bazelrc('test --config=download_clang') 1716 1717 # SYCL / ROCm / CUDA are mutually exclusive. 1718 # At most 1 GPU platform can be configured. 1719 gpu_platform_count = 0 1720 if environ_cp.get('TF_NEED_OPENCL_SYCL') == '1': 1721 gpu_platform_count += 1 1722 if environ_cp.get('TF_NEED_ROCM') == '1': 1723 gpu_platform_count += 1 1724 if environ_cp.get('TF_NEED_CUDA') == '1': 1725 gpu_platform_count += 1 1726 if gpu_platform_count >= 2: 1727 raise UserInputError('SYCL / CUDA / ROCm are mututally exclusive. ' 1728 'At most 1 GPU platform can be configured.') 1729 1730 set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False) 1731 if environ_cp.get('TF_NEED_MPI') == '1': 1732 set_mpi_home(environ_cp) 1733 set_other_mpi_vars(environ_cp) 1734 1735 set_cc_opt_flags(environ_cp) 1736 set_system_libs_flag(environ_cp) 1737 if is_windows(): 1738 set_windows_build_flags(environ_cp) 1739 1740 # Add a config option to build TensorFlow 2.0 API. 1741 write_to_bazelrc('build:v2 --define=tf_api_version=2') 1742 1743 if get_var(environ_cp, 'TF_SET_ANDROID_WORKSPACE', 'android workspace', False, 1744 ('Would you like to interactively configure ./WORKSPACE for ' 1745 'Android builds?'), 'Searching for NDK and SDK installations.', 1746 'Not configuring the WORKSPACE for Android builds.'): 1747 create_android_ndk_rule(environ_cp) 1748 create_android_sdk_rule(environ_cp) 1749 1750 system_specific_test_config(os.environ) 1751 1752 if get_var( 1753 environ_cp, 'TF_CONFIGURE_APPLE_BAZEL_RULES', 1754 'Configure Bazel rules for Apple platforms', False, 1755 ('Would you like to configure Bazel rules for building on Apple platforms?' 1756 ), 'Configuring Bazel rules for Apple platforms.', 1757 'Not configuring Bazel rules for Apple platforms.'): 1758 configure_apple_bazel_rules() 1759 1760 print('Preconfigured Bazel build configs. You can use any of the below by ' 1761 'adding "--config=<>" to your build command. See .bazelrc for more ' 1762 'details.') 1763 config_info_line('mkl', 'Build with MKL support.') 1764 config_info_line('monolithic', 'Config for mostly static monolithic build.') 1765 config_info_line('gdr', 'Build with GDR support.') 1766 config_info_line('verbs', 'Build with libverbs support.') 1767 config_info_line('ngraph', 'Build with Intel nGraph support.') 1768 config_info_line('numa', 'Build with NUMA support.') 1769 config_info_line( 1770 'dynamic_kernels', 1771 '(Experimental) Build kernels into separate shared objects.') 1772 1773 print('Preconfigured Bazel build configs to DISABLE default on features:') 1774 config_info_line('noaws', 'Disable AWS S3 filesystem support.') 1775 config_info_line('nogcp', 'Disable GCP support.') 1776 config_info_line('nohdfs', 'Disable HDFS support.') 1777 config_info_line('noignite', 'Disable Apache Ignite support.') 1778 config_info_line('nokafka', 'Disable Apache Kafka support.') 1779 config_info_line('nonccl', 'Disable NVIDIA NCCL support.') 1780 1781 1782if __name__ == '__main__': 1783 main() 1784