1 # Copyright 2013 The Chromium OS Authors. All rights reserved. 2 # Use of this source code is governed by a BSD-style license that can be 3 # found in the LICENSE file. 4 """Utilities for toolchain build.""" 5 6 from __future__ import print_function 7 8 __author__ = 'asharif@google.com (Ahmad Sharif)' 9 10 from contextlib import contextmanager 11 import os 12 import re 13 import shutil 14 import sys 15 import traceback 16 17 import command_executer 18 import logger 19 20 CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts' 21 TOOLCHAIN_UTILS_PATH = '~/trunk/src/platform/dev/toolchain_utils.sh' 22 23 24 def GetChromeOSVersionFromLSBVersion(lsb_version): 25 """Get Chromeos version from Lsb version.""" 26 ce = command_executer.GetCommandExecuter() 27 command = ('git ls-remote ' 28 'https://chromium.googlesource.com/chromiumos/manifest.git') 29 ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False) 30 assert ret == 0, 'Command %s failed' % command 31 lower = [] 32 for line in out.splitlines(): 33 mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line) 34 if mo: 35 revision = int(mo.group(1)) 36 build = int(mo.group(2)) 37 lsb_build = int(lsb_version.split('.')[0]) 38 if lsb_build > build: 39 lower.append(revision) 40 lower = sorted(lower) 41 if lower: 42 return 'R%d-%s' % (lower[-1] + 1, lsb_version) 43 else: 44 return 'Unknown' 45 46 47 def ApplySubs(string, *substitutions): 48 for pattern, replacement in substitutions: 49 string = re.sub(pattern, replacement, string) 50 return string 51 52 53 def UnitToNumber(unit_num, base=1000): 54 """Convert a number with unit to float.""" 55 unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3} 56 unit_num = unit_num.lower() 57 mo = re.search(r'(\d*)(.+)?', unit_num) 58 number = mo.group(1) 59 unit = mo.group(2) 60 if not unit: 61 return float(number) 62 for k, v in unit_dict.items(): 63 if k.startswith(unit): 64 return float(number) * v 65 raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num)) 66 67 68 def GetFilenameFromString(string): 69 return ApplySubs(string, (r'/', '__'), (r'\s', '_'), (r'[\\$="?^]', ''),) 70 71 72 def GetRoot(scr_name): 73 """Break up pathname into (dir+name).""" 74 abs_path = os.path.abspath(scr_name) 75 return (os.path.dirname(abs_path), os.path.basename(abs_path)) 76 77 78 def GetChromeOSKeyFile(chromeos_root): 79 return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts', 80 'ssh_keys', 'testing_rsa') 81 82 83 def GetChrootPath(chromeos_root): 84 return os.path.join(chromeos_root, 'chroot') 85 86 87 def GetInsideChrootPath(chromeos_root, file_path): 88 if not file_path.startswith(GetChrootPath(chromeos_root)): 89 raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" % 90 (file_path, chromeos_root)) 91 return file_path[len(GetChrootPath(chromeos_root)):] 92 93 94 def GetOutsideChrootPath(chromeos_root, file_path): 95 return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/')) 96 97 98 def FormatQuotedCommand(command): 99 return ApplySubs(command, ('"', r'\"')) 100 101 102 def FormatCommands(commands): 103 return ApplySubs( 104 str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n')) 105 106 107 def GetImageDir(chromeos_root, board): 108 return os.path.join(chromeos_root, 'src', 'build', 'images', board) 109 110 111 def LabelLatestImage(chromeos_root, board, label, vanilla_path=None): 112 image_dir = GetImageDir(chromeos_root, board) 113 latest_image_dir = os.path.join(image_dir, 'latest') 114 latest_image_dir = os.path.realpath(latest_image_dir) 115 latest_image_dir = os.path.basename(latest_image_dir) 116 retval = 0 117 with WorkingDirectory(image_dir): 118 command = 'ln -sf -T %s %s' % (latest_image_dir, label) 119 ce = command_executer.GetCommandExecuter() 120 retval = ce.RunCommand(command) 121 if retval: 122 return retval 123 if vanilla_path: 124 command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla') 125 retval2 = ce.RunCommand(command) 126 return retval2 127 return retval 128 129 130 def DoesLabelExist(chromeos_root, board, label): 131 image_label = os.path.join(GetImageDir(chromeos_root, board), label) 132 return os.path.exists(image_label) 133 134 135 def GetBuildPackagesCommand(board, usepkg=False, debug=False): 136 if usepkg: 137 usepkg_flag = '--usepkg' 138 else: 139 usepkg_flag = '--nousepkg' 140 if debug: 141 withdebug_flag = '--withdebug' 142 else: 143 withdebug_flag = '--nowithdebug' 144 return ('%s/build_packages %s --withdev --withtest --withautotest ' 145 '--skip_toolchain_update %s --board=%s ' 146 '--accept_licenses=@CHROMEOS' % 147 (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board)) 148 149 150 def GetBuildImageCommand(board, dev=False): 151 dev_args = '' 152 if dev: 153 dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs' 154 return ('%s/build_image --board=%s %s test' % 155 (CHROMEOS_SCRIPTS_DIR, board, dev_args)) 156 157 158 def GetSetupBoardCommand(board, 159 gcc_version=None, 160 binutils_version=None, 161 usepkg=None, 162 force=None): 163 """Get setup_board command.""" 164 options = [] 165 166 if gcc_version: 167 options.append('--gcc_version=%s' % gcc_version) 168 169 if binutils_version: 170 options.append('--binutils_version=%s' % binutils_version) 171 172 if usepkg: 173 options.append('--usepkg') 174 else: 175 options.append('--nousepkg') 176 177 if force: 178 options.append('--force') 179 180 options.append('--accept_licenses=@CHROMEOS') 181 182 return ('%s/setup_board --board=%s %s' % 183 (CHROMEOS_SCRIPTS_DIR, board, ' '.join(options))) 184 185 186 def CanonicalizePath(path): 187 path = os.path.expanduser(path) 188 path = os.path.realpath(path) 189 return path 190 191 192 def GetCtargetFromBoard(board, chromeos_root): 193 """Get Ctarget from board.""" 194 base_board = board.split('_')[0] 195 command = ('source %s; get_ctarget_from_board %s' % 196 (TOOLCHAIN_UTILS_PATH, base_board)) 197 ce = command_executer.GetCommandExecuter() 198 ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) 199 if ret != 0: 200 raise ValueError('Board %s is invalid!' % board) 201 # Remove ANSI escape sequences. 202 out = StripANSIEscapeSequences(out) 203 return out.strip() 204 205 206 def GetArchFromBoard(board, chromeos_root): 207 """Get Arch from board.""" 208 base_board = board.split('_')[0] 209 command = ('source %s; get_board_arch %s' % 210 (TOOLCHAIN_UTILS_PATH, base_board)) 211 ce = command_executer.GetCommandExecuter() 212 ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command) 213 if ret != 0: 214 raise ValueError('Board %s is invalid!' % board) 215 # Remove ANSI escape sequences. 216 out = StripANSIEscapeSequences(out) 217 return out.strip() 218 219 220 def GetGccLibsDestForBoard(board, chromeos_root): 221 """Get gcc libs destination from board.""" 222 arch = GetArchFromBoard(board, chromeos_root) 223 if arch == 'x86': 224 return '/build/%s/usr/lib/gcc/' % board 225 if arch == 'amd64': 226 return '/build/%s/usr/lib64/gcc/' % board 227 if arch == 'arm': 228 return '/build/%s/usr/lib/gcc/' % board 229 if arch == 'arm64': 230 return '/build/%s/usr/lib/gcc/' % board 231 raise ValueError('Arch %s is invalid!' % arch) 232 233 234 def StripANSIEscapeSequences(string): 235 string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string) 236 return string 237 238 239 def GetChromeSrcDir(): 240 return 'var/cache/distfiles/target/chrome-src/src' 241 242 243 def GetEnvStringFromDict(env_dict): 244 return ' '.join(["%s=\"%s\"" % var for var in env_dict.items()]) 245 246 247 def MergeEnvStringWithDict(env_string, env_dict, prepend=True): 248 """Merge env string with dict.""" 249 if not env_string.strip(): 250 return GetEnvStringFromDict(env_dict) 251 override_env_list = [] 252 ce = command_executer.GetCommandExecuter() 253 for k, v in env_dict.items(): 254 v = v.strip("\"'") 255 if prepend: 256 new_env = "%s=\"%s $%s\"" % (k, v, k) 257 else: 258 new_env = "%s=\"$%s %s\"" % (k, k, v) 259 command = '; '.join([env_string, new_env, 'echo $%s' % k]) 260 ret, out, _ = ce.RunCommandWOutput(command) 261 override_env_list.append('%s=%r' % (k, out.strip())) 262 ret = env_string + ' ' + ' '.join(override_env_list) 263 return ret.strip() 264 265 266 def GetAllImages(chromeos_root, board): 267 ce = command_executer.GetCommandExecuter() 268 command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' % 269 (chromeos_root, board)) 270 ret, out, _ = ce.RunCommandWOutput(command) 271 assert ret == 0, 'Could not run command: %s' % command 272 return out.splitlines() 273 274 275 def IsFloat(text): 276 if text is None: 277 return False 278 try: 279 float(text) 280 return True 281 except ValueError: 282 return False 283 284 285 def RemoveChromeBrowserObjectFiles(chromeos_root, board): 286 """Remove any object files from all the posible locations.""" 287 out_dir = os.path.join( 288 GetChrootPath(chromeos_root), 289 'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board) 290 if os.path.exists(out_dir): 291 shutil.rmtree(out_dir) 292 logger.GetLogger().LogCmd('rm -rf %s' % out_dir) 293 out_dir = os.path.join( 294 GetChrootPath(chromeos_root), 295 'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board) 296 if os.path.exists(out_dir): 297 shutil.rmtree(out_dir) 298 logger.GetLogger().LogCmd('rm -rf %s' % out_dir) 299 300 301 @contextmanager 302 def WorkingDirectory(new_dir): 303 """Get the working directory.""" 304 old_dir = os.getcwd() 305 if old_dir != new_dir: 306 msg = 'cd %s' % new_dir 307 logger.GetLogger().LogCmd(msg) 308 os.chdir(new_dir) 309 yield new_dir 310 if old_dir != new_dir: 311 msg = 'cd %s' % old_dir 312 logger.GetLogger().LogCmd(msg) 313 os.chdir(old_dir) 314 315 316 def HasGitStagedChanges(git_dir): 317 """Return True if git repository has staged changes.""" 318 command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format( 319 git_dir) 320 return command_executer.GetCommandExecuter().RunCommand( 321 command, 322 print_to_console=False) 323 324 325 def HasGitUnstagedChanges(git_dir): 326 """Return True if git repository has un-staged changes.""" 327 command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir) 328 return command_executer.GetCommandExecuter().RunCommand( 329 command, 330 print_to_console=False) 331 332 333 def HasGitUntrackedChanges(git_dir): 334 """Return True if git repository has un-tracked changes.""" 335 command = ('cd {0} && test -z ' 336 '$(git ls-files --exclude-standard --others)').format(git_dir) 337 return command_executer.GetCommandExecuter().RunCommand( 338 command, 339 print_to_console=False) 340 341 342 def GitGetCommitHash(git_dir, commit_symbolic_name): 343 """Return githash for the symbolic git commit. 344 345 For example, commit_symbolic_name could be 346 "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git 347 hash for this symbolic name. 348 349 Args: 350 git_dir: a git working tree. 351 commit_symbolic_name: a symbolic name for a particular git commit. 352 353 Returns: 354 The git hash for the symbolic name or None if fails. 355 """ 356 357 command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format( 358 git_dir, commit_symbolic_name) 359 rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 360 command, 361 print_to_console=False) 362 if rv == 0: 363 return out.strip() 364 return None 365 366 367 def IsGitTreeClean(git_dir): 368 """Test if git tree has no local changes. 369 370 Args: 371 git_dir: git tree directory. 372 373 Returns: 374 True if git dir is clean. 375 """ 376 if HasGitStagedChanges(git_dir): 377 logger.GetLogger().LogWarning('Git tree has staged changes.') 378 return False 379 if HasGitUnstagedChanges(git_dir): 380 logger.GetLogger().LogWarning('Git tree has unstaged changes.') 381 return False 382 if HasGitUntrackedChanges(git_dir): 383 logger.GetLogger().LogWarning('Git tree has un-tracked changes.') 384 return False 385 return True 386 387 388 def GetGitChangesAsList(git_dir, path=None, staged=False): 389 """Get changed files as a list. 390 391 Args: 392 git_dir: git tree directory. 393 path: a relative path that is part of the tree directory, could be null. 394 staged: whether to include staged files as well. 395 396 Returns: 397 A list containing all the changed files. 398 """ 399 command = 'cd {0} && git diff --name-only'.format(git_dir) 400 if staged: 401 command += ' --cached' 402 if path: 403 command += ' -- ' + path 404 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 405 command, 406 print_to_console=False) 407 rv = [] 408 for line in out.splitlines(): 409 rv.append(line) 410 return rv 411 412 413 def IsChromeOsTree(chromeos_root): 414 return (os.path.isdir(os.path.join(chromeos_root, 415 'src/third_party/chromiumos-overlay')) and 416 os.path.isdir(os.path.join(chromeos_root, 'manifest'))) 417 418 419 def DeleteChromeOsTree(chromeos_root, dry_run=False): 420 """Delete a ChromeOs tree *safely*. 421 422 Args: 423 chromeos_root: dir of the tree, could be a relative one (but be careful) 424 dry_run: only prints out the command if True 425 426 Returns: 427 True if everything is ok. 428 """ 429 if not IsChromeOsTree(chromeos_root): 430 logger.GetLogger().LogWarning( 431 '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format( 432 chromeos_root)) 433 return False 434 cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root) 435 if dry_run: 436 print(cmd0) 437 else: 438 if command_executer.GetCommandExecuter().RunCommand( 439 cmd0, 440 print_to_console=True) != 0: 441 return False 442 443 cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && ' 444 'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && ' 445 'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME').format( 446 chromeos_root) 447 if dry_run: 448 print(cmd1) 449 return True 450 451 return command_executer.GetCommandExecuter().RunCommand( 452 cmd1, 453 print_to_console=True) == 0 454 455 456 def ApplyGerritPatches(chromeos_root, 457 gerrit_patch_string, 458 branch='cros/master'): 459 """Apply gerrit patches on a chromeos tree. 460 461 Args: 462 chromeos_root: chromeos tree path 463 gerrit_patch_string: a patch string just like the one gives to cbuildbot, 464 'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch. 465 branch: the tree based on which to apply the patches. 466 467 Returns: 468 True if success. 469 """ 470 471 ### First of all, we need chromite libs 472 sys.path.append(os.path.join(chromeos_root, 'chromite')) 473 # Imports below are ok after modifying path to add chromite. 474 # Pylint cannot detect that and complains. 475 # pylint: disable=import-error 476 from lib import git 477 from lib import gerrit 478 manifest = git.ManifestCheckout(chromeos_root) 479 patch_list = gerrit_patch_string.split(' ') 480 ### This takes time, print log information. 481 logger.GetLogger().LogOutput('Retrieving patch information from server ...') 482 patch_info_list = gerrit.GetGerritPatchInfo(patch_list) 483 for pi in patch_info_list: 484 project_checkout = manifest.FindCheckout(pi.project, strict=False) 485 if not project_checkout: 486 logger.GetLogger().LogError( 487 'Failed to find patch project "{project}" in manifest.'.format( 488 project=pi.project)) 489 return False 490 491 pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref) 492 try: 493 project_git_path = project_checkout.GetPath(absolute=True) 494 logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format( 495 pi_str, project_git_path)) 496 pi.Apply(project_git_path, branch, trivial=False) 497 except Exception: 498 traceback.print_exc(file=sys.stdout) 499 logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str)) 500 return False 501 return True 502 503 504 def BooleanPrompt(prompt='Do you want to continue?', 505 default=True, 506 true_value='yes', 507 false_value='no', 508 prolog=None): 509 """Helper function for processing boolean choice prompts. 510 511 Args: 512 prompt: The question to present to the user. 513 default: Boolean to return if the user just presses enter. 514 true_value: The text to display that represents a True returned. 515 false_value: The text to display that represents a False returned. 516 prolog: The text to display before prompt. 517 518 Returns: 519 True or False. 520 """ 521 true_value, false_value = true_value.lower(), false_value.lower() 522 true_text, false_text = true_value, false_value 523 if true_value == false_value: 524 raise ValueError('true_value and false_value must differ: got %r' % 525 true_value) 526 527 if default: 528 true_text = true_text[0].upper() + true_text[1:] 529 else: 530 false_text = false_text[0].upper() + false_text[1:] 531 532 prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text)) 533 534 if prolog: 535 prompt = ('\n%s\n%s' % (prolog, prompt)) 536 537 while True: 538 try: 539 response = raw_input(prompt).lower() 540 except EOFError: 541 # If the user hits CTRL+D, or stdin is disabled, use the default. 542 print() 543 response = None 544 except KeyboardInterrupt: 545 # If the user hits CTRL+C, just exit the process. 546 print() 547 print('CTRL+C detected; exiting') 548 sys.exit() 549 550 if not response: 551 return default 552 if true_value.startswith(response): 553 if not false_value.startswith(response): 554 return True 555 # common prefix between the two... 556 elif false_value.startswith(response): 557 return False 558 559 def rgb2short(r, g, b): 560 """ Converts RGB values to xterm-256 color. """ 561 562 redcolor = [255, 124, 160, 196, 9 ] 563 greencolor = [255, 118, 82, 46, 10 ] 564 565 if g == 0: 566 return redcolor[r/52] 567 if r == 0: 568 return greencolor[g/52] 569 return 4 570