1# Copyright 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for toolchain build."""
5
6from __future__ import print_function
7
8__author__ = 'asharif@google.com (Ahmad Sharif)'
9
10from contextlib import contextmanager
11import os
12import re
13import shutil
14import sys
15import traceback
16
17import command_executer
18import logger
19
20CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
21TOOLCHAIN_UTILS_PATH = '~/trunk/src/platform/dev/toolchain_utils.sh'
22
23
24def GetChromeOSVersionFromLSBVersion(lsb_version):
25  """Get Chromeos version from Lsb version."""
26  ce = command_executer.GetCommandExecuter()
27  command = ('git ls-remote '
28             'https://chromium.googlesource.com/chromiumos/manifest.git')
29  ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
30  assert ret == 0, 'Command %s failed' % command
31  lower = []
32  for line in out.splitlines():
33    mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
34    if mo:
35      revision = int(mo.group(1))
36      build = int(mo.group(2))
37      lsb_build = int(lsb_version.split('.')[0])
38      if lsb_build > build:
39        lower.append(revision)
40  lower = sorted(lower)
41  if lower:
42    return 'R%d-%s' % (lower[-1] + 1, lsb_version)
43  else:
44    return 'Unknown'
45
46
47def ApplySubs(string, *substitutions):
48  for pattern, replacement in substitutions:
49    string = re.sub(pattern, replacement, string)
50  return string
51
52
53def UnitToNumber(unit_num, base=1000):
54  """Convert a number with unit to float."""
55  unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
56  unit_num = unit_num.lower()
57  mo = re.search(r'(\d*)(.+)?', unit_num)
58  number = mo.group(1)
59  unit = mo.group(2)
60  if not unit:
61    return float(number)
62  for k, v in unit_dict.items():
63    if k.startswith(unit):
64      return float(number) * v
65  raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
66
67
68def GetFilenameFromString(string):
69  return ApplySubs(
70      string,
71      (r'/', '__'),
72      (r'\s', '_'),
73      (r'[\\$="?^]', ''),)
74
75
76def GetRoot(scr_name):
77  """Break up pathname into (dir+name)."""
78  abs_path = os.path.abspath(scr_name)
79  return (os.path.dirname(abs_path), os.path.basename(abs_path))
80
81
82def GetChromeOSKeyFile(chromeos_root):
83  return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
84                      'ssh_keys', 'testing_rsa')
85
86
87def GetChrootPath(chromeos_root):
88  return os.path.join(chromeos_root, 'chroot')
89
90
91def GetInsideChrootPath(chromeos_root, file_path):
92  if not file_path.startswith(GetChrootPath(chromeos_root)):
93    raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
94                       (file_path, chromeos_root))
95  return file_path[len(GetChrootPath(chromeos_root)):]
96
97
98def GetOutsideChrootPath(chromeos_root, file_path):
99  return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
100
101
102def FormatQuotedCommand(command):
103  return ApplySubs(command, ('"', r'\"'))
104
105
106def FormatCommands(commands):
107  return ApplySubs(
108      str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
109
110
111def GetImageDir(chromeos_root, board):
112  return os.path.join(chromeos_root, 'src', 'build', 'images', board)
113
114
115def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
116  image_dir = GetImageDir(chromeos_root, board)
117  latest_image_dir = os.path.join(image_dir, 'latest')
118  latest_image_dir = os.path.realpath(latest_image_dir)
119  latest_image_dir = os.path.basename(latest_image_dir)
120  retval = 0
121  with WorkingDirectory(image_dir):
122    command = 'ln -sf -T %s %s' % (latest_image_dir, label)
123    ce = command_executer.GetCommandExecuter()
124    retval = ce.RunCommand(command)
125    if retval:
126      return retval
127    if vanilla_path:
128      command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
129      retval2 = ce.RunCommand(command)
130      return retval2
131  return retval
132
133
134def DoesLabelExist(chromeos_root, board, label):
135  image_label = os.path.join(GetImageDir(chromeos_root, board), label)
136  return os.path.exists(image_label)
137
138
139def GetBuildPackagesCommand(board, usepkg=False, debug=False):
140  if usepkg:
141    usepkg_flag = '--usepkg'
142  else:
143    usepkg_flag = '--nousepkg'
144  if debug:
145    withdebug_flag = '--withdebug'
146  else:
147    withdebug_flag = '--nowithdebug'
148  return ('%s/build_packages %s --withdev --withtest --withautotest '
149          '--skip_toolchain_update %s --board=%s '
150          '--accept_licenses=@CHROMEOS' % (CHROMEOS_SCRIPTS_DIR, usepkg_flag,
151                                           withdebug_flag, board))
152
153
154def GetBuildImageCommand(board, dev=False):
155  dev_args = ''
156  if dev:
157    dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
158  return ('%s/build_image --board=%s %s test' % (CHROMEOS_SCRIPTS_DIR, board,
159                                                 dev_args))
160
161
162def GetSetupBoardCommand(board,
163                         gcc_version=None,
164                         binutils_version=None,
165                         usepkg=None,
166                         force=None):
167  """Get setup_board command."""
168  options = []
169
170  if gcc_version:
171    options.append('--gcc_version=%s' % gcc_version)
172
173  if binutils_version:
174    options.append('--binutils_version=%s' % binutils_version)
175
176  if usepkg:
177    options.append('--usepkg')
178  else:
179    options.append('--nousepkg')
180
181  if force:
182    options.append('--force')
183
184  options.append('--accept_licenses=@CHROMEOS')
185
186  return ('%s/setup_board --board=%s %s' % (CHROMEOS_SCRIPTS_DIR, board,
187                                            ' '.join(options)))
188
189
190def CanonicalizePath(path):
191  path = os.path.expanduser(path)
192  path = os.path.realpath(path)
193  return path
194
195
196def GetCtargetFromBoard(board, chromeos_root):
197  """Get Ctarget from board."""
198  base_board = board.split('_')[0]
199  command = ('source %s; get_ctarget_from_board %s' % (TOOLCHAIN_UTILS_PATH,
200                                                       base_board))
201  ce = command_executer.GetCommandExecuter()
202  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
203  if ret != 0:
204    raise ValueError('Board %s is invalid!' % board)
205  # Remove ANSI escape sequences.
206  out = StripANSIEscapeSequences(out)
207  return out.strip()
208
209
210def GetArchFromBoard(board, chromeos_root):
211  """Get Arch from board."""
212  base_board = board.split('_')[0]
213  command = ('source %s; get_board_arch %s' % (TOOLCHAIN_UTILS_PATH,
214                                               base_board))
215  ce = command_executer.GetCommandExecuter()
216  ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
217  if ret != 0:
218    raise ValueError('Board %s is invalid!' % board)
219  # Remove ANSI escape sequences.
220  out = StripANSIEscapeSequences(out)
221  return out.strip()
222
223
224def GetGccLibsDestForBoard(board, chromeos_root):
225  """Get gcc libs destination from board."""
226  arch = GetArchFromBoard(board, chromeos_root)
227  if arch == 'x86':
228    return '/build/%s/usr/lib/gcc/' % board
229  if arch == 'amd64':
230    return '/build/%s/usr/lib64/gcc/' % board
231  if arch == 'arm':
232    return '/build/%s/usr/lib/gcc/' % board
233  if arch == 'arm64':
234    return '/build/%s/usr/lib/gcc/' % board
235  raise ValueError('Arch %s is invalid!' % arch)
236
237
238def StripANSIEscapeSequences(string):
239  string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
240  return string
241
242
243def GetChromeSrcDir():
244  return 'var/cache/distfiles/target/chrome-src/src'
245
246
247def GetEnvStringFromDict(env_dict):
248  return ' '.join(["%s=\"%s\"" % var for var in env_dict.items()])
249
250
251def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
252  """Merge env string with dict."""
253  if not env_string.strip():
254    return GetEnvStringFromDict(env_dict)
255  override_env_list = []
256  ce = command_executer.GetCommandExecuter()
257  for k, v in env_dict.items():
258    v = v.strip("\"'")
259    if prepend:
260      new_env = "%s=\"%s $%s\"" % (k, v, k)
261    else:
262      new_env = "%s=\"$%s %s\"" % (k, k, v)
263    command = '; '.join([env_string, new_env, 'echo $%s' % k])
264    ret, out, _ = ce.RunCommandWOutput(command)
265    override_env_list.append('%s=%r' % (k, out.strip()))
266  ret = env_string + ' ' + ' '.join(override_env_list)
267  return ret.strip()
268
269
270def GetAllImages(chromeos_root, board):
271  ce = command_executer.GetCommandExecuter()
272  command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
273             (chromeos_root, board))
274  ret, out, _ = ce.RunCommandWOutput(command)
275  assert ret == 0, 'Could not run command: %s' % command
276  return out.splitlines()
277
278
279def IsFloat(text):
280  if text is None:
281    return False
282  try:
283    float(text)
284    return True
285  except ValueError:
286    return False
287
288
289def RemoveChromeBrowserObjectFiles(chromeos_root, board):
290  """Remove any object files from all the posible locations."""
291  out_dir = os.path.join(
292      GetChrootPath(chromeos_root),
293      'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
294  if os.path.exists(out_dir):
295    shutil.rmtree(out_dir)
296    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
297  out_dir = os.path.join(
298      GetChrootPath(chromeos_root),
299      'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
300  if os.path.exists(out_dir):
301    shutil.rmtree(out_dir)
302    logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
303
304
305@contextmanager
306def WorkingDirectory(new_dir):
307  """Get the working directory."""
308  old_dir = os.getcwd()
309  if old_dir != new_dir:
310    msg = 'cd %s' % new_dir
311    logger.GetLogger().LogCmd(msg)
312  os.chdir(new_dir)
313  yield new_dir
314  if old_dir != new_dir:
315    msg = 'cd %s' % old_dir
316    logger.GetLogger().LogCmd(msg)
317  os.chdir(old_dir)
318
319
320def HasGitStagedChanges(git_dir):
321  """Return True if git repository has staged changes."""
322  command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
323      git_dir)
324  return command_executer.GetCommandExecuter().RunCommand(
325      command, print_to_console=False)
326
327
328def HasGitUnstagedChanges(git_dir):
329  """Return True if git repository has un-staged changes."""
330  command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
331  return command_executer.GetCommandExecuter().RunCommand(
332      command, print_to_console=False)
333
334
335def HasGitUntrackedChanges(git_dir):
336  """Return True if git repository has un-tracked changes."""
337  command = ('cd {0} && test -z '
338             '$(git ls-files --exclude-standard --others)').format(git_dir)
339  return command_executer.GetCommandExecuter().RunCommand(
340      command, print_to_console=False)
341
342
343def GitGetCommitHash(git_dir, commit_symbolic_name):
344  """Return githash for the symbolic git commit.
345
346  For example, commit_symbolic_name could be
347  "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
348  hash for this symbolic name.
349
350  Args:
351    git_dir: a git working tree.
352    commit_symbolic_name: a symbolic name for a particular git commit.
353
354  Returns:
355    The git hash for the symbolic name or None if fails.
356  """
357
358  command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
359      git_dir, commit_symbolic_name)
360  rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
361      command, print_to_console=False)
362  if rv == 0:
363    return out.strip()
364  return None
365
366
367def IsGitTreeClean(git_dir):
368  """Test if git tree has no local changes.
369
370  Args:
371    git_dir: git tree directory.
372
373  Returns:
374    True if git dir is clean.
375  """
376  if HasGitStagedChanges(git_dir):
377    logger.GetLogger().LogWarning('Git tree has staged changes.')
378    return False
379  if HasGitUnstagedChanges(git_dir):
380    logger.GetLogger().LogWarning('Git tree has unstaged changes.')
381    return False
382  if HasGitUntrackedChanges(git_dir):
383    logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
384    return False
385  return True
386
387
388def GetGitChangesAsList(git_dir, path=None, staged=False):
389  """Get changed files as a list.
390
391  Args:
392    git_dir: git tree directory.
393    path: a relative path that is part of the tree directory, could be null.
394    staged: whether to include staged files as well.
395
396  Returns:
397    A list containing all the changed files.
398  """
399  command = 'cd {0} && git diff --name-only'.format(git_dir)
400  if staged:
401    command += ' --cached'
402  if path:
403    command += ' -- ' + path
404  _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
405      command, print_to_console=False)
406  rv = []
407  for line in out.splitlines():
408    rv.append(line)
409  return rv
410
411
412def IsChromeOsTree(chromeos_root):
413  return (os.path.isdir(
414      os.path.join(chromeos_root, 'src/third_party/chromiumos-overlay')) and
415          os.path.isdir(os.path.join(chromeos_root, 'manifest')))
416
417
418def DeleteChromeOsTree(chromeos_root, dry_run=False):
419  """Delete a ChromeOs tree *safely*.
420
421  Args:
422    chromeos_root: dir of the tree, could be a relative one (but be careful)
423    dry_run: only prints out the command if True
424
425  Returns:
426    True if everything is ok.
427  """
428  if not IsChromeOsTree(chromeos_root):
429    logger.GetLogger().LogWarning(
430        '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
431            chromeos_root))
432    return False
433  cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
434  if dry_run:
435    print(cmd0)
436  else:
437    if command_executer.GetCommandExecuter().RunCommand(
438        cmd0, print_to_console=True) != 0:
439      return False
440
441  cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
442          'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
443          'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME'
444         ).format(chromeos_root)
445  if dry_run:
446    print(cmd1)
447    return True
448
449  return command_executer.GetCommandExecuter().RunCommand(
450      cmd1, print_to_console=True) == 0
451
452
453def ApplyGerritPatches(chromeos_root, gerrit_patch_string,
454                       branch='cros/master'):
455  """Apply gerrit patches on a chromeos tree.
456
457  Args:
458    chromeos_root: chromeos tree path
459    gerrit_patch_string: a patch string just like the one gives to cbuildbot,
460    'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
461    branch: the tree based on which to apply the patches.
462
463  Returns:
464    True if success.
465  """
466
467  ### First of all, we need chromite libs
468  sys.path.append(os.path.join(chromeos_root, 'chromite'))
469  # Imports below are ok after modifying path to add chromite.
470  # Pylint cannot detect that and complains.
471  # pylint: disable=import-error
472  from lib import git
473  from lib import gerrit
474  manifest = git.ManifestCheckout(chromeos_root)
475  patch_list = gerrit_patch_string.split(' ')
476  ### This takes time, print log information.
477  logger.GetLogger().LogOutput('Retrieving patch information from server ...')
478  patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
479  for pi in patch_info_list:
480    project_checkout = manifest.FindCheckout(pi.project, strict=False)
481    if not project_checkout:
482      logger.GetLogger().LogError(
483          'Failed to find patch project "{project}" in manifest.'.format(
484              project=pi.project))
485      return False
486
487    pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
488    try:
489      project_git_path = project_checkout.GetPath(absolute=True)
490      logger.GetLogger().LogOutput(
491          'Applying patch "{0}" in "{1}" ...'.format(pi_str, project_git_path))
492      pi.Apply(project_git_path, branch, trivial=False)
493    except Exception:
494      traceback.print_exc(file=sys.stdout)
495      logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
496      return False
497  return True
498
499
500def BooleanPrompt(prompt='Do you want to continue?',
501                  default=True,
502                  true_value='yes',
503                  false_value='no',
504                  prolog=None):
505  """Helper function for processing boolean choice prompts.
506
507  Args:
508    prompt: The question to present to the user.
509    default: Boolean to return if the user just presses enter.
510    true_value: The text to display that represents a True returned.
511    false_value: The text to display that represents a False returned.
512    prolog: The text to display before prompt.
513
514  Returns:
515    True or False.
516  """
517  true_value, false_value = true_value.lower(), false_value.lower()
518  true_text, false_text = true_value, false_value
519  if true_value == false_value:
520    raise ValueError(
521        'true_value and false_value must differ: got %r' % true_value)
522
523  if default:
524    true_text = true_text[0].upper() + true_text[1:]
525  else:
526    false_text = false_text[0].upper() + false_text[1:]
527
528  prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
529
530  if prolog:
531    prompt = ('\n%s\n%s' % (prolog, prompt))
532
533  while True:
534    try:
535      response = raw_input(prompt).lower()
536    except EOFError:
537      # If the user hits CTRL+D, or stdin is disabled, use the default.
538      print()
539      response = None
540    except KeyboardInterrupt:
541      # If the user hits CTRL+C, just exit the process.
542      print()
543      print('CTRL+C detected; exiting')
544      sys.exit()
545
546    if not response:
547      return default
548    if true_value.startswith(response):
549      if not false_value.startswith(response):
550        return True
551      # common prefix between the two...
552    elif false_value.startswith(response):
553      return False
554
555
556# pylint: disable=unused-argument
557def rgb2short(r, g, b):
558  """Converts RGB values to xterm-256 color."""
559
560  redcolor = [255, 124, 160, 196, 9]
561  greencolor = [255, 118, 82, 46, 10]
562
563  if g == 0:
564    return redcolor[r / 52]
565  if r == 0:
566    return greencolor[g / 52]
567  return 4
568