• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 # Copyright 2013 The Chromium OS Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
4 """Utilities for toolchain build."""
5 
6 from __future__ import print_function
7 
8 __author__ = 'asharif@google.com (Ahmad Sharif)'
9 
10 from contextlib import contextmanager
11 import os
12 import re
13 import shutil
14 import sys
15 import traceback
16 
17 import command_executer
18 import logger
19 
20 CHROMEOS_SCRIPTS_DIR = '~/trunk/src/scripts'
21 TOOLCHAIN_UTILS_PATH = '~/trunk/src/platform/dev/toolchain_utils.sh'
22 
23 
24 def GetChromeOSVersionFromLSBVersion(lsb_version):
25   """Get Chromeos version from Lsb version."""
26   ce = command_executer.GetCommandExecuter()
27   command = ('git ls-remote '
28              'https://chromium.googlesource.com/chromiumos/manifest.git')
29   ret, out, _ = ce.RunCommandWOutput(command, print_to_console=False)
30   assert ret == 0, 'Command %s failed' % command
31   lower = []
32   for line in out.splitlines():
33     mo = re.search(r'refs/heads/release-R(\d+)-(\d+)\.B', line)
34     if mo:
35       revision = int(mo.group(1))
36       build = int(mo.group(2))
37       lsb_build = int(lsb_version.split('.')[0])
38       if lsb_build > build:
39         lower.append(revision)
40   lower = sorted(lower)
41   if lower:
42     return 'R%d-%s' % (lower[-1] + 1, lsb_version)
43   else:
44     return 'Unknown'
45 
46 
47 def ApplySubs(string, *substitutions):
48   for pattern, replacement in substitutions:
49     string = re.sub(pattern, replacement, string)
50   return string
51 
52 
53 def UnitToNumber(unit_num, base=1000):
54   """Convert a number with unit to float."""
55   unit_dict = {'kilo': base, 'mega': base**2, 'giga': base**3}
56   unit_num = unit_num.lower()
57   mo = re.search(r'(\d*)(.+)?', unit_num)
58   number = mo.group(1)
59   unit = mo.group(2)
60   if not unit:
61     return float(number)
62   for k, v in unit_dict.items():
63     if k.startswith(unit):
64       return float(number) * v
65   raise RuntimeError('Unit: %s not found in byte: %s!' % (unit, unit_num))
66 
67 
68 def GetFilenameFromString(string):
69   return ApplySubs(string, (r'/', '__'), (r'\s', '_'), (r'[\\$="?^]', ''),)
70 
71 
72 def GetRoot(scr_name):
73   """Break up pathname into (dir+name)."""
74   abs_path = os.path.abspath(scr_name)
75   return (os.path.dirname(abs_path), os.path.basename(abs_path))
76 
77 
78 def GetChromeOSKeyFile(chromeos_root):
79   return os.path.join(chromeos_root, 'src', 'scripts', 'mod_for_test_scripts',
80                       'ssh_keys', 'testing_rsa')
81 
82 
83 def GetChrootPath(chromeos_root):
84   return os.path.join(chromeos_root, 'chroot')
85 
86 
87 def GetInsideChrootPath(chromeos_root, file_path):
88   if not file_path.startswith(GetChrootPath(chromeos_root)):
89     raise RuntimeError("File: %s doesn't seem to be in the chroot: %s" %
90                        (file_path, chromeos_root))
91   return file_path[len(GetChrootPath(chromeos_root)):]
92 
93 
94 def GetOutsideChrootPath(chromeos_root, file_path):
95   return os.path.join(GetChrootPath(chromeos_root), file_path.lstrip('/'))
96 
97 
98 def FormatQuotedCommand(command):
99   return ApplySubs(command, ('"', r'\"'))
100 
101 
102 def FormatCommands(commands):
103   return ApplySubs(
104       str(commands), ('&&', '&&\n'), (';', ';\n'), (r'\n+\s*', '\n'))
105 
106 
107 def GetImageDir(chromeos_root, board):
108   return os.path.join(chromeos_root, 'src', 'build', 'images', board)
109 
110 
111 def LabelLatestImage(chromeos_root, board, label, vanilla_path=None):
112   image_dir = GetImageDir(chromeos_root, board)
113   latest_image_dir = os.path.join(image_dir, 'latest')
114   latest_image_dir = os.path.realpath(latest_image_dir)
115   latest_image_dir = os.path.basename(latest_image_dir)
116   retval = 0
117   with WorkingDirectory(image_dir):
118     command = 'ln -sf -T %s %s' % (latest_image_dir, label)
119     ce = command_executer.GetCommandExecuter()
120     retval = ce.RunCommand(command)
121     if retval:
122       return retval
123     if vanilla_path:
124       command = 'ln -sf -T %s %s' % (vanilla_path, 'vanilla')
125       retval2 = ce.RunCommand(command)
126       return retval2
127   return retval
128 
129 
130 def DoesLabelExist(chromeos_root, board, label):
131   image_label = os.path.join(GetImageDir(chromeos_root, board), label)
132   return os.path.exists(image_label)
133 
134 
135 def GetBuildPackagesCommand(board, usepkg=False, debug=False):
136   if usepkg:
137     usepkg_flag = '--usepkg'
138   else:
139     usepkg_flag = '--nousepkg'
140   if debug:
141     withdebug_flag = '--withdebug'
142   else:
143     withdebug_flag = '--nowithdebug'
144   return ('%s/build_packages %s --withdev --withtest --withautotest '
145           '--skip_toolchain_update %s --board=%s '
146           '--accept_licenses=@CHROMEOS' %
147           (CHROMEOS_SCRIPTS_DIR, usepkg_flag, withdebug_flag, board))
148 
149 
150 def GetBuildImageCommand(board, dev=False):
151   dev_args = ''
152   if dev:
153     dev_args = '--noenable_rootfs_verification --disk_layout=2gb-rootfs'
154   return ('%s/build_image --board=%s %s test' %
155           (CHROMEOS_SCRIPTS_DIR, board, dev_args))
156 
157 
158 def GetSetupBoardCommand(board,
159                          gcc_version=None,
160                          binutils_version=None,
161                          usepkg=None,
162                          force=None):
163   """Get setup_board command."""
164   options = []
165 
166   if gcc_version:
167     options.append('--gcc_version=%s' % gcc_version)
168 
169   if binutils_version:
170     options.append('--binutils_version=%s' % binutils_version)
171 
172   if usepkg:
173     options.append('--usepkg')
174   else:
175     options.append('--nousepkg')
176 
177   if force:
178     options.append('--force')
179 
180   options.append('--accept_licenses=@CHROMEOS')
181 
182   return ('%s/setup_board --board=%s %s' %
183           (CHROMEOS_SCRIPTS_DIR, board, ' '.join(options)))
184 
185 
186 def CanonicalizePath(path):
187   path = os.path.expanduser(path)
188   path = os.path.realpath(path)
189   return path
190 
191 
192 def GetCtargetFromBoard(board, chromeos_root):
193   """Get Ctarget from board."""
194   base_board = board.split('_')[0]
195   command = ('source %s; get_ctarget_from_board %s' %
196              (TOOLCHAIN_UTILS_PATH, base_board))
197   ce = command_executer.GetCommandExecuter()
198   ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
199   if ret != 0:
200     raise ValueError('Board %s is invalid!' % board)
201   # Remove ANSI escape sequences.
202   out = StripANSIEscapeSequences(out)
203   return out.strip()
204 
205 
206 def GetArchFromBoard(board, chromeos_root):
207   """Get Arch from board."""
208   base_board = board.split('_')[0]
209   command = ('source %s; get_board_arch %s' %
210              (TOOLCHAIN_UTILS_PATH, base_board))
211   ce = command_executer.GetCommandExecuter()
212   ret, out, _ = ce.ChrootRunCommandWOutput(chromeos_root, command)
213   if ret != 0:
214     raise ValueError('Board %s is invalid!' % board)
215   # Remove ANSI escape sequences.
216   out = StripANSIEscapeSequences(out)
217   return out.strip()
218 
219 
220 def GetGccLibsDestForBoard(board, chromeos_root):
221   """Get gcc libs destination from board."""
222   arch = GetArchFromBoard(board, chromeos_root)
223   if arch == 'x86':
224     return '/build/%s/usr/lib/gcc/' % board
225   if arch == 'amd64':
226     return '/build/%s/usr/lib64/gcc/' % board
227   if arch == 'arm':
228     return '/build/%s/usr/lib/gcc/' % board
229   if arch == 'arm64':
230     return '/build/%s/usr/lib/gcc/' % board
231   raise ValueError('Arch %s is invalid!' % arch)
232 
233 
234 def StripANSIEscapeSequences(string):
235   string = re.sub(r'\x1b\[[0-9]*[a-zA-Z]', '', string)
236   return string
237 
238 
239 def GetChromeSrcDir():
240   return 'var/cache/distfiles/target/chrome-src/src'
241 
242 
243 def GetEnvStringFromDict(env_dict):
244   return ' '.join(["%s=\"%s\"" % var for var in env_dict.items()])
245 
246 
247 def MergeEnvStringWithDict(env_string, env_dict, prepend=True):
248   """Merge env string with dict."""
249   if not env_string.strip():
250     return GetEnvStringFromDict(env_dict)
251   override_env_list = []
252   ce = command_executer.GetCommandExecuter()
253   for k, v in env_dict.items():
254     v = v.strip("\"'")
255     if prepend:
256       new_env = "%s=\"%s $%s\"" % (k, v, k)
257     else:
258       new_env = "%s=\"$%s %s\"" % (k, k, v)
259     command = '; '.join([env_string, new_env, 'echo $%s' % k])
260     ret, out, _ = ce.RunCommandWOutput(command)
261     override_env_list.append('%s=%r' % (k, out.strip()))
262   ret = env_string + ' ' + ' '.join(override_env_list)
263   return ret.strip()
264 
265 
266 def GetAllImages(chromeos_root, board):
267   ce = command_executer.GetCommandExecuter()
268   command = ('find %s/src/build/images/%s -name chromiumos_test_image.bin' %
269              (chromeos_root, board))
270   ret, out, _ = ce.RunCommandWOutput(command)
271   assert ret == 0, 'Could not run command: %s' % command
272   return out.splitlines()
273 
274 
275 def IsFloat(text):
276   if text is None:
277     return False
278   try:
279     float(text)
280     return True
281   except ValueError:
282     return False
283 
284 
285 def RemoveChromeBrowserObjectFiles(chromeos_root, board):
286   """Remove any object files from all the posible locations."""
287   out_dir = os.path.join(
288       GetChrootPath(chromeos_root),
289       'var/cache/chromeos-chrome/chrome-src/src/out_%s' % board)
290   if os.path.exists(out_dir):
291     shutil.rmtree(out_dir)
292     logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
293   out_dir = os.path.join(
294       GetChrootPath(chromeos_root),
295       'var/cache/chromeos-chrome/chrome-src-internal/src/out_%s' % board)
296   if os.path.exists(out_dir):
297     shutil.rmtree(out_dir)
298     logger.GetLogger().LogCmd('rm -rf %s' % out_dir)
299 
300 
301 @contextmanager
302 def WorkingDirectory(new_dir):
303   """Get the working directory."""
304   old_dir = os.getcwd()
305   if old_dir != new_dir:
306     msg = 'cd %s' % new_dir
307     logger.GetLogger().LogCmd(msg)
308   os.chdir(new_dir)
309   yield new_dir
310   if old_dir != new_dir:
311     msg = 'cd %s' % old_dir
312     logger.GetLogger().LogCmd(msg)
313   os.chdir(old_dir)
314 
315 
316 def HasGitStagedChanges(git_dir):
317   """Return True if git repository has staged changes."""
318   command = 'cd {0} && git diff --quiet --cached --exit-code HEAD'.format(
319       git_dir)
320   return command_executer.GetCommandExecuter().RunCommand(
321       command,
322       print_to_console=False)
323 
324 
325 def HasGitUnstagedChanges(git_dir):
326   """Return True if git repository has un-staged changes."""
327   command = 'cd {0} && git diff --quiet --exit-code HEAD'.format(git_dir)
328   return command_executer.GetCommandExecuter().RunCommand(
329       command,
330       print_to_console=False)
331 
332 
333 def HasGitUntrackedChanges(git_dir):
334   """Return True if git repository has un-tracked changes."""
335   command = ('cd {0} && test -z '
336              '$(git ls-files --exclude-standard --others)').format(git_dir)
337   return command_executer.GetCommandExecuter().RunCommand(
338       command,
339       print_to_console=False)
340 
341 
342 def GitGetCommitHash(git_dir, commit_symbolic_name):
343   """Return githash for the symbolic git commit.
344 
345   For example, commit_symbolic_name could be
346   "cros/gcc.gnu.org/branches/gcc/gcc-4_8-mobile, this function returns the git
347   hash for this symbolic name.
348 
349   Args:
350     git_dir: a git working tree.
351     commit_symbolic_name: a symbolic name for a particular git commit.
352 
353   Returns:
354     The git hash for the symbolic name or None if fails.
355   """
356 
357   command = ('cd {0} && git log -n 1 --pretty="format:%H" {1}').format(
358       git_dir, commit_symbolic_name)
359   rv, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
360       command,
361       print_to_console=False)
362   if rv == 0:
363     return out.strip()
364   return None
365 
366 
367 def IsGitTreeClean(git_dir):
368   """Test if git tree has no local changes.
369 
370   Args:
371     git_dir: git tree directory.
372 
373   Returns:
374     True if git dir is clean.
375   """
376   if HasGitStagedChanges(git_dir):
377     logger.GetLogger().LogWarning('Git tree has staged changes.')
378     return False
379   if HasGitUnstagedChanges(git_dir):
380     logger.GetLogger().LogWarning('Git tree has unstaged changes.')
381     return False
382   if HasGitUntrackedChanges(git_dir):
383     logger.GetLogger().LogWarning('Git tree has un-tracked changes.')
384     return False
385   return True
386 
387 
388 def GetGitChangesAsList(git_dir, path=None, staged=False):
389   """Get changed files as a list.
390 
391   Args:
392     git_dir: git tree directory.
393     path: a relative path that is part of the tree directory, could be null.
394     staged: whether to include staged files as well.
395 
396   Returns:
397     A list containing all the changed files.
398   """
399   command = 'cd {0} && git diff --name-only'.format(git_dir)
400   if staged:
401     command += ' --cached'
402   if path:
403     command += ' -- ' + path
404   _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput(
405       command,
406       print_to_console=False)
407   rv = []
408   for line in out.splitlines():
409     rv.append(line)
410   return rv
411 
412 
413 def IsChromeOsTree(chromeos_root):
414   return (os.path.isdir(os.path.join(chromeos_root,
415                                      'src/third_party/chromiumos-overlay')) and
416           os.path.isdir(os.path.join(chromeos_root, 'manifest')))
417 
418 
419 def DeleteChromeOsTree(chromeos_root, dry_run=False):
420   """Delete a ChromeOs tree *safely*.
421 
422   Args:
423     chromeos_root: dir of the tree, could be a relative one (but be careful)
424     dry_run: only prints out the command if True
425 
426   Returns:
427     True if everything is ok.
428   """
429   if not IsChromeOsTree(chromeos_root):
430     logger.GetLogger().LogWarning(
431         '"{0}" does not seem to be a valid chromeos tree, do nothing.'.format(
432             chromeos_root))
433     return False
434   cmd0 = 'cd {0} && cros_sdk --delete'.format(chromeos_root)
435   if dry_run:
436     print(cmd0)
437   else:
438     if command_executer.GetCommandExecuter().RunCommand(
439         cmd0,
440         print_to_console=True) != 0:
441       return False
442 
443   cmd1 = ('export CHROMEOSDIRNAME="$(dirname $(cd {0} && pwd))" && '
444           'export CHROMEOSBASENAME="$(basename $(cd {0} && pwd))" && '
445           'cd $CHROMEOSDIRNAME && sudo rm -fr $CHROMEOSBASENAME').format(
446               chromeos_root)
447   if dry_run:
448     print(cmd1)
449     return True
450 
451   return command_executer.GetCommandExecuter().RunCommand(
452       cmd1,
453       print_to_console=True) == 0
454 
455 
456 def ApplyGerritPatches(chromeos_root,
457                        gerrit_patch_string,
458                        branch='cros/master'):
459   """Apply gerrit patches on a chromeos tree.
460 
461   Args:
462     chromeos_root: chromeos tree path
463     gerrit_patch_string: a patch string just like the one gives to cbuildbot,
464     'id1 id2 *id3 ... idn'. A prefix of '* means this is an internal patch.
465     branch: the tree based on which to apply the patches.
466 
467   Returns:
468     True if success.
469   """
470 
471   ### First of all, we need chromite libs
472   sys.path.append(os.path.join(chromeos_root, 'chromite'))
473   # Imports below are ok after modifying path to add chromite.
474   # Pylint cannot detect that and complains.
475   # pylint: disable=import-error
476   from lib import git
477   from lib import gerrit
478   manifest = git.ManifestCheckout(chromeos_root)
479   patch_list = gerrit_patch_string.split(' ')
480   ### This takes time, print log information.
481   logger.GetLogger().LogOutput('Retrieving patch information from server ...')
482   patch_info_list = gerrit.GetGerritPatchInfo(patch_list)
483   for pi in patch_info_list:
484     project_checkout = manifest.FindCheckout(pi.project, strict=False)
485     if not project_checkout:
486       logger.GetLogger().LogError(
487           'Failed to find patch project "{project}" in manifest.'.format(
488               project=pi.project))
489       return False
490 
491     pi_str = '{project}:{ref}'.format(project=pi.project, ref=pi.ref)
492     try:
493       project_git_path = project_checkout.GetPath(absolute=True)
494       logger.GetLogger().LogOutput('Applying patch "{0}" in "{1}" ...'.format(
495           pi_str, project_git_path))
496       pi.Apply(project_git_path, branch, trivial=False)
497     except Exception:
498       traceback.print_exc(file=sys.stdout)
499       logger.GetLogger().LogError('Failed to apply patch "{0}"'.format(pi_str))
500       return False
501   return True
502 
503 
504 def BooleanPrompt(prompt='Do you want to continue?',
505                   default=True,
506                   true_value='yes',
507                   false_value='no',
508                   prolog=None):
509   """Helper function for processing boolean choice prompts.
510 
511   Args:
512     prompt: The question to present to the user.
513     default: Boolean to return if the user just presses enter.
514     true_value: The text to display that represents a True returned.
515     false_value: The text to display that represents a False returned.
516     prolog: The text to display before prompt.
517 
518   Returns:
519     True or False.
520   """
521   true_value, false_value = true_value.lower(), false_value.lower()
522   true_text, false_text = true_value, false_value
523   if true_value == false_value:
524     raise ValueError('true_value and false_value must differ: got %r' %
525                      true_value)
526 
527   if default:
528     true_text = true_text[0].upper() + true_text[1:]
529   else:
530     false_text = false_text[0].upper() + false_text[1:]
531 
532   prompt = ('\n%s (%s/%s)? ' % (prompt, true_text, false_text))
533 
534   if prolog:
535     prompt = ('\n%s\n%s' % (prolog, prompt))
536 
537   while True:
538     try:
539       response = raw_input(prompt).lower()
540     except EOFError:
541       # If the user hits CTRL+D, or stdin is disabled, use the default.
542       print()
543       response = None
544     except KeyboardInterrupt:
545       # If the user hits CTRL+C, just exit the process.
546       print()
547       print('CTRL+C detected; exiting')
548       sys.exit()
549 
550     if not response:
551       return default
552     if true_value.startswith(response):
553       if not false_value.startswith(response):
554         return True
555       # common prefix between the two...
556     elif false_value.startswith(response):
557       return False
558 
559 def rgb2short(r, g, b):
560   """  Converts RGB values to xterm-256 color. """
561 
562   redcolor = [255, 124, 160, 196, 9 ]
563   greencolor = [255, 118, 82, 46, 10 ]
564 
565   if g == 0:
566     return redcolor[r/52]
567   if r == 0:
568     return greencolor[g/52]
569   return 4
570