1#!/usr/bin/env python2
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Script to use remote try-bot build image with local gcc."""
7
8from __future__ import print_function
9
10import argparse
11import glob
12import os
13import re
14import shutil
15import socket
16import sys
17import tempfile
18import time
19
20from cros_utils import command_executer
21from cros_utils import logger
22from cros_utils import manifest_versions
23from cros_utils import misc
24
25BRANCH = 'the_actual_branch_used_in_this_script'
26TMP_BRANCH = 'tmp_branch'
27SLEEP_TIME = 600
28
29# pylint: disable=anomalous-backslash-in-string
30
31
32def GetPatchNum(output):
33  lines = output.splitlines()
34  line = [l for l in lines if 'googlesource' in l][0]
35  patch_num = re.findall(r'\d+', line)[0]
36  if 'chrome-internal' in line:
37    patch_num = '*' + patch_num
38  return str(patch_num)
39
40
41def GetPatchString(patch):
42  if patch:
43    return '+'.join(patch)
44  return 'NO_PATCH'
45
46
47def FindVersionForToolchain(branch, chromeos_root):
48  """Find the version number in artifacts link in the tryserver email."""
49  # For example: input:  toolchain-3701.42.B
50  #              output: R26-3701.42.1
51  digits = branch.split('-')[1].split('B')[0]
52  manifest_dir = os.path.join(chromeos_root, 'manifest-internal')
53  os.chdir(manifest_dir)
54  major_version = digits.split('.')[0]
55  ce = command_executer.GetCommandExecuter()
56  command = 'repo sync . && git branch -a | grep {0}'.format(major_version)
57  _, branches, _ = ce.RunCommandWOutput(command, print_to_console=False)
58  m = re.search(r'(R\d+)', branches)
59  if not m:
60    logger.GetLogger().LogFatal('Cannot find version for branch {0}'
61                                .format(branch))
62  version = m.group(0) + '-' + digits + '1'
63  return version
64
65
66def FindBuildId(description):
67  """Find the build id of the build at trybot server."""
68  running_time = 0
69  while True:
70    (result, number) = FindBuildIdFromLog(description)
71    if result >= 0:
72      return (result, number)
73    logger.GetLogger().LogOutput('{0} minutes passed.'
74                                 .format(running_time / 60))
75    logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
76    time.sleep(SLEEP_TIME)
77    running_time += SLEEP_TIME
78
79
80def FindBuildIdFromLog(description):
81  """Get the build id from build log."""
82  # returns tuple (result, buildid)
83  # result == 0, buildid > 0, the build was successful and we have a build id
84  # result > 0, buildid > 0,  the whole build failed for some reason but we
85  #                           do have a build id.
86  # result == -1, buildid == -1, we have not found a finished build for this
87  #                              description yet
88
89  file_dir = os.path.dirname(os.path.realpath(__file__))
90  commands = ('{0}/cros_utils/buildbot_json.py builds '
91              'http://chromegw/p/tryserver.chromiumos/'.format(file_dir))
92  ce = command_executer.GetCommandExecuter()
93  _, buildinfo, _ = ce.RunCommandWOutput(commands, print_to_console=False)
94
95  my_info = buildinfo.splitlines()
96  current_line = 1
97  running_job = False
98  result = -1
99
100  # result == 0, we have a successful build
101  # result > 0, we have a failed build but build id may be valid
102  # result == -1, we have not found a finished build for this description
103  while current_line < len(my_info):
104    my_dict = {}
105    while True:
106      key = my_info[current_line].split(':')[0].strip()
107      value = my_info[current_line].split(':', 1)[1].strip()
108      my_dict[key] = value
109      current_line += 1
110      if 'Build' in key or current_line == len(my_info):
111        break
112    if ('True' not in my_dict['completed'] and
113        str(description) in my_dict['reason']):
114      running_job = True
115    if ('True' not in my_dict['completed'] or
116        str(description) not in my_dict['reason']):
117      continue
118    result = int(my_dict['result'])
119    build_id = int(my_dict['number'])
120    if result == 0:
121      return (result, build_id)
122    else:
123      # Found a finished failed build.
124      # Keep searching to find a successful one
125      pass
126
127  if result > 0 and not running_job:
128    return (result, build_id)
129  return (-1, -1)
130
131
132def DownloadImage(target, index, dest, version):
133  """Download artifacts from cloud."""
134  if not os.path.exists(dest):
135    os.makedirs(dest)
136
137  rversion = manifest_versions.RFormatCrosVersion(version)
138  print(str(rversion))
139  #  ls_cmd = ("gsutil ls gs://chromeos-image-archive/trybot-{0}/{1}-b{2}"
140  #            .format(target, rversion, index))
141  ls_cmd = ('gsutil ls gs://chromeos-image-archive/trybot-{0}/*-b{2}'.format(
142      target, index))
143
144  download_cmd = ('$(which gsutil) cp {0} {1}'.format('{0}', dest))
145  ce = command_executer.GetCommandExecuter()
146
147  _, out, _ = ce.RunCommandWOutput(ls_cmd, print_to_console=True)
148  lines = out.splitlines()
149  download_files = [
150      'autotest.tar', 'chromeos-chrome', 'chromiumos_test_image', 'debug.tgz',
151      'sysroot_chromeos-base_chromeos-chrome.tar.xz'
152  ]
153  for line in lines:
154    if any([e in line for e in download_files]):
155      cmd = download_cmd.format(line)
156      if ce.RunCommand(cmd):
157        logger.GetLogger().LogFatal('Command {0} failed, existing...'
158                                    .format(cmd))
159
160
161def UnpackImage(dest):
162  """Unpack the image, the chroot build dir."""
163  chrome_tbz2 = glob.glob(dest + '/*.tbz2')[0]
164  commands = ('tar xJf {0}/sysroot_chromeos-base_chromeos-chrome.tar.xz '
165              '-C {0} &&'
166              'tar xjf {1} -C {0} &&'
167              'tar xzf {0}/debug.tgz  -C {0}/usr/lib/ &&'
168              'tar xf {0}/autotest.tar -C {0}/usr/local/ &&'
169              'tar xJf {0}/chromiumos_test_image.tar.xz -C {0}'.format(
170                  dest, chrome_tbz2))
171  ce = command_executer.GetCommandExecuter()
172  return ce.RunCommand(commands)
173
174
175def RemoveOldBranch():
176  """Remove the branch with name BRANCH."""
177  ce = command_executer.GetCommandExecuter()
178  command = 'git rev-parse --abbrev-ref HEAD'
179  _, out, _ = ce.RunCommandWOutput(command)
180  if BRANCH in out:
181    command = 'git checkout -B {0}'.format(TMP_BRANCH)
182    ce.RunCommand(command)
183  command = "git commit -m 'nouse'"
184  ce.RunCommand(command)
185  command = 'git branch -D {0}'.format(BRANCH)
186  ce.RunCommand(command)
187
188
189def UploadManifest(manifest, chromeos_root, branch='master'):
190  """Copy the manifest to $chromeos_root/manifest-internal and upload."""
191  chromeos_root = misc.CanonicalizePath(chromeos_root)
192  manifest_dir = os.path.join(chromeos_root, 'manifest-internal')
193  os.chdir(manifest_dir)
194  ce = command_executer.GetCommandExecuter()
195
196  RemoveOldBranch()
197
198  if branch != 'master':
199    branch = '{0}'.format(branch)
200  command = 'git checkout -b {0} -t cros-internal/{1}'.format(BRANCH, branch)
201  ret = ce.RunCommand(command)
202  if ret:
203    raise RuntimeError('Command {0} failed'.format(command))
204
205  # We remove the default.xml, which is the symbolic link of full.xml.
206  # After that, we copy our xml file to default.xml.
207  # We did this because the full.xml might be updated during the
208  # run of the script.
209  os.remove(os.path.join(manifest_dir, 'default.xml'))
210  shutil.copyfile(manifest, os.path.join(manifest_dir, 'default.xml'))
211  return UploadPatch(manifest)
212
213
214def GetManifestPatch(manifests, version, chromeos_root, branch='master'):
215  """Return a gerrit patch number given a version of manifest file."""
216  temp_dir = tempfile.mkdtemp()
217  to_file = os.path.join(temp_dir, 'default.xml')
218  manifests.GetManifest(version, to_file)
219  return UploadManifest(to_file, chromeos_root, branch)
220
221
222def UploadPatch(source):
223  """Up load patch to gerrit, return patch number."""
224  commands = ('git add -A . &&'
225              "git commit -m 'test' -m 'BUG=None' -m 'TEST=None' "
226              "-m 'hostname={0}' -m 'source={1}'".format(
227                  socket.gethostname(), source))
228  ce = command_executer.GetCommandExecuter()
229  ce.RunCommand(commands)
230
231  commands = ('yes | repo upload .   --cbr --no-verify')
232  _, _, err = ce.RunCommandWOutput(commands)
233  return GetPatchNum(err)
234
235
236def ReplaceSysroot(chromeos_root, dest_dir, target):
237  """Copy unpacked sysroot and image to chromeos_root."""
238  ce = command_executer.GetCommandExecuter()
239  # get the board name from "board-release". board may contain "-"
240  board = target.rsplit('-', 1)[0]
241  board_dir = os.path.join(chromeos_root, 'chroot', 'build', board)
242  command = 'sudo rm -rf {0}'.format(board_dir)
243  ce.RunCommand(command)
244
245  command = 'sudo mv {0} {1}'.format(dest_dir, board_dir)
246  ce.RunCommand(command)
247
248  image_dir = os.path.join(chromeos_root, 'src', 'build', 'images', board,
249                           'latest')
250  command = 'rm -rf {0} && mkdir -p {0}'.format(image_dir)
251  ce.RunCommand(command)
252
253  command = 'mv {0}/chromiumos_test_image.bin {1}'.format(board_dir, image_dir)
254  return ce.RunCommand(command)
255
256
257def GccBranchForToolchain(branch):
258  if branch == 'toolchain-3428.65.B':
259    return 'release-R25-3428.B'
260  else:
261    return None
262
263
264def GetGccBranch(branch):
265  """Get the remote branch name from branch or version."""
266  ce = command_executer.GetCommandExecuter()
267  command = 'git branch -a | grep {0}'.format(branch)
268  _, out, _ = ce.RunCommandWOutput(command)
269  if not out:
270    release_num = re.match(r'.*(R\d+)-*', branch)
271    if release_num:
272      release_num = release_num.group(0)
273      command = 'git branch -a | grep {0}'.format(release_num)
274      _, out, _ = ce.RunCommandWOutput(command)
275      if not out:
276        GccBranchForToolchain(branch)
277  if not out:
278    out = 'remotes/cros/master'
279  new_branch = out.splitlines()[0]
280  return new_branch
281
282
283def UploadGccPatch(chromeos_root, gcc_dir, branch):
284  """Upload local gcc to gerrit and get the CL number."""
285  ce = command_executer.GetCommandExecuter()
286  gcc_dir = misc.CanonicalizePath(gcc_dir)
287  gcc_path = os.path.join(chromeos_root, 'src/third_party/gcc')
288  assert os.path.isdir(gcc_path), ('{0} is not a valid chromeos root'
289                                   .format(chromeos_root))
290  assert os.path.isdir(gcc_dir), ('{0} is not a valid dir for gcc'
291                                  'source'.format(gcc_dir))
292  os.chdir(gcc_path)
293  RemoveOldBranch()
294  if not branch:
295    branch = 'master'
296  branch = GetGccBranch(branch)
297  command = ('git checkout -b {0} -t {1} && ' 'rm -rf *'.format(BRANCH, branch))
298  ce.RunCommand(command, print_to_console=False)
299
300  command = ("rsync -az --exclude='*.svn' --exclude='*.git'"
301             ' {0}/ .'.format(gcc_dir))
302  ce.RunCommand(command)
303  return UploadPatch(gcc_dir)
304
305
306def RunRemote(chromeos_root, branch, patches, is_local, target, chrome_version,
307              dest_dir):
308  """The actual running commands."""
309  ce = command_executer.GetCommandExecuter()
310
311  if is_local:
312    local_flag = '--local -r {0}'.format(dest_dir)
313  else:
314    local_flag = '--remote'
315  patch = ''
316  for p in patches:
317    patch += ' -g {0}'.format(p)
318  cbuildbot_path = os.path.join(chromeos_root, 'chromite/cbuildbot')
319  os.chdir(cbuildbot_path)
320  branch_flag = ''
321  if branch != 'master':
322    branch_flag = ' -b {0}'.format(branch)
323  chrome_version_flag = ''
324  if chrome_version:
325    chrome_version_flag = ' --chrome_version={0}'.format(chrome_version)
326  description = '{0}_{1}_{2}'.format(branch, GetPatchString(patches), target)
327  command = ('yes | ./cbuildbot {0} {1} {2} {3} {4} {5}'
328             ' --remote-description={6}'
329             ' --chrome_rev=tot'.format(patch, branch_flag, chrome_version,
330                                        local_flag, chrome_version_flag, target,
331                                        description))
332  ce.RunCommand(command)
333
334  return description
335
336
337def Main(argv):
338  """The main function."""
339  # Common initializations
340  parser = argparse.ArgumentParser()
341  parser.add_argument(
342      '-c',
343      '--chromeos_root',
344      required=True,
345      dest='chromeos_root',
346      help='The chromeos_root')
347  parser.add_argument(
348      '-g', '--gcc_dir', default='', dest='gcc_dir', help='The gcc dir')
349  parser.add_argument(
350      '-t',
351      '--target',
352      required=True,
353      dest='target',
354      help=('The target to be build, the list is at'
355            ' $(chromeos_root)/chromite/buildbot/cbuildbot'
356            ' --list -all'))
357  parser.add_argument('-l', '--local', action='store_true')
358  parser.add_argument(
359      '-d',
360      '--dest_dir',
361      dest='dest_dir',
362      help=('The dir to build the whole chromeos if'
363            ' --local is set'))
364  parser.add_argument(
365      '--chrome_version',
366      dest='chrome_version',
367      default='',
368      help='The chrome version to use. '
369      'Default it will use the latest one.')
370  parser.add_argument(
371      '--chromeos_version',
372      dest='chromeos_version',
373      default='',
374      help=('The chromeos version to use.'
375            '(1) A release version in the format: '
376            "'\d+\.\d+\.\d+\.\d+.*'"
377            "(2) 'latest_lkgm' for the latest lkgm version"))
378  parser.add_argument(
379      '-r',
380      '--replace_sysroot',
381      action='store_true',
382      help=('Whether or not to replace the build/$board dir'
383            'under the chroot of chromeos_root and copy '
384            'the image to src/build/image/$board/latest.'
385            ' Default is False'))
386  parser.add_argument(
387      '-b',
388      '--branch',
389      dest='branch',
390      default='',
391      help=('The branch to run trybot, default is None'))
392  parser.add_argument(
393      '-p',
394      '--patch',
395      dest='patch',
396      default='',
397      help=('The patches to be applied, the patches numbers '
398            "be seperated by ','"))
399
400  script_dir = os.path.dirname(os.path.realpath(__file__))
401
402  args = parser.parse_args(argv[1:])
403  target = args.target
404  if args.patch:
405    patch = args.patch.split(',')
406  else:
407    patch = []
408  chromeos_root = misc.CanonicalizePath(args.chromeos_root)
409  if args.chromeos_version and args.branch:
410    raise RuntimeError('You can not set chromeos_version and branch at the '
411                       'same time.')
412
413  manifests = None
414  if args.branch:
415    chromeos_version = ''
416    branch = args.branch
417  else:
418    chromeos_version = args.chromeos_version
419    manifests = manifest_versions.ManifestVersions()
420    if chromeos_version == 'latest_lkgm':
421      chromeos_version = manifests.TimeToVersion(time.mktime(time.gmtime()))
422      logger.GetLogger().LogOutput('found version %s for latest LKGM' %
423                                   (chromeos_version))
424    # TODO: this script currently does not handle the case where the version
425    # is not in the "master" branch
426    branch = 'master'
427
428  if chromeos_version:
429    manifest_patch = GetManifestPatch(manifests, chromeos_version,
430                                      chromeos_root)
431    patch.append(manifest_patch)
432  if args.gcc_dir:
433    # TODO: everytime we invoke this script we are getting a different
434    # patch for GCC even if GCC has not changed. The description should
435    # be based on the MD5 of the GCC patch contents.
436    patch.append(UploadGccPatch(chromeos_root, args.gcc_dir, branch))
437  description = RunRemote(chromeos_root, branch, patch, args.local, target,
438                          args.chrome_version, args.dest_dir)
439  if args.local or not args.dest_dir:
440    # TODO: We are not checktng the result of cbuild_bot in here!
441    return 0
442
443  # return value:
444  # 0 => build bot was successful and image was put where requested
445  # 1 => Build bot FAILED but image was put where requested
446  # 2 => Build bot failed or BUild bot was successful but and image was
447  #      not generated or could not be put where expected
448
449  os.chdir(script_dir)
450  dest_dir = misc.CanonicalizePath(args.dest_dir)
451  (bot_result, build_id) = FindBuildId(description)
452  if bot_result > 0 and build_id > 0:
453    logger.GetLogger().LogError('Remote trybot failed but image was generated')
454    bot_result = 1
455  elif bot_result > 0:
456    logger.GetLogger().LogError('Remote trybot failed. No image was generated')
457    return 2
458  if 'toolchain' in branch:
459    chromeos_version = FindVersionForToolchain(branch, chromeos_root)
460    assert not manifest_versions.IsRFormatCrosVersion(chromeos_version)
461  DownloadImage(target, build_id, dest_dir, chromeos_version)
462  ret = UnpackImage(dest_dir)
463  if ret != 0:
464    return 2
465  # todo: return a more inteligent return value
466  if not args.replace_sysroot:
467    return bot_result
468
469  ret = ReplaceSysroot(chromeos_root, args.dest_dir, target)
470  if ret != 0:
471    return 2
472
473  # got an image and we were successful in placing it where requested
474  return bot_result
475
476
477if __name__ == '__main__':
478  retval = Main(sys.argv)
479  sys.exit(retval)
480