1# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Utilities for launching and accessing ChromeOS buildbots."""
5
6from __future__ import print_function
7
8import ast
9import json
10import os
11import re
12import time
13
14from cros_utils import command_executer
15from cros_utils import logger
16
17INITIAL_SLEEP_TIME = 7200  # 2 hours; wait time before polling buildbot.
18SLEEP_TIME = 600  # 10 minutes; time between polling of buildbot.
19
20# Some of our slower builders (llmv-next) are taking more
21# than 8 hours. So, increase this TIME_OUT to 9 hours.
22TIME_OUT = 32400  # Decide the build is dead or will never finish
23
24
25class BuildbotTimeout(Exception):
26  """Exception to throw when a buildbot operation timesout."""
27  pass
28
29
30def RunCommandInPath(path, cmd):
31  ce = command_executer.GetCommandExecuter()
32  cwd = os.getcwd()
33  os.chdir(path)
34  status, stdout, stderr = ce.RunCommandWOutput(cmd, print_to_console=False)
35  os.chdir(cwd)
36  return status, stdout, stderr
37
38
39def PeekTrybotImage(chromeos_root, buildbucket_id):
40  """Get the artifact URL of a given tryjob.
41
42  Args:
43    buildbucket_id: buildbucket-id
44    chromeos_root: root dir of chrome os checkout
45
46  Returns:
47    (status, url) where status can be 'pass', 'fail', 'running',
48                  and url looks like:
49    gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
50  """
51  command = (
52      'cros buildresult --report json --buildbucket-id %s' % buildbucket_id)
53  rc, out, _ = RunCommandInPath(chromeos_root, command)
54
55  # Current implementation of cros buildresult returns fail when a job is still
56  # running.
57  if rc != 0:
58    return ('running', None)
59
60  results = json.loads(out)[buildbucket_id]
61
62  return (results['status'], results['artifacts_url'].rstrip('/'))
63
64
65def ParseTryjobBuildbucketId(msg):
66  """Find the buildbucket-id in the messages from `cros tryjob`.
67
68  Args:
69    msg: messages from `cros tryjob`
70
71  Returns:
72    buildbucket-id, which will be passed to `cros buildresult`
73  """
74  output_list = ast.literal_eval(msg)
75  output_dict = output_list[0]
76  if 'buildbucket_id' in output_dict:
77    return output_dict['buildbucket_id']
78  return None
79
80
81def SubmitTryjob(chromeos_root,
82                 buildbot_name,
83                 patch_list,
84                 tryjob_flags=None,
85                 build_toolchain=False):
86  """Calls `cros tryjob ...`
87
88  Args:
89    chromeos_root: the path to the ChromeOS root, needed for finding chromite
90                   and launching the buildbot.
91    buildbot_name: the name of the buildbot queue, such as lumpy-release or
92                   daisy-paladin.
93    patch_list: a python list of the patches, if any, for the buildbot to use.
94    tryjob_flags: See cros tryjob --help for available options.
95    build_toolchain: builds and uses the latest toolchain, rather than the
96                     prebuilt one in SDK.
97
98  Returns:
99    buildbucket id
100  """
101  patch_arg = ''
102  if patch_list:
103    for p in patch_list:
104      patch_arg = patch_arg + ' -g ' + repr(p)
105  if not tryjob_flags:
106    tryjob_flags = []
107  if build_toolchain:
108    tryjob_flags.append('--latest-toolchain')
109  tryjob_flags = ' '.join(tryjob_flags)
110
111  # Launch buildbot with appropriate flags.
112  build = buildbot_name
113  command = ('cros tryjob --yes --json --nochromesdk  %s %s %s' %
114             (tryjob_flags, patch_arg, build))
115  print('CMD: %s' % command)
116  _, out, _ = RunCommandInPath(chromeos_root, command)
117  buildbucket_id = ParseTryjobBuildbucketId(out)
118  print('buildbucket_id: %s' % repr(buildbucket_id))
119  if not buildbucket_id:
120    logger.GetLogger().LogFatal('Error occurred while launching trybot job: '
121                                '%s' % command)
122  return buildbucket_id
123
124
125def GetTrybotImage(chromeos_root,
126                   buildbot_name,
127                   patch_list,
128                   tryjob_flags=None,
129                   build_toolchain=False,
130                   async=False):
131  """Launch buildbot and get resulting trybot artifact name.
132
133  This function launches a buildbot with the appropriate flags to
134  build the test ChromeOS image, with the current ToT mobile compiler.  It
135  checks every 10 minutes to see if the trybot has finished.  When the trybot
136  has finished, it parses the resulting report logs to find the trybot
137  artifact (if one was created), and returns that artifact name.
138
139  Args:
140    chromeos_root: the path to the ChromeOS root, needed for finding chromite
141                   and launching the buildbot.
142    buildbot_name: the name of the buildbot queue, such as lumpy-release or
143                   daisy-paladin.
144    patch_list: a python list of the patches, if any, for the buildbot to use.
145    tryjob_flags: See cros tryjob --help for available options.
146    build_toolchain: builds and uses the latest toolchain, rather than the
147                     prebuilt one in SDK.
148    async: don't wait for artifacts; just return the buildbucket id
149
150  Returns:
151    (buildbucket id, partial image url) e.g.
152    (8952271933586980528, trybot-elm-release-tryjob/R67-10480.0.0-b2373596)
153  """
154  buildbucket_id = SubmitTryjob(chromeos_root, buildbot_name, patch_list,
155                                tryjob_flags, build_toolchain)
156  if async:
157    return buildbucket_id, ' '
158
159  # The trybot generally takes more than 2 hours to finish.
160  # Wait two hours before polling the status.
161  time.sleep(INITIAL_SLEEP_TIME)
162  elapsed = INITIAL_SLEEP_TIME
163  status = 'running'
164  image = ''
165  while True:
166    status, image = PeekTrybotImage(chromeos_root, buildbucket_id)
167    if status == 'running':
168      if elapsed > TIME_OUT:
169        logger.GetLogger().LogFatal(
170            'Unable to get build result for target %s.' % buildbot_name)
171      else:
172        wait_msg = 'Unable to find build result; job may be running.'
173        logger.GetLogger().LogOutput(wait_msg)
174      logger.GetLogger().LogOutput('{0} minutes elapsed.'.format(elapsed / 60))
175      logger.GetLogger().LogOutput('Sleeping {0} seconds.'.format(SLEEP_TIME))
176      time.sleep(SLEEP_TIME)
177      elapsed += SLEEP_TIME
178    else:
179      break
180
181  if not buildbot_name.endswith('-toolchain') and status == 'fail':
182    # For rotating testers, we don't care about their status
183    # result, because if any HWTest failed it will be non-zero.
184    #
185    # The nightly performance tests do not run HWTests, so if
186    # their status is non-zero, we do care.  In this case
187    # non-zero means the image itself probably did not build.
188    image = ''
189
190  if not image:
191    logger.GetLogger().LogError(
192        'Trybot job (buildbucket id: %s) failed with'
193        'status %s; no trybot image generated. ' % (buildbucket_id, status))
194  else:
195    # Convert full gs path to what crosperf expects. For example, convert
196    # gs://chromeos-image-archive/trybot-elm-release-tryjob/R67-10468.0.0-b20789
197    # to
198    # trybot-elm-release-tryjob/R67-10468.0.0-b20789
199    image = '/'.join(image.split('/')[-2:])
200
201  logger.GetLogger().LogOutput("image is '%s'" % image)
202  logger.GetLogger().LogOutput('status is %s' % status)
203  return buildbucket_id, image
204
205
206def DoesImageExist(chromeos_root, build):
207  """Check if the image for the given build exists."""
208
209  ce = command_executer.GetCommandExecuter()
210  command = ('gsutil ls gs://chromeos-image-archive/%s'
211             '/chromiumos_test_image.tar.xz' % (build))
212  ret = ce.ChrootRunCommand(chromeos_root, command, print_to_console=False)
213  return not ret
214
215
216def WaitForImage(chromeos_root, build):
217  """Wait for an image to be ready."""
218
219  elapsed_time = 0
220  while elapsed_time < TIME_OUT:
221    if DoesImageExist(chromeos_root, build):
222      return
223    logger.GetLogger().LogOutput(
224        'Image %s not ready, waiting for 10 minutes' % build)
225    time.sleep(SLEEP_TIME)
226    elapsed_time += SLEEP_TIME
227
228  logger.GetLogger().LogOutput(
229      'Image %s not found, waited for %d hours' % (build, (TIME_OUT / 3600)))
230  raise BuildbotTimeout('Timeout while waiting for image %s' % build)
231
232
233def GetLatestImage(chromeos_root, path):
234  """Get latest image"""
235
236  fmt = re.compile(r'R([0-9]+)-([0-9]+).([0-9]+).([0-9]+)')
237
238  ce = command_executer.GetCommandExecuter()
239  command = ('gsutil ls gs://chromeos-image-archive/%s' % path)
240  _, out, _ = ce.ChrootRunCommandWOutput(
241      chromeos_root, command, print_to_console=False)
242  candidates = [l.split('/')[-2] for l in out.split()]
243  candidates = map(fmt.match, candidates)
244  candidates = [[int(r) for r in m.group(1, 2, 3, 4)] for m in candidates if m]
245  candidates.sort(reverse=True)
246  for c in candidates:
247    build = '%s/R%d-%d.%d.%d' % (path, c[0], c[1], c[2], c[3])
248    if DoesImageExist(chromeos_root, build):
249      return build
250