1#!/usr/bin/env python2
2#
3# Copyright 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Script for running nightly compiler tests on ChromeOS.
7
8This script launches a buildbot to build ChromeOS with the latest compiler on
9a particular board; then it finds and downloads the trybot image and the
10corresponding official image, and runs crosperf performance tests comparing
11the two.  It then generates a report, emails it to the c-compiler-chrome, as
12well as copying the images into the seven-day reports directory.
13"""
14
15# Script to test different toolchains against ChromeOS benchmarks.
16
17from __future__ import print_function
18
19import argparse
20import datetime
21import os
22import re
23import sys
24import time
25
26from cros_utils import command_executer
27from cros_utils import logger
28
29from cros_utils import buildbot_utils
30
31# CL that updated GCC ebuilds to use 'next_gcc'.
32USE_NEXT_GCC_PATCH = '230260'
33
34# CL that uses LLVM to build the peppy image.
35USE_LLVM_PATCH = '295217'
36
37# CL that uses LLVM-Next to build the images (includes chrome).
38USE_LLVM_NEXT_PATCH = '424123'
39
40CROSTC_ROOT = '/usr/local/google/crostc'
41ROLE_ACCOUNT = 'mobiletc-prebuild'
42TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
43MAIL_PROGRAM = '~/var/bin/mail-sheriff'
44PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
45NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
46
47IMAGE_DIR = '{board}-{image_type}'
48IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
49IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR
50TRYBOT_IMAGE_FS = 'trybot-' + IMAGE_FS + '-{build_id}'
51PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
52IMAGE_RE_GROUPS = {
53    'board': r'(?P<board>\S+)',
54    'image_type': r'(?P<image_type>\S+)',
55    'chrome_version': r'(?P<chrome_version>R\d+)',
56    'tip': r'(?P<tip>\d+)',
57    'branch': r'(?P<branch>\d+)',
58    'branch_branch': r'(?P<branch_branch>\d+)',
59    'build_id': r'(?P<build_id>b\d+)'
60}
61TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
62
63
64class ToolchainComparator(object):
65  """Class for doing the nightly tests work."""
66
67  def __init__(self,
68               board,
69               remotes,
70               chromeos_root,
71               weekday,
72               patches,
73               noschedv2=False):
74    self._board = board
75    self._remotes = remotes
76    self._chromeos_root = chromeos_root
77    self._base_dir = os.getcwd()
78    self._ce = command_executer.GetCommandExecuter()
79    self._l = logger.GetLogger()
80    self._build = '%s-release' % board
81    self._patches = patches.split(',')
82    self._patches_string = '_'.join(str(p) for p in self._patches)
83    self._noschedv2 = noschedv2
84
85    if not weekday:
86      self._weekday = time.strftime('%a')
87    else:
88      self._weekday = weekday
89    timestamp = datetime.datetime.strftime(datetime.datetime.now(),
90                                           '%Y-%m-%d_%H:%M:%S')
91    self._reports_dir = os.path.join(
92        NIGHTLY_TESTS_DIR,
93        '%s.%s' % (timestamp, board),)
94
95  def _GetVanillaImageName(self, trybot_image):
96    """Given a trybot artifact name, get latest vanilla image name.
97
98    Args:
99      trybot_image: artifact name such as
100          'trybot-daisy-release/R40-6394.0.0-b1389'
101
102    Returns:
103      Latest official image name, e.g. 'daisy-release/R57-9089.0.0'.
104    """
105    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
106    assert mo
107    dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict())
108    version = buildbot_utils.GetGSContent(self._chromeos_root,
109                                          dirname + '/LATEST-master')
110    return dirname + '/' + version
111
112  def _GetNonAFDOImageName(self, trybot_image):
113    """Given a trybot artifact name, get corresponding non-AFDO image name.
114
115    We get the non-AFDO image from the PFQ builders. This image
116    is not generated for all the boards and, the closest PFQ image
117    was the one build for the previous ChromeOS version (the chrome
118    used in the current version is the one validated in the previous
119    version).
120    The previous ChromeOS does not always exist either. So, we try
121    a couple of versions before.
122
123    Args:
124      trybot_image: artifact name such as
125          'trybot-daisy-release/R40-6394.0.0-b1389'
126
127    Returns:
128      Corresponding chrome PFQ image name, e.g.
129      'daisy-chrome-pfq/R40-6393.0.0-rc1'.
130    """
131    mo = re.search(TRYBOT_IMAGE_RE, trybot_image)
132    assert mo
133    image_dict = mo.groupdict()
134    image_dict['image_type'] = 'chrome-pfq'
135    for _ in xrange(2):
136      image_dict['tip'] = str(int(image_dict['tip']) - 1)
137      nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
138      if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
139        return nonafdo_image
140    return ''
141
142  def _FinishSetup(self):
143    """Make sure testing_rsa file is properly set up."""
144    # Fix protections on ssh key
145    command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
146               '/chrome-src-internal/src/third_party/chromite/ssh_keys'
147               '/testing_rsa')
148    ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
149    if ret_val != 0:
150      raise RuntimeError('chmod for testing_rsa failed')
151
152  def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
153    """Create crosperf experiment file.
154
155    Given the names of the trybot, vanilla and non-AFDO images, create the
156    appropriate crosperf experiment file and launch crosperf on it.
157    """
158    experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
159    experiment_file_name = '%s_toolchain_experiment.txt' % self._board
160
161    compiler_string = 'gcc'
162    if USE_LLVM_NEXT_PATCH in self._patches_string:
163      experiment_file_name = '%s_llvm_next_experiment.txt' % self._board
164      compiler_string = 'llvm_next'
165    elif USE_LLVM_PATCH in self._patches_string:
166      experiment_file_name = '%s_llvm_experiment.txt' % self._board
167      compiler_string = 'llvm'
168
169    experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
170    experiment_header = """
171    board: %s
172    remote: %s
173    retries: 1
174    """ % (self._board, self._remotes)
175    experiment_tests = """
176    benchmark: all_toolchain_perf {
177      suite: telemetry_Crosperf
178      iterations: 3
179    }
180
181    benchmark: page_cycler_v2.typical_25 {
182      suite: telemetry_Crosperf
183      iterations: 2
184      run_local: False
185      retries: 0
186    }
187    """
188
189    with open(experiment_file, 'w') as f:
190      f.write(experiment_header)
191      f.write(experiment_tests)
192
193      # Now add vanilla to test file.
194      official_image = """
195          vanilla_image {
196            chromeos_root: %s
197            build: %s
198            compiler: gcc
199          }
200          """ % (self._chromeos_root, vanilla_image)
201      f.write(official_image)
202
203      # Now add non-AFDO image to test file.
204      if nonafdo_image:
205        official_nonafdo_image = """
206          nonafdo_image {
207            chromeos_root: %s
208            build: %s
209            compiler: gcc
210          }
211          """ % (self._chromeos_root, nonafdo_image)
212        f.write(official_nonafdo_image)
213
214      label_string = '%s_trybot_image' % compiler_string
215      if USE_NEXT_GCC_PATCH in self._patches:
216        label_string = 'gcc_next_trybot_image'
217
218      # Reuse autotest files from vanilla image for trybot images
219      autotest_files = os.path.join('/tmp', vanilla_image, 'autotest_files')
220      experiment_image = """
221          %s {
222            chromeos_root: %s
223            build: %s
224            autotest_path: %s
225            compiler: %s
226          }
227          """ % (label_string, self._chromeos_root, trybot_image,
228                 autotest_files, compiler_string)
229      f.write(experiment_image)
230
231    crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
232    noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
233    command = ('{crosperf} --no_email=True --results_dir={r_dir} '
234               '--json_report=True {noschedv2_opts} {exp_file}').format(
235                   crosperf=crosperf,
236                   r_dir=self._reports_dir,
237                   noschedv2_opts=noschedv2_opts,
238                   exp_file=experiment_file)
239
240    ret = self._ce.RunCommand(command)
241    if ret != 0:
242      raise RuntimeError('Crosperf execution error!')
243    else:
244      # Copy json report to pending archives directory.
245      command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
246      ret = self._ce.RunCommand(command)
247    return
248
249  def _SendEmail(self):
250    """Find email message generated by crosperf and send it."""
251    filename = os.path.join(self._reports_dir, 'msg_body.html')
252    if (os.path.exists(filename) and
253        os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
254      email_title = 'buildbot test results'
255      if USE_LLVM_NEXT_PATCH in self._patches_string:
256        email_title = 'buildbot llvm_next test results'
257      elif USE_LLVM_PATCH in self._patches_string:
258        email_title = 'buildbot llvm test results'
259      command = ('cat %s | %s -s "%s, %s" -team -html' %
260                 (filename, MAIL_PROGRAM, email_title, self._board))
261      self._ce.RunCommand(command)
262
263  def DoAll(self):
264    """Main function inside ToolchainComparator class.
265
266    Launch trybot, get image names, create crosperf experiment file, run
267    crosperf, and copy images into seven-day report directories.
268    """
269    date_str = datetime.date.today()
270    description = 'master_%s_%s_%s' % (self._patches_string, self._build,
271                                       date_str)
272    build_id, trybot_image = buildbot_utils.GetTrybotImage(
273        self._chromeos_root,
274        self._build,
275        self._patches,
276        description,
277        other_flags=['--notests'],
278        build_toolchain=True)
279
280    print('trybot_url: \
281       https://uberchromegw.corp.google.com/i/chromiumos.tryserver/builders/release/builds/%s'
282          % build_id)
283    if len(trybot_image) == 0:
284      self._l.LogError('Unable to find trybot_image for %s!' % description)
285      return 1
286
287    vanilla_image = self._GetVanillaImageName(trybot_image)
288    nonafdo_image = self._GetNonAFDOImageName(trybot_image)
289
290    print('trybot_image: %s' % trybot_image)
291    print('vanilla_image: %s' % vanilla_image)
292    print('nonafdo_image: %s' % nonafdo_image)
293
294    if os.getlogin() == ROLE_ACCOUNT:
295      self._FinishSetup()
296
297    self._TestImages(trybot_image, vanilla_image, nonafdo_image)
298    self._SendEmail()
299    return 0
300
301
302def Main(argv):
303  """The main function."""
304
305  # Common initializations
306  command_executer.InitCommandExecuter()
307  parser = argparse.ArgumentParser()
308  parser.add_argument(
309      '--remote', dest='remote', help='Remote machines to run tests on.')
310  parser.add_argument(
311      '--board', dest='board', default='x86-zgb', help='The target board.')
312  parser.add_argument(
313      '--chromeos_root',
314      dest='chromeos_root',
315      help='The chromeos root from which to run tests.')
316  parser.add_argument(
317      '--weekday',
318      default='',
319      dest='weekday',
320      help='The day of the week for which to run tests.')
321  parser.add_argument(
322      '--patch',
323      dest='patches',
324      help='The patches to use for the testing, '
325      "seprate the patch numbers with ',' "
326      'for more than one patches.')
327  parser.add_argument(
328      '--noschedv2',
329      dest='noschedv2',
330      action='store_true',
331      default=False,
332      help='Pass --noschedv2 to crosperf.')
333
334  options = parser.parse_args(argv[1:])
335  if not options.board:
336    print('Please give a board.')
337    return 1
338  if not options.remote:
339    print('Please give at least one remote machine.')
340    return 1
341  if not options.chromeos_root:
342    print('Please specify the ChromeOS root directory.')
343    return 1
344  if options.patches:
345    patches = options.patches
346  else:
347    patches = USE_NEXT_GCC_PATCH
348
349  fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
350                           options.weekday, patches, options.noschedv2)
351  return fc.DoAll()
352
353
354if __name__ == '__main__':
355  retval = Main(sys.argv)
356  sys.exit(retval)
357