1#!/usr/bin/env python2
2#
3# Copyright 2017 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Script for running llvm validation tests on ChromeOS.
7
8This script launches a buildbot to build ChromeOS with the llvm on
9a particular board; then it finds and downloads the trybot image and the
10corresponding official image, and runs test for correctness.
11It then generates a report, emails it to the c-compiler-chrome, as
12well as copying the result into a directory.
13"""
14
15# Script to test different toolchains against ChromeOS benchmarks.
16
17from __future__ import print_function
18
19import argparse
20import datetime
21import os
22import sys
23import time
24
25from cros_utils import command_executer
26from cros_utils import logger
27
28from cros_utils import buildbot_utils
29
30CROSTC_ROOT = '/usr/local/google/crostc'
31ROLE_ACCOUNT = 'mobiletc-prebuild'
32TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
33MAIL_PROGRAM = '~/var/bin/mail-sheriff'
34VALIDATION_RESULT_DIR = os.path.join(CROSTC_ROOT, 'validation_result')
35START_DATE = datetime.date(2016, 1, 1)
36TEST_PER_DAY = 4
37DATA_DIR = '/google/data/rw/users/mo/mobiletc-prebuild/waterfall-report-data/'
38
39# Information about Rotating Boards
40#  Board        Arch     Reference    Platform      Kernel
41#                        Board                      Version
42#  ------------ -------  ------------ ------------- -------
43#  cave         x86_64   glados       skylake-y     3.18
44#  daisy        armv7    daisy        exynos-5250   3.8.11
45#  elm          aarch64  oak          mediatek-8173 3.18
46#  fizz         x86_64   fizz         kabylake-u/r  4.4.*
47#  gale         armv7                               3.18
48#  grunt        x86_64   grunt        stoney ridge  4.14.*
49#  guado_moblab x86_64                              3.14
50#  kevin        aarch64  gru          rockchip-3399 4.4.*
51#  lakitu       x86_64                              4.4.*
52#  lars         x86_64   kunimitsu    skylake-u     3.18
53#  link         x86_64   ivybridge    ivybridge     3.8.11
54#  nautilus     x86_64   poppy        kabylake-y    4.4.*
55#  nyan_big     armv7    nyan         tegra         3.10.18
56#  peach_pit    armv7    peach        exynos-5420   3.8.11
57#  peppy        x86_64   slippy       haswell       3.8.11
58#  samus        x86_64   auron        broadwell     3.14
59#  snappy       x86_64   reef         apollo lake   4.4.*
60#  swanky       x86_64   rambi        baytrail      4.4.*
61#  terra        x86_64   strago       braswell      3.18
62#  veyron_jaq   armv7    veyron-pinky rockchip-3288 3.14
63#  whirlwind    armv7                               3.14
64#  zoombini     x86_64   zoombini     cannonlake-y  4.14.*
65
66TEST_BOARD = [
67    'cave',
68    'daisy',
69    # 'elm', tested by arm64-llvm-next-toolchain builder.
70    'fizz',
71    'gale',
72    'grunt',
73    'guado_moblab',
74    'kevin',
75    'lakitu',
76    'lars',
77    'link',
78    'nautilus',
79    'nyan_big',
80    'peach_pit',
81    'peppy',
82    # 'samus', tested by amd64-llvm-next-toolchain builder.
83    'snappy',
84    'swanky',
85    'terra',
86    # 'veyron_jaq', tested by arm-llvm-next-toolchain builder.
87    'whirlwind',
88    'zoombini',
89]
90
91
92class ToolchainVerifier(object):
93  """Class for the toolchain verifier."""
94
95  def __init__(self, board, chromeos_root, weekday, patches, compiler):
96    self._board = board
97    self._chromeos_root = chromeos_root
98    self._base_dir = os.getcwd()
99    self._ce = command_executer.GetCommandExecuter()
100    self._l = logger.GetLogger()
101    self._compiler = compiler
102    self._build = '%s-%s-toolchain-tryjob' % (board, compiler)
103    self._patches = patches.split(',') if patches else []
104    self._patches_string = '_'.join(str(p) for p in self._patches)
105
106    if not weekday:
107      self._weekday = time.strftime('%a')
108    else:
109      self._weekday = weekday
110    self._reports = os.path.join(VALIDATION_RESULT_DIR, compiler, board)
111
112  def DoAll(self):
113    """Main function inside ToolchainComparator class.
114
115    Launch trybot, get image names, create crosperf experiment file, run
116    crosperf, and copy images into seven-day report directories.
117    """
118    buildbucket_id, _ = buildbot_utils.GetTrybotImage(
119        self._chromeos_root,
120        self._build,
121        self._patches,
122        tryjob_flags=['--hwtest'],
123        async=True)
124
125    return buildbucket_id
126
127
128def WriteRotatingReportsData(results_dict, date):
129  """Write data for waterfall report."""
130  fname = '%d-%02d-%02d.builds' % (date.year, date.month, date.day)
131  filename = os.path.join(DATA_DIR, 'rotating-builders', fname)
132  with open(filename, 'w') as out_file:
133    for board in results_dict.keys():
134      buildbucket_id = results_dict[board]
135      out_file.write('%s,%s\n' % (buildbucket_id, board))
136
137
138def Main(argv):
139  """The main function."""
140
141  # Common initializations
142  command_executer.InitCommandExecuter()
143  parser = argparse.ArgumentParser()
144  parser.add_argument(
145      '--chromeos_root',
146      dest='chromeos_root',
147      help='The chromeos root from which to run tests.')
148  parser.add_argument(
149      '--weekday',
150      default='',
151      dest='weekday',
152      help='The day of the week for which to run tests.')
153  parser.add_argument(
154      '--board', default='', dest='board', help='The board to test.')
155  parser.add_argument(
156      '--patch',
157      dest='patches',
158      default='',
159      help='The patches to use for the testing, '
160      "seprate the patch numbers with ',' "
161      'for more than one patches.')
162  parser.add_argument(
163      '--compiler',
164      dest='compiler',
165      help='Which compiler (llvm, llvm-next or gcc) to use for '
166      'testing.')
167
168  options = parser.parse_args(argv[1:])
169  if not options.chromeos_root:
170    print('Please specify the ChromeOS root directory.')
171    return 1
172  if not options.compiler:
173    print('Please specify which compiler to test (gcc, llvm, or llvm-next).')
174    return 1
175
176  if options.board:
177    fv = ToolchainVerifier(options.board, options.chromeos_root,
178                           options.weekday, options.patches, options.compiler)
179    return fv.Doall()
180
181  today = datetime.date.today()
182  delta = today - START_DATE
183  days = delta.days
184
185  start_board = (days * TEST_PER_DAY) % len(TEST_BOARD)
186  results_dict = dict()
187  for i in range(TEST_PER_DAY):
188    try:
189      board = TEST_BOARD[(start_board + i) % len(TEST_BOARD)]
190      fv = ToolchainVerifier(board, options.chromeos_root, options.weekday,
191                             options.patches, options.compiler)
192      buildbucket_id = fv.DoAll()
193      if buildbucket_id:
194        results_dict[board] = buildbucket_id
195    except SystemExit:
196      logfile = os.path.join(VALIDATION_RESULT_DIR, options.compiler, board)
197      with open(logfile, 'w') as f:
198        f.write('Verifier got an exception, please check the log.\n')
199  WriteRotatingReportsData(results_dict, today)
200
201
202if __name__ == '__main__':
203  retval = Main(sys.argv)
204  sys.exit(retval)
205