1# -*- coding: utf-8 -*-
2# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""SuiteRunner defines the interface from crosperf to test script."""
7
8from __future__ import division
9from __future__ import print_function
10
11import json
12import os
13import pipes
14import shlex
15import time
16
17from cros_utils import command_executer
18
19TEST_THAT_PATH = '/usr/bin/test_that'
20TAST_PATH = '/usr/bin/tast'
21SKYLAB_PATH = '/usr/local/bin/skylab'
22GS_UTIL = 'src/chromium/depot_tools/gsutil.py'
23AUTOTEST_DIR = '/mnt/host/source/src/third_party/autotest/files'
24CHROME_MOUNT_DIR = '/tmp/chrome_root'
25
26
27def GetProfilerArgs(profiler_args):
28  # Remove "--" from in front of profiler args.
29  args_list = shlex.split(profiler_args)
30  new_list = []
31  for arg in args_list:
32    if arg[0:2] == '--':
33      arg = arg[2:]
34    new_list.append(arg)
35  args_list = new_list
36
37  # Remove "perf_options=" from middle of profiler args.
38  new_list = []
39  for arg in args_list:
40    idx = arg.find('perf_options=')
41    if idx != -1:
42      prefix = arg[0:idx]
43      suffix = arg[idx + len('perf_options=') + 1:-1]
44      new_arg = prefix + "'" + suffix + "'"
45      new_list.append(new_arg)
46    else:
47      new_list.append(arg)
48  args_list = new_list
49
50  return ' '.join(args_list)
51
52
53def GetDutConfigArgs(dut_config):
54  return 'dut_config={}'.format(pipes.quote(json.dumps(dut_config)))
55
56
57class SuiteRunner(object):
58  """This defines the interface from crosperf to test script."""
59
60  def __init__(self,
61               dut_config,
62               logger_to_use=None,
63               log_level='verbose',
64               cmd_exec=None,
65               cmd_term=None):
66    self.logger = logger_to_use
67    self.log_level = log_level
68    self._ce = cmd_exec or command_executer.GetCommandExecuter(
69        self.logger, log_level=self.log_level)
70    # DUT command executer.
71    # Will be initialized and used within Run.
72    self._ct = cmd_term or command_executer.CommandTerminator()
73    self.dut_config = dut_config
74
75  def Run(self, cros_machine, label, benchmark, test_args, profiler_args):
76    machine_name = cros_machine.name
77    for i in range(0, benchmark.retries + 1):
78      if label.skylab:
79        ret_tup = self.Skylab_Run(label, benchmark, test_args, profiler_args)
80      else:
81        if benchmark.suite == 'tast':
82          ret_tup = self.Tast_Run(machine_name, label, benchmark)
83        else:
84          ret_tup = self.Test_That_Run(machine_name, label, benchmark,
85                                       test_args, profiler_args)
86      if ret_tup[0] != 0:
87        self.logger.LogOutput('benchmark %s failed. Retries left: %s' %
88                              (benchmark.name, benchmark.retries - i))
89      elif i > 0:
90        self.logger.LogOutput(
91            'benchmark %s succeded after %s retries' % (benchmark.name, i))
92        break
93      else:
94        self.logger.LogOutput(
95            'benchmark %s succeded on first try' % benchmark.name)
96        break
97    return ret_tup
98
99  def RemoveTelemetryTempFile(self, machine, chromeos_root):
100    filename = 'telemetry@%s' % machine
101    fullname = os.path.join(chromeos_root, 'chroot', 'tmp', filename)
102    if os.path.exists(fullname):
103      os.remove(fullname)
104
105  def GenTestArgs(self, benchmark, test_args, profiler_args):
106    args_list = []
107
108    if benchmark.suite != 'telemetry_Crosperf' and profiler_args:
109      self.logger.LogFatal('Tests other than telemetry_Crosperf do not '
110                           'support profiler.')
111
112    if test_args:
113      # Strip double quotes off args (so we can wrap them in single
114      # quotes, to pass through to Telemetry).
115      if test_args[0] == '"' and test_args[-1] == '"':
116        test_args = test_args[1:-1]
117      args_list.append("test_args='%s'" % test_args)
118
119    args_list.append(GetDutConfigArgs(self.dut_config))
120
121    if not (benchmark.suite == 'telemetry_Crosperf' or
122            benchmark.suite == 'crosperf_Wrapper'):
123      self.logger.LogWarning('Please make sure the server test has stage for '
124                             'device setup.\n')
125    else:
126      args_list.append('test=%s' % benchmark.test_name)
127      if benchmark.suite == 'telemetry_Crosperf':
128        args_list.append('run_local=%s' % benchmark.run_local)
129        args_list.append(GetProfilerArgs(profiler_args))
130
131    return args_list
132
133  # TODO(zhizhouy): Currently do not support passing arguments or running
134  # customized tast tests, as we do not have such requirements.
135  def Tast_Run(self, machine, label, benchmark):
136    # Remove existing tast results
137    command = 'rm -rf /usr/local/autotest/results/*'
138    self._ce.CrosRunCommand(
139        command, machine=machine, chromeos_root=label.chromeos_root)
140
141    command = ' '.join(
142        [TAST_PATH, 'run', '-build=False', machine, benchmark.test_name])
143
144    if self.log_level != 'verbose':
145      self.logger.LogOutput('Running test.')
146      self.logger.LogOutput('CMD: %s' % command)
147
148    return self._ce.ChrootRunCommandWOutput(
149        label.chromeos_root, command, command_terminator=self._ct)
150
151  def Test_That_Run(self, machine, label, benchmark, test_args, profiler_args):
152    """Run the test_that test.."""
153
154    # Remove existing test_that results
155    command = 'rm -rf /usr/local/autotest/results/*'
156    self._ce.CrosRunCommand(
157        command, machine=machine, chromeos_root=label.chromeos_root)
158
159    if benchmark.suite == 'telemetry_Crosperf':
160      if not os.path.isdir(label.chrome_src):
161        self.logger.LogFatal('Cannot find chrome src dir to '
162                             'run telemetry: %s' % label.chrome_src)
163      # Check for and remove temporary file that may have been left by
164      # previous telemetry runs (and which might prevent this run from
165      # working).
166      self.RemoveTelemetryTempFile(machine, label.chromeos_root)
167
168    # --autotest_dir specifies which autotest directory to use.
169    autotest_dir_arg = '--autotest_dir=%s' % (
170        label.autotest_path if label.autotest_path else AUTOTEST_DIR)
171
172    # --fast avoids unnecessary copies of syslogs.
173    fast_arg = '--fast'
174    board_arg = '--board=%s' % label.board
175
176    args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
177    args_arg = '--args=%s' % pipes.quote(' '.join(args_list))
178
179    command = ' '.join([
180        TEST_THAT_PATH, autotest_dir_arg, fast_arg, board_arg, args_arg,
181        machine, benchmark.suite if
182        (benchmark.suite == 'telemetry_Crosperf' or
183         benchmark.suite == 'crosperf_Wrapper') else benchmark.test_name
184    ])
185
186    # Use --no-ns-pid so that cros_sdk does not create a different
187    # process namespace and we can kill process created easily by their
188    # process group.
189    chrome_root_options = ('--no-ns-pid '
190                           '--chrome_root={0} --chrome_root_mount={1} '
191                           'FEATURES="-usersandbox" '
192                           'CHROME_ROOT={1}'.format(label.chrome_src,
193                                                    CHROME_MOUNT_DIR))
194
195    if self.log_level != 'verbose':
196      self.logger.LogOutput('Running test.')
197      self.logger.LogOutput('CMD: %s' % command)
198
199    return self._ce.ChrootRunCommandWOutput(
200        label.chromeos_root,
201        command,
202        command_terminator=self._ct,
203        cros_sdk_options=chrome_root_options)
204
205  def DownloadResult(self, label, task_id):
206    gsutil_cmd = os.path.join(label.chromeos_root, GS_UTIL)
207    result_dir = 'gs://chromeos-autotest-results/swarming-%s' % task_id
208    download_path = os.path.join(label.chromeos_root, 'chroot/tmp')
209    ls_command = '%s ls %s' % (gsutil_cmd,
210                               os.path.join(result_dir, 'autoserv_test'))
211    cp_command = '%s -mq cp -r %s %s' % (gsutil_cmd, result_dir, download_path)
212
213    # Server sometimes will not be able to generate the result directory right
214    # after the test. Will try to access this gs location every 60s for
215    # RETRY_LIMIT mins.
216    t = 0
217    RETRY_LIMIT = 10
218    while t < RETRY_LIMIT:
219      t += 1
220      status = self._ce.RunCommand(ls_command, print_to_console=False)
221      if status == 0:
222        break
223      if t < RETRY_LIMIT:
224        self.logger.LogOutput('Result directory not generated yet, '
225                              'retry (%d) in 60s.' % t)
226        time.sleep(60)
227      else:
228        self.logger.LogOutput('No result directory for task %s' % task_id)
229        return status
230
231    # Wait for 60s to make sure server finished writing to gs location.
232    time.sleep(60)
233
234    status = self._ce.RunCommand(cp_command)
235    if status != 0:
236      self.logger.LogOutput('Cannot download results from task %s' % task_id)
237    else:
238      self.logger.LogOutput('Result downloaded for task %s' % task_id)
239    return status
240
241  def Skylab_Run(self, label, benchmark, test_args, profiler_args):
242    """Run the test via skylab.."""
243    options = []
244    if label.board:
245      options.append('-board=%s' % label.board)
246    if label.build:
247      options.append('-image=%s' % label.build)
248    # TODO: now only put toolchain pool here, user need to be able to specify
249    # which pool to use. Need to request feature to not use this option at all.
250    options.append('-pool=toolchain')
251
252    args_list = self.GenTestArgs(benchmark, test_args, profiler_args)
253    options.append('-test-args=%s' % pipes.quote(' '.join(args_list)))
254
255    dimensions = []
256    for dut in label.remote:
257      dimensions.append('-dim dut_name:%s' % dut.rstrip('.cros'))
258
259    command = (('%s create-test %s %s %s') % \
260              (SKYLAB_PATH, ' '.join(dimensions), ' '.join(options),
261               benchmark.suite if
262               (benchmark.suite == 'telemetry_Crosperf' or
263                benchmark.suite == 'crosperf_Wrapper')
264               else benchmark.test_name))
265
266    if self.log_level != 'verbose':
267      self.logger.LogOutput('Starting skylab test.')
268      self.logger.LogOutput('CMD: %s' % command)
269    ret_tup = self._ce.RunCommandWOutput(command, command_terminator=self._ct)
270
271    if ret_tup[0] != 0:
272      self.logger.LogOutput('Skylab test not created successfully.')
273      return ret_tup
274
275    # Std output of the command will look like:
276    # Created request at https://ci.chromium.org/../cros_test_platform/b12345
277    # We want to parse it and get the id number of the task, which is the
278    # number in the very end of the link address.
279    task_id = ret_tup[1].strip().split('b')[-1]
280
281    command = ('skylab wait-task %s' % task_id)
282    if self.log_level != 'verbose':
283      self.logger.LogOutput('Waiting for skylab test to finish.')
284      self.logger.LogOutput('CMD: %s' % command)
285
286    ret_tup = self._ce.RunCommandWOutput(command, command_terminator=self._ct)
287
288    # The output of `wait-task` command will be a combination of verbose and a
289    # json format result in the end. The json result looks like this:
290    # {"task-result":
291    #   {"name":"Test Platform Invocation",
292    #    "state":"", "failure":false, "success":true,
293    #    "task-run-id":"12345",
294    #    "task-run-url":"https://ci.chromium.org/.../cros_test_platform/b12345",
295    #    "task-logs-url":""
296    #    },
297    #  "stdout":"",
298    #  "child-results":
299    #    [{"name":"graphics_WebGLAquarium",
300    #      "state":"", "failure":false, "success":true, "task-run-id":"",
301    #      "task-run-url":"https://chromeos-swarming.appspot.com/task?id=1234",
302    #      "task-logs-url":"https://stainless.corp.google.com/1234/"}
303    #    ]
304    # }
305    # We need the task id of the child-results to download result.
306    output = json.loads(ret_tup[1].split('\n')[-1])
307    output = output['child-results'][0]
308    if output['success']:
309      task_id = output['task-run-url'].split('=')[-1]
310      if self.DownloadResult(label, task_id) == 0:
311        result_dir = '\nResults placed in tmp/swarming-%s\n' % task_id
312        return (ret_tup[0], result_dir, ret_tup[2])
313    return ret_tup
314
315  def CommandTerminator(self):
316    return self._ct
317
318  def Terminate(self):
319    self._ct.Terminate()
320
321
322class MockSuiteRunner(object):
323  """Mock suite runner for test."""
324
325  def __init__(self):
326    self._true = True
327
328  def Run(self, *_args):
329    if self._true:
330      return [0, '', '']
331    else:
332      return [0, '', '']
333