1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Run Performance Test Bisect Tool
7
8This script is used by a try bot to run the bisect script with the parameters
9specified in the bisect config file. It checks out a copy of the depot in
10a subdirectory 'bisect' of the working directory provided, annd runs the
11bisect scrip there.
12"""
13
14import optparse
15import os
16import platform
17import subprocess
18import sys
19import traceback
20
21from auto_bisect import bisect_perf_regression
22from auto_bisect import bisect_utils
23from auto_bisect import math_utils
24
25CROS_BOARD_ENV = 'BISECT_CROS_BOARD'
26CROS_IP_ENV = 'BISECT_CROS_IP'
27
28SCRIPT_DIR = os.path.dirname(__file__)
29SRC_DIR = os.path.join(SCRIPT_DIR, os.path.pardir)
30BISECT_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'auto_bisect', 'bisect.cfg')
31RUN_TEST_CONFIG_PATH = os.path.join(SCRIPT_DIR, 'run-perf-test.cfg')
32WEBKIT_RUN_TEST_CONFIG_PATH = os.path.join(
33    SRC_DIR, 'third_party', 'WebKit', 'Tools', 'run-perf-test.cfg')
34BISECT_SCRIPT_DIR = os.path.join(SCRIPT_DIR, 'auto_bisect')
35
36
37class Goma(object):
38
39  def __init__(self, path_to_goma):
40    self._abs_path_to_goma = None
41    self._abs_path_to_goma_file = None
42    if not path_to_goma:
43      return
44    self._abs_path_to_goma = os.path.abspath(path_to_goma)
45    filename = 'goma_ctl.bat' if os.name == 'nt' else 'goma_ctl.sh'
46    self._abs_path_to_goma_file = os.path.join(self._abs_path_to_goma, filename)
47
48  def __enter__(self):
49    if self._HasGomaPath():
50      self._SetupAndStart()
51    return self
52
53  def __exit__(self, *_):
54    if self._HasGomaPath():
55      self._Stop()
56
57  def _HasGomaPath(self):
58    return bool(self._abs_path_to_goma)
59
60  def _SetupEnvVars(self):
61    if os.name == 'nt':
62      os.environ['CC'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
63          ' cl.exe')
64      os.environ['CXX'] = (os.path.join(self._abs_path_to_goma, 'gomacc.exe') +
65          ' cl.exe')
66    else:
67      os.environ['PATH'] = os.pathsep.join([self._abs_path_to_goma,
68          os.environ['PATH']])
69
70  def _SetupAndStart(self):
71    """Sets up goma and launches it.
72
73    Args:
74      path_to_goma: Path to goma directory.
75
76    Returns:
77      True if successful."""
78    self._SetupEnvVars()
79
80    # Sometimes goma is lingering around if something went bad on a previous
81    # run. Stop it before starting a new process. Can ignore the return code
82    # since it will return an error if it wasn't running.
83    self._Stop()
84
85    if subprocess.call([self._abs_path_to_goma_file, 'start']):
86      raise RuntimeError('Goma failed to start.')
87
88  def _Stop(self):
89    subprocess.call([self._abs_path_to_goma_file, 'stop'])
90
91
92def _LoadConfigFile(config_file_path):
93  """Attempts to load the specified config file as a module
94  and grab the global config dict.
95
96  Args:
97    config_file_path: Path to the config file.
98
99  Returns:
100    If successful, returns the config dict loaded from the file. If no
101    such dictionary could be loaded, returns the empty dictionary.
102  """
103  try:
104    local_vars = {}
105    execfile(config_file_path, local_vars)
106    return local_vars['config']
107  except Exception:
108    print
109    traceback.print_exc()
110    print
111    return {}
112
113
114def _ValidateConfigFile(config_contents, valid_parameters):
115  """Validates the config file contents, checking whether all values are
116  non-empty.
117
118  Args:
119    config_contents: A config dictionary.
120    valid_parameters: A list of parameters to check for.
121
122  Returns:
123    True if valid.
124  """
125  for parameter in valid_parameters:
126    if parameter not in config_contents:
127      return False
128    value = config_contents[parameter]
129    if not value or type(value) is not str:
130      return False
131  return True
132
133
134def _ValidatePerfConfigFile(config_contents):
135  """Validates the perf config file contents.
136
137  This is used when we're doing a perf try job, rather than a bisect.
138  The config file is called run-perf-test.cfg by default.
139
140  The parameters checked are the required parameters; any additional optional
141  parameters won't be checked and validation will still pass.
142
143  Args:
144    config_contents: A config dictionary.
145
146  Returns:
147    True if valid.
148  """
149  valid_parameters = [
150      'command',
151      'repeat_count',
152      'truncate_percent',
153      'max_time_minutes',
154  ]
155  return _ValidateConfigFile(config_contents, valid_parameters)
156
157
158def _ValidateBisectConfigFile(config_contents):
159  """Validates the bisect config file contents.
160
161  The parameters checked are the required parameters; any additional optional
162  parameters won't be checked and validation will still pass.
163
164  Args:
165    config_contents: A config dictionary.
166
167  Returns:
168    True if valid.
169  """
170  valid_params = [
171      'command',
172      'good_revision',
173      'bad_revision',
174      'metric',
175      'repeat_count',
176      'truncate_percent',
177      'max_time_minutes',
178  ]
179  return _ValidateConfigFile(config_contents, valid_params)
180
181
182def _OutputFailedResults(text_to_print):
183  bisect_utils.OutputAnnotationStepStart('Results - Failed')
184  print
185  print text_to_print
186  print
187  bisect_utils.OutputAnnotationStepClosed()
188
189
190def _CreateBisectOptionsFromConfig(config):
191  print config['command']
192  opts_dict = {}
193  opts_dict['command'] = config['command']
194  opts_dict['metric'] = config.get('metric')
195
196  if config['repeat_count']:
197    opts_dict['repeat_test_count'] = int(config['repeat_count'])
198
199  if config['truncate_percent']:
200    opts_dict['truncate_percent'] = int(config['truncate_percent'])
201
202  if config['max_time_minutes']:
203    opts_dict['max_time_minutes'] = int(config['max_time_minutes'])
204
205  if config.has_key('use_goma'):
206    opts_dict['use_goma'] = config['use_goma']
207  if config.has_key('goma_dir'):
208    opts_dict['goma_dir'] = config['goma_dir']
209
210  opts_dict['build_preference'] = 'ninja'
211  opts_dict['output_buildbot_annotations'] = True
212
213  if '--browser=cros' in config['command']:
214    opts_dict['target_platform'] = 'cros'
215
216    if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
217      opts_dict['cros_board'] = os.environ[CROS_BOARD_ENV]
218      opts_dict['cros_remote_ip'] = os.environ[CROS_IP_ENV]
219    else:
220      raise RuntimeError('CrOS build selected, but BISECT_CROS_IP or'
221          'BISECT_CROS_BOARD undefined.')
222  elif 'android' in config['command']:
223    if 'android-chrome-shell' in config['command']:
224      opts_dict['target_platform'] = 'android'
225    elif 'android-chrome' in config['command']:
226      opts_dict['target_platform'] = 'android-chrome'
227    else:
228      opts_dict['target_platform'] = 'android'
229
230  return bisect_perf_regression.BisectOptions.FromDict(opts_dict)
231
232
233def _RunPerformanceTest(config):
234  """Runs a performance test with and without the current patch.
235
236  Args:
237    config: Contents of the config file, a dictionary.
238
239  Attempts to build and run the current revision with and without the
240  current patch, with the parameters passed in.
241  """
242  # Bisect script expects to be run from the src directory
243  os.chdir(SRC_DIR)
244
245  bisect_utils.OutputAnnotationStepStart('Building With Patch')
246
247  opts = _CreateBisectOptionsFromConfig(config)
248  b = bisect_perf_regression.BisectPerformanceMetrics(None, opts)
249
250  if bisect_utils.RunGClient(['runhooks']):
251    raise RuntimeError('Failed to run gclient runhooks')
252
253  if not b.BuildCurrentRevision('chromium'):
254    raise RuntimeError('Patched version failed to build.')
255
256  bisect_utils.OutputAnnotationStepClosed()
257  bisect_utils.OutputAnnotationStepStart('Running With Patch')
258
259  results_with_patch = b.RunPerformanceTestAndParseResults(
260      opts.command, opts.metric, reset_on_first_run=True, results_label='Patch')
261
262  if results_with_patch[1]:
263    raise RuntimeError('Patched version failed to run performance test.')
264
265  bisect_utils.OutputAnnotationStepClosed()
266
267  bisect_utils.OutputAnnotationStepStart('Reverting Patch')
268  # TODO: When this is re-written to recipes, this should use bot_update's
269  # revert mechanism to fully revert the client. But for now, since we know that
270  # the perf try bot currently only supports src/ and src/third_party/WebKit, we
271  # simply reset those two directories.
272  bisect_utils.CheckRunGit(['reset', '--hard'])
273  bisect_utils.CheckRunGit(['reset', '--hard'],
274                           os.path.join('third_party', 'WebKit'))
275  bisect_utils.OutputAnnotationStepClosed()
276
277  bisect_utils.OutputAnnotationStepStart('Building Without Patch')
278
279  if bisect_utils.RunGClient(['runhooks']):
280    raise RuntimeError('Failed to run gclient runhooks')
281
282  if not b.BuildCurrentRevision('chromium'):
283    raise RuntimeError('Unpatched version failed to build.')
284
285  bisect_utils.OutputAnnotationStepClosed()
286  bisect_utils.OutputAnnotationStepStart('Running Without Patch')
287
288  results_without_patch = b.RunPerformanceTestAndParseResults(
289      opts.command, opts.metric, upload_on_last_run=True, results_label='ToT')
290
291  if results_without_patch[1]:
292    raise RuntimeError('Unpatched version failed to run performance test.')
293
294  # Find the link to the cloud stored results file.
295  output = results_without_patch[2]
296  cloud_file_link = [t for t in output.splitlines()
297      if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
298  if cloud_file_link:
299    # What we're getting here is basically "View online at http://..." so parse
300    # out just the URL portion.
301    cloud_file_link = cloud_file_link[0]
302    cloud_file_link = [t for t in cloud_file_link.split(' ')
303        if 'storage.googleapis.com/chromium-telemetry/html-results/' in t]
304    assert cloud_file_link, 'Couldn\'t parse URL from output.'
305    cloud_file_link = cloud_file_link[0]
306  else:
307    cloud_file_link = ''
308
309  # Calculate the % difference in the means of the 2 runs.
310  percent_diff_in_means = None
311  std_err = None
312  if (results_with_patch[0].has_key('mean') and
313      results_with_patch[0].has_key('values')):
314    percent_diff_in_means = (results_with_patch[0]['mean'] /
315        max(0.0001, results_without_patch[0]['mean'])) * 100.0 - 100.0
316    std_err = math_utils.PooledStandardError(
317        [results_with_patch[0]['values'], results_without_patch[0]['values']])
318
319  bisect_utils.OutputAnnotationStepClosed()
320  if percent_diff_in_means is not None and std_err is not None:
321    bisect_utils.OutputAnnotationStepStart('Results - %.02f +- %0.02f delta' %
322        (percent_diff_in_means, std_err))
323    print ' %s %s %s' % (''.center(10, ' '), 'Mean'.center(20, ' '),
324        'Std. Error'.center(20, ' '))
325    print ' %s %s %s' % ('Patch'.center(10, ' '),
326        ('%.02f' % results_with_patch[0]['mean']).center(20, ' '),
327        ('%.02f' % results_with_patch[0]['std_err']).center(20, ' '))
328    print ' %s %s %s' % ('No Patch'.center(10, ' '),
329        ('%.02f' % results_without_patch[0]['mean']).center(20, ' '),
330        ('%.02f' % results_without_patch[0]['std_err']).center(20, ' '))
331    if cloud_file_link:
332      bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
333    bisect_utils.OutputAnnotationStepClosed()
334  elif cloud_file_link:
335    bisect_utils.OutputAnnotationStepLink('HTML Results', cloud_file_link)
336
337
338def _SetupAndRunPerformanceTest(config, path_to_goma):
339  """Attempts to build and run the current revision with and without the
340  current patch, with the parameters passed in.
341
342  Args:
343    config: The config read from run-perf-test.cfg.
344    path_to_goma: Path to goma directory.
345
346  Returns:
347    An exit code: 0 on success, otherwise 1.
348  """
349  if platform.release() == 'XP':
350    print 'Windows XP is not supported for perf try jobs because it lacks '
351    print 'goma support. Please refer to crbug.com/330900.'
352    return 1
353  try:
354    with Goma(path_to_goma) as _:
355      config['use_goma'] = bool(path_to_goma)
356      if config['use_goma']:
357        config['goma_dir'] = os.path.abspath(path_to_goma)
358      _RunPerformanceTest(config)
359    return 0
360  except RuntimeError, e:
361    bisect_utils.OutputAnnotationStepClosed()
362    _OutputFailedResults('Error: %s' % e.message)
363    return 1
364
365
366def _RunBisectionScript(
367    config, working_directory, path_to_goma, path_to_extra_src, dry_run):
368  """Attempts to execute the bisect script with the given parameters.
369
370  Args:
371    config: A dict containing the parameters to pass to the script.
372    working_directory: A working directory to provide to the bisect script,
373      where it will store it's own copy of the depot.
374    path_to_goma: Path to goma directory.
375    path_to_extra_src: Path to extra source file.
376    dry_run: Do a dry run, skipping sync, build, and performance testing steps.
377
378  Returns:
379    An exit status code: 0 on success, otherwise 1.
380  """
381  _PrintConfigStep(config)
382
383  cmd = ['python', os.path.join(BISECT_SCRIPT_DIR, 'bisect_perf_regression.py'),
384         '-c', config['command'],
385         '-g', config['good_revision'],
386         '-b', config['bad_revision'],
387         '-m', config['metric'],
388         '--working_directory', working_directory,
389         '--output_buildbot_annotations']
390
391  if config.get('metric'):
392    cmd.extend(['-m', config['metric']])
393
394  if config['repeat_count']:
395    cmd.extend(['-r', config['repeat_count']])
396
397  if config['truncate_percent']:
398    cmd.extend(['-t', config['truncate_percent']])
399
400  if config['max_time_minutes']:
401    cmd.extend(['--max_time_minutes', config['max_time_minutes']])
402
403  if config.has_key('bisect_mode'):
404    cmd.extend(['--bisect_mode', config['bisect_mode']])
405
406  cmd.extend(['--build_preference', 'ninja'])
407
408  if '--browser=cros' in config['command']:
409    cmd.extend(['--target_platform', 'cros'])
410
411    if os.environ[CROS_BOARD_ENV] and os.environ[CROS_IP_ENV]:
412      cmd.extend(['--cros_board', os.environ[CROS_BOARD_ENV]])
413      cmd.extend(['--cros_remote_ip', os.environ[CROS_IP_ENV]])
414    else:
415      print ('Error: Cros build selected, but BISECT_CROS_IP or'
416             'BISECT_CROS_BOARD undefined.\n')
417      return 1
418
419  if 'android' in config['command']:
420    if 'android-chrome-shell' in config['command']:
421      cmd.extend(['--target_platform', 'android'])
422    elif 'android-chrome' in config['command']:
423      cmd.extend(['--target_platform', 'android-chrome'])
424    else:
425      cmd.extend(['--target_platform', 'android'])
426
427  if path_to_goma:
428    # For Windows XP platforms, goma service is not supported.
429    # Moreover we don't compile chrome when gs_bucket flag is set instead
430    # use builds archives, therefore ignore goma service for Windows XP.
431    # See http://crbug.com/330900.
432    if config.get('gs_bucket') and platform.release() == 'XP':
433      print ('Goma doesn\'t have a win32 binary, therefore it is not supported '
434             'on Windows XP platform. Please refer to crbug.com/330900.')
435      path_to_goma = None
436    cmd.append('--use_goma')
437
438  if path_to_extra_src:
439    cmd.extend(['--extra_src', path_to_extra_src])
440
441  # These flags are used to download build archives from cloud storage if
442  # available, otherwise will post a try_job_http request to build it on the
443  # try server.
444  if config.get('gs_bucket'):
445    if config.get('builder_host') and config.get('builder_port'):
446      cmd.extend(['--gs_bucket', config['gs_bucket'],
447                  '--builder_host', config['builder_host'],
448                  '--builder_port', config['builder_port']
449                 ])
450    else:
451      print ('Error: Specified gs_bucket, but missing builder_host or '
452             'builder_port information in config.')
453      return 1
454
455  if dry_run:
456    cmd.extend(['--debug_ignore_build', '--debug_ignore_sync',
457        '--debug_ignore_perf_test'])
458  cmd = [str(c) for c in cmd]
459
460  with Goma(path_to_goma) as _:
461    return_code = subprocess.call(cmd)
462
463  if return_code:
464    print ('Error: bisect_perf_regression.py returned with error %d\n'
465           % return_code)
466
467  return return_code
468
469
470def _PrintConfigStep(config):
471  """Prints out the given config, along with Buildbot annotations."""
472  bisect_utils.OutputAnnotationStepStart('Config')
473  print
474  for k, v in config.iteritems():
475    print '  %s : %s' % (k, v)
476  print
477  bisect_utils.OutputAnnotationStepClosed()
478
479
480def _OptionParser():
481  """Returns the options parser for run-bisect-perf-regression.py."""
482  usage = ('%prog [options] [-- chromium-options]\n'
483           'Used by a try bot to run the bisection script using the parameters'
484           ' provided in the auto_bisect/bisect.cfg file.')
485  parser = optparse.OptionParser(usage=usage)
486  parser.add_option('-w', '--working_directory',
487                    type='str',
488                    help='A working directory to supply to the bisection '
489                    'script, which will use it as the location to checkout '
490                    'a copy of the chromium depot.')
491  parser.add_option('-p', '--path_to_goma',
492                    type='str',
493                    help='Path to goma directory. If this is supplied, goma '
494                    'builds will be enabled.')
495  parser.add_option('--path_to_config',
496                    type='str',
497                    help='Path to the config file to use. If this is supplied, '
498                    'the bisect script will use this to override the default '
499                    'config file path. The script will attempt to load it '
500                    'as a bisect config first, then a perf config.')
501  parser.add_option('--extra_src',
502                    type='str',
503                    help='Path to extra source file. If this is supplied, '
504                    'bisect script will use this to override default behavior.')
505  parser.add_option('--dry_run',
506                    action="store_true",
507                    help='The script will perform the full bisect, but '
508                    'without syncing, building, or running the performance '
509                    'tests.')
510  return parser
511
512
513def main():
514  """Entry point for run-bisect-perf-regression.py.
515
516  Reads the config file, and then tries to either bisect a regression or
517  just run a performance test, depending on the particular config parameters
518  specified in the config file.
519  """
520  parser = _OptionParser()
521  opts, _ = parser.parse_args()
522
523  # Use the default config file path unless one was specified.
524  config_path = BISECT_CONFIG_PATH
525  if opts.path_to_config:
526    config_path = opts.path_to_config
527  config = _LoadConfigFile(config_path)
528
529  # Check if the config is valid for running bisect job.
530  config_is_valid = _ValidateBisectConfigFile(config)
531
532  if config and config_is_valid:
533    if not opts.working_directory:
534      print 'Error: missing required parameter: --working_directory\n'
535      parser.print_help()
536      return 1
537
538    return _RunBisectionScript(
539        config, opts.working_directory, opts.path_to_goma, opts.extra_src,
540        opts.dry_run)
541
542  # If it wasn't valid for running a bisect, then maybe the user wanted
543  # to run a perf test instead of a bisect job. Try reading any possible
544  # perf test config files.
545  perf_cfg_files = [RUN_TEST_CONFIG_PATH, WEBKIT_RUN_TEST_CONFIG_PATH]
546  for current_perf_cfg_file in perf_cfg_files:
547    if opts.path_to_config:
548      path_to_perf_cfg = opts.path_to_config
549    else:
550      path_to_perf_cfg = os.path.join(
551          os.path.abspath(os.path.dirname(sys.argv[0])),
552          current_perf_cfg_file)
553
554    config = _LoadConfigFile(path_to_perf_cfg)
555    config_is_valid = _ValidatePerfConfigFile(config)
556
557    if config and config_is_valid:
558      return _SetupAndRunPerformanceTest(config, opts.path_to_goma)
559
560  print ('Error: Could not load config file. Double check your changes to '
561         'auto_bisect/bisect.cfg or run-perf-test.cfg for syntax errors.\n')
562  return 1
563
564
565if __name__ == '__main__':
566  sys.exit(main())
567