1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Copyright 2015 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""MB - the Meta-Build wrapper around GYP and GN
8
9MB is a wrapper script for GYP and GN that can be used to generate build files
10for sets of canned configurations and analyze them.
11"""
12
13from __future__ import print_function
14
15import argparse
16import ast
17import errno
18import json
19import os
20import pipes
21import pprint
22import re
23import shutil
24import sys
25import subprocess
26import tempfile
27import traceback
28import urllib2
29
30from collections import OrderedDict
31
32CHROMIUM_SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(
33    os.path.abspath(__file__))))
34sys.path = [os.path.join(CHROMIUM_SRC_DIR, 'build')] + sys.path
35
36import gn_helpers
37
38
39def main(args):
40  mbw = MetaBuildWrapper()
41  return mbw.Main(args)
42
43
44class MetaBuildWrapper(object):
45  def __init__(self):
46    self.chromium_src_dir = CHROMIUM_SRC_DIR
47    self.default_config = os.path.join(self.chromium_src_dir, 'infra', 'mb',
48                                       'mb_config.pyl')
49    self.executable = sys.executable
50    self.platform = sys.platform
51    self.sep = os.sep
52    self.args = argparse.Namespace()
53    self.configs = {}
54    self.masters = {}
55    self.mixins = {}
56
57  def Main(self, args):
58    self.ParseArgs(args)
59    try:
60      ret = self.args.func()
61      if ret:
62        self.DumpInputFiles()
63      return ret
64    except KeyboardInterrupt:
65      self.Print('interrupted, exiting', stream=sys.stderr)
66      return 130
67    except Exception:
68      self.DumpInputFiles()
69      s = traceback.format_exc()
70      for l in s.splitlines():
71        self.Print(l)
72      return 1
73
74  def ParseArgs(self, argv):
75    def AddCommonOptions(subp):
76      subp.add_argument('-b', '--builder',
77                        help='builder name to look up config from')
78      subp.add_argument('-m', '--master',
79                        help='master name to look up config from')
80      subp.add_argument('-c', '--config',
81                        help='configuration to analyze')
82      subp.add_argument('--phase', type=int,
83                        help=('build phase for a given build '
84                              '(int in [1, 2, ...))'))
85      subp.add_argument('-f', '--config-file', metavar='PATH',
86                        default=self.default_config,
87                        help='path to config file '
88                            '(default is //tools/mb/mb_config.pyl)')
89      subp.add_argument('-g', '--goma-dir',
90                        help='path to goma directory')
91      subp.add_argument('--gyp-script', metavar='PATH',
92                        default=self.PathJoin('build', 'gyp_chromium'),
93                        help='path to gyp script relative to project root '
94                             '(default is %(default)s)')
95      subp.add_argument('--android-version-code',
96                        help='Sets GN arg android_default_version_code and '
97                             'GYP_DEFINE app_manifest_version_code')
98      subp.add_argument('--android-version-name',
99                        help='Sets GN arg android_default_version_name and '
100                             'GYP_DEFINE app_manifest_version_name')
101      subp.add_argument('-n', '--dryrun', action='store_true',
102                        help='Do a dry run (i.e., do nothing, just print '
103                             'the commands that will run)')
104      subp.add_argument('-v', '--verbose', action='store_true',
105                        help='verbose logging')
106
107    parser = argparse.ArgumentParser(prog='mb')
108    subps = parser.add_subparsers()
109
110    subp = subps.add_parser('analyze',
111                            help='analyze whether changes to a set of files '
112                                 'will cause a set of binaries to be rebuilt.')
113    AddCommonOptions(subp)
114    subp.add_argument('path', nargs=1,
115                      help='path build was generated into.')
116    subp.add_argument('input_path', nargs=1,
117                      help='path to a file containing the input arguments '
118                           'as a JSON object.')
119    subp.add_argument('output_path', nargs=1,
120                      help='path to a file containing the output arguments '
121                           'as a JSON object.')
122    subp.set_defaults(func=self.CmdAnalyze)
123
124    subp = subps.add_parser('gen',
125                            help='generate a new set of build files')
126    AddCommonOptions(subp)
127    subp.add_argument('--swarming-targets-file',
128                      help='save runtime dependencies for targets listed '
129                           'in file.')
130    subp.add_argument('path', nargs=1,
131                      help='path to generate build into')
132    subp.set_defaults(func=self.CmdGen)
133
134    subp = subps.add_parser('isolate',
135                            help='generate the .isolate files for a given'
136                                 'binary')
137    AddCommonOptions(subp)
138    subp.add_argument('path', nargs=1,
139                      help='path build was generated into')
140    subp.add_argument('target', nargs=1,
141                      help='ninja target to generate the isolate for')
142    subp.set_defaults(func=self.CmdIsolate)
143
144    subp = subps.add_parser('lookup',
145                            help='look up the command for a given config or '
146                                 'builder')
147    AddCommonOptions(subp)
148    subp.set_defaults(func=self.CmdLookup)
149
150    subp = subps.add_parser(
151        'run',
152        help='build and run the isolated version of a '
153             'binary',
154        formatter_class=argparse.RawDescriptionHelpFormatter)
155    subp.description = (
156        'Build, isolate, and run the given binary with the command line\n'
157        'listed in the isolate. You may pass extra arguments after the\n'
158        'target; use "--" if the extra arguments need to include switches.\n'
159        '\n'
160        'Examples:\n'
161        '\n'
162        '  % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n'
163        '    //out/Default content_browsertests\n'
164        '\n'
165        '  % tools/mb/mb.py run out/Default content_browsertests\n'
166        '\n'
167        '  % tools/mb/mb.py run out/Default content_browsertests -- \\\n'
168        '    --test-launcher-retry-limit=0'
169        '\n'
170    )
171
172    AddCommonOptions(subp)
173    subp.add_argument('-j', '--jobs', dest='jobs', type=int,
174                      help='Number of jobs to pass to ninja')
175    subp.add_argument('--no-build', dest='build', default=True,
176                      action='store_false',
177                      help='Do not build, just isolate and run')
178    subp.add_argument('path', nargs=1,
179                      help=('path to generate build into (or use).'
180                            ' This can be either a regular path or a '
181                            'GN-style source-relative path like '
182                            '//out/Default.'))
183    subp.add_argument('target', nargs=1,
184                      help='ninja target to build and run')
185    subp.add_argument('extra_args', nargs='*',
186                      help=('extra args to pass to the isolate to run. Use '
187                            '"--" as the first arg if you need to pass '
188                            'switches'))
189    subp.set_defaults(func=self.CmdRun)
190
191    subp = subps.add_parser('validate',
192                            help='validate the config file')
193    subp.add_argument('-f', '--config-file', metavar='PATH',
194                      default=self.default_config,
195                      help='path to config file '
196                          '(default is //infra/mb/mb_config.pyl)')
197    subp.set_defaults(func=self.CmdValidate)
198
199    subp = subps.add_parser('audit',
200                            help='Audit the config file to track progress')
201    subp.add_argument('-f', '--config-file', metavar='PATH',
202                      default=self.default_config,
203                      help='path to config file '
204                          '(default is //infra/mb/mb_config.pyl)')
205    subp.add_argument('-i', '--internal', action='store_true',
206                      help='check internal masters also')
207    subp.add_argument('-m', '--master', action='append',
208                      help='master to audit (default is all non-internal '
209                           'masters in file)')
210    subp.add_argument('-u', '--url-template', action='store',
211                      default='https://build.chromium.org/p/'
212                              '{master}/json/builders',
213                      help='URL scheme for JSON APIs to buildbot '
214                           '(default: %(default)s) ')
215    subp.add_argument('-c', '--check-compile', action='store_true',
216                      help='check whether tbd and master-only bots actually'
217                           ' do compiles')
218    subp.set_defaults(func=self.CmdAudit)
219
220    subp = subps.add_parser('help',
221                            help='Get help on a subcommand.')
222    subp.add_argument(nargs='?', action='store', dest='subcommand',
223                      help='The command to get help for.')
224    subp.set_defaults(func=self.CmdHelp)
225
226    self.args = parser.parse_args(argv)
227
228  def DumpInputFiles(self):
229
230    def DumpContentsOfFilePassedTo(arg_name, path):
231      if path and self.Exists(path):
232        self.Print("\n# To recreate the file passed to %s:" % arg_name)
233        self.Print("%% cat > %s <<EOF)" % path)
234        contents = self.ReadFile(path)
235        self.Print(contents)
236        self.Print("EOF\n%\n")
237
238    if getattr(self.args, 'input_path', None):
239      DumpContentsOfFilePassedTo(
240          'argv[0] (input_path)', self.args.input_path[0])
241    if getattr(self.args, 'swarming_targets_file', None):
242      DumpContentsOfFilePassedTo(
243          '--swarming-targets-file', self.args.swarming_targets_file)
244
245  def CmdAnalyze(self):
246    vals = self.Lookup()
247    self.ClobberIfNeeded(vals)
248    if vals['type'] == 'gn':
249      return self.RunGNAnalyze(vals)
250    else:
251      return self.RunGYPAnalyze(vals)
252
253  def CmdGen(self):
254    vals = self.Lookup()
255    self.ClobberIfNeeded(vals)
256    if vals['type'] == 'gn':
257      return self.RunGNGen(vals)
258    else:
259      return self.RunGYPGen(vals)
260
261  def CmdHelp(self):
262    if self.args.subcommand:
263      self.ParseArgs([self.args.subcommand, '--help'])
264    else:
265      self.ParseArgs(['--help'])
266
267  def CmdIsolate(self):
268    vals = self.GetConfig()
269    if not vals:
270      return 1
271
272    if vals['type'] == 'gn':
273      return self.RunGNIsolate(vals)
274    else:
275      return self.Build('%s_run' % self.args.target[0])
276
277  def CmdLookup(self):
278    vals = self.Lookup()
279    if vals['type'] == 'gn':
280      cmd = self.GNCmd('gen', '_path_')
281      gn_args = self.GNArgs(vals)
282      self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
283      env = None
284    else:
285      cmd, env = self.GYPCmd('_path_', vals)
286
287    self.PrintCmd(cmd, env)
288    return 0
289
290  def CmdRun(self):
291    vals = self.GetConfig()
292    if not vals:
293      return 1
294
295    build_dir = self.args.path[0]
296    target = self.args.target[0]
297
298    if vals['type'] == 'gn':
299      if self.args.build:
300        ret = self.Build(target)
301        if ret:
302          return ret
303      ret = self.RunGNIsolate(vals)
304      if ret:
305        return ret
306    else:
307      ret = self.Build('%s_run' % target)
308      if ret:
309        return ret
310
311    cmd = [
312        self.executable,
313        self.PathJoin('tools', 'swarming_client', 'isolate.py'),
314        'run',
315        '-s',
316        self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
317    ]
318    if self.args.extra_args:
319        cmd += ['--'] + self.args.extra_args
320
321    ret, _, _ = self.Run(cmd, force_verbose=False, buffer_output=False)
322
323    return ret
324
325  def CmdValidate(self, print_ok=True):
326    errs = []
327
328    # Read the file to make sure it parses.
329    self.ReadConfigFile()
330
331    # Build a list of all of the configs referenced by builders.
332    all_configs = {}
333    for master in self.masters:
334      for config in self.masters[master].values():
335        if isinstance(config, list):
336          for c in config:
337            all_configs[c] = master
338        else:
339          all_configs[config] = master
340
341    # Check that every referenced args file or config actually exists.
342    for config, loc in all_configs.items():
343      if config.startswith('//'):
344        if not self.Exists(self.ToAbsPath(config)):
345          errs.append('Unknown args file "%s" referenced from "%s".' %
346                      (config, loc))
347      elif not config in self.configs:
348        errs.append('Unknown config "%s" referenced from "%s".' %
349                    (config, loc))
350
351    # Check that every actual config is actually referenced.
352    for config in self.configs:
353      if not config in all_configs:
354        errs.append('Unused config "%s".' % config)
355
356    # Figure out the whole list of mixins, and check that every mixin
357    # listed by a config or another mixin actually exists.
358    referenced_mixins = set()
359    for config, mixins in self.configs.items():
360      for mixin in mixins:
361        if not mixin in self.mixins:
362          errs.append('Unknown mixin "%s" referenced by config "%s".' %
363                      (mixin, config))
364        referenced_mixins.add(mixin)
365
366    for mixin in self.mixins:
367      for sub_mixin in self.mixins[mixin].get('mixins', []):
368        if not sub_mixin in self.mixins:
369          errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
370                      (sub_mixin, mixin))
371        referenced_mixins.add(sub_mixin)
372
373    # Check that every mixin defined is actually referenced somewhere.
374    for mixin in self.mixins:
375      if not mixin in referenced_mixins:
376        errs.append('Unreferenced mixin "%s".' % mixin)
377
378    if errs:
379      raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
380                    '\n  ' + '\n  '.join(errs))
381
382    if print_ok:
383      self.Print('mb config file %s looks ok.' % self.args.config_file)
384    return 0
385
386  def CmdAudit(self):
387    """Track the progress of the GYP->GN migration on the bots."""
388
389    # First, make sure the config file is okay, but don't print anything
390    # if it is (it will throw an error if it isn't).
391    self.CmdValidate(print_ok=False)
392
393    stats = OrderedDict()
394    STAT_MASTER_ONLY = 'Master only'
395    STAT_CONFIG_ONLY = 'Config only'
396    STAT_TBD = 'Still TBD'
397    STAT_GYP = 'Still GYP'
398    STAT_DONE = 'Done (on GN)'
399    stats[STAT_MASTER_ONLY] = 0
400    stats[STAT_CONFIG_ONLY] = 0
401    stats[STAT_TBD] = 0
402    stats[STAT_GYP] = 0
403    stats[STAT_DONE] = 0
404
405    def PrintBuilders(heading, builders, notes):
406      stats.setdefault(heading, 0)
407      stats[heading] += len(builders)
408      if builders:
409        self.Print('  %s:' % heading)
410        for builder in sorted(builders):
411          self.Print('    %s%s' % (builder, notes[builder]))
412
413    self.ReadConfigFile()
414
415    masters = self.args.master or self.masters
416    for master in sorted(masters):
417      url = self.args.url_template.replace('{master}', master)
418
419      self.Print('Auditing %s' % master)
420
421      MASTERS_TO_SKIP = (
422        'client.skia',
423        'client.v8.fyi',
424        'tryserver.v8',
425      )
426      if master in MASTERS_TO_SKIP:
427        # Skip these bots because converting them is the responsibility of
428        # those teams and out of scope for the Chromium migration to GN.
429        self.Print('  Skipped (out of scope)')
430        self.Print('')
431        continue
432
433      INTERNAL_MASTERS = ('official.desktop', 'official.desktop.continuous',
434                          'internal.client.kitchensync')
435      if master in INTERNAL_MASTERS and not self.args.internal:
436        # Skip these because the servers aren't accessible by default ...
437        self.Print('  Skipped (internal)')
438        self.Print('')
439        continue
440
441      try:
442        # Fetch the /builders contents from the buildbot master. The
443        # keys of the dict are the builder names themselves.
444        json_contents = self.Fetch(url)
445        d = json.loads(json_contents)
446      except Exception as e:
447        self.Print(str(e))
448        return 1
449
450      config_builders = set(self.masters[master])
451      master_builders = set(d.keys())
452      both = master_builders & config_builders
453      master_only = master_builders - config_builders
454      config_only = config_builders - master_builders
455      tbd = set()
456      gyp = set()
457      done = set()
458      notes = {builder: '' for builder in config_builders | master_builders}
459
460      for builder in both:
461        config = self.masters[master][builder]
462        if config == 'tbd':
463          tbd.add(builder)
464        elif isinstance(config, list):
465          vals = self.FlattenConfig(config[0])
466          if vals['type'] == 'gyp':
467            gyp.add(builder)
468          else:
469            done.add(builder)
470        elif config.startswith('//'):
471          done.add(builder)
472        else:
473          vals = self.FlattenConfig(config)
474          if vals['type'] == 'gyp':
475            gyp.add(builder)
476          else:
477            done.add(builder)
478
479      if self.args.check_compile and (tbd or master_only):
480        either = tbd | master_only
481        for builder in either:
482          notes[builder] = ' (' + self.CheckCompile(master, builder) +')'
483
484      if master_only or config_only or tbd or gyp:
485        PrintBuilders(STAT_MASTER_ONLY, master_only, notes)
486        PrintBuilders(STAT_CONFIG_ONLY, config_only, notes)
487        PrintBuilders(STAT_TBD, tbd, notes)
488        PrintBuilders(STAT_GYP, gyp, notes)
489      else:
490        self.Print('  All GN!')
491
492      stats[STAT_DONE] += len(done)
493
494      self.Print('')
495
496    fmt = '{:<27} {:>4}'
497    self.Print(fmt.format('Totals', str(sum(int(v) for v in stats.values()))))
498    self.Print(fmt.format('-' * 27, '----'))
499    for stat, count in stats.items():
500      self.Print(fmt.format(stat, str(count)))
501
502    return 0
503
504  def GetConfig(self):
505    build_dir = self.args.path[0]
506
507    vals = {}
508    if self.args.builder or self.args.master or self.args.config:
509      vals = self.Lookup()
510      if vals['type'] == 'gn':
511        # Re-run gn gen in order to ensure the config is consistent with the
512        # build dir.
513        self.RunGNGen(vals)
514      return vals
515
516    mb_type_path = self.PathJoin(self.ToAbsPath(build_dir), 'mb_type')
517    if not self.Exists(mb_type_path):
518      toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
519                                     'toolchain.ninja')
520      if not self.Exists(toolchain_path):
521        self.Print('Must either specify a path to an existing GN build dir '
522                   'or pass in a -m/-b pair or a -c flag to specify the '
523                   'configuration')
524        return {}
525      else:
526        mb_type = 'gn'
527    else:
528      mb_type = self.ReadFile(mb_type_path).strip()
529
530    if mb_type == 'gn':
531      vals = self.GNValsFromDir(build_dir)
532    else:
533      vals = {}
534    vals['type'] = mb_type
535
536    return vals
537
538  def GNValsFromDir(self, build_dir):
539    args_contents = ""
540    gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')
541    if self.Exists(gn_args_path):
542      args_contents = self.ReadFile(gn_args_path)
543    gn_args = []
544    for l in args_contents.splitlines():
545      fields = l.split(' ')
546      name = fields[0]
547      val = ' '.join(fields[2:])
548      gn_args.append('%s=%s' % (name, val))
549
550    return {
551      'gn_args': ' '.join(gn_args),
552      'type': 'gn',
553    }
554
555  def Lookup(self):
556    vals = self.ReadBotConfig()
557    if not vals:
558      self.ReadConfigFile()
559      config = self.ConfigFromArgs()
560      if config.startswith('//'):
561        if not self.Exists(self.ToAbsPath(config)):
562          raise MBErr('args file "%s" not found' % config)
563        vals = {
564          'args_file': config,
565          'cros_passthrough': False,
566          'gn_args': '',
567          'gyp_crosscompile': False,
568          'gyp_defines': '',
569          'type': 'gn',
570        }
571      else:
572        if not config in self.configs:
573          raise MBErr('Config "%s" not found in %s' %
574                      (config, self.args.config_file))
575        vals = self.FlattenConfig(config)
576
577    # Do some basic sanity checking on the config so that we
578    # don't have to do this in every caller.
579    assert 'type' in vals, 'No meta-build type specified in the config'
580    assert vals['type'] in ('gn', 'gyp'), (
581        'Unknown meta-build type "%s"' % vals['gn_args'])
582
583    return vals
584
585  def ReadBotConfig(self):
586    if not self.args.master or not self.args.builder:
587      return {}
588    path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots',
589                         self.args.master, self.args.builder + '.json')
590    if not self.Exists(path):
591      return {}
592
593    contents = json.loads(self.ReadFile(path))
594    gyp_vals = contents.get('GYP_DEFINES', {})
595    if isinstance(gyp_vals, dict):
596      gyp_defines = ' '.join('%s=%s' % (k, v) for k, v in gyp_vals.items())
597    else:
598      gyp_defines = ' '.join(gyp_vals)
599    gn_args = ' '.join(contents.get('gn_args', []))
600
601    return {
602        'args_file': '',
603        'cros_passthrough': False,
604        'gn_args': gn_args,
605        'gyp_crosscompile': False,
606        'gyp_defines': gyp_defines,
607        'type': contents.get('mb_type', ''),
608    }
609
610  def ReadConfigFile(self):
611    if not self.Exists(self.args.config_file):
612      raise MBErr('config file not found at %s' % self.args.config_file)
613
614    try:
615      contents = ast.literal_eval(self.ReadFile(self.args.config_file))
616    except SyntaxError as e:
617      raise MBErr('Failed to parse config file "%s": %s' %
618                 (self.args.config_file, e))
619
620    self.configs = contents['configs']
621    self.masters = contents['masters']
622    self.mixins = contents['mixins']
623
624  def ConfigFromArgs(self):
625    if self.args.config:
626      if self.args.master or self.args.builder:
627        raise MBErr('Can not specific both -c/--config and -m/--master or '
628                    '-b/--builder')
629
630      return self.args.config
631
632    if not self.args.master or not self.args.builder:
633      raise MBErr('Must specify either -c/--config or '
634                  '(-m/--master and -b/--builder)')
635
636    if not self.args.master in self.masters:
637      raise MBErr('Master name "%s" not found in "%s"' %
638                  (self.args.master, self.args.config_file))
639
640    if not self.args.builder in self.masters[self.args.master]:
641      raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
642                  (self.args.builder, self.args.master, self.args.config_file))
643
644    config = self.masters[self.args.master][self.args.builder]
645    if isinstance(config, list):
646      if self.args.phase is None:
647        raise MBErr('Must specify a build --phase for %s on %s' %
648                    (self.args.builder, self.args.master))
649      phase = int(self.args.phase)
650      if phase < 1 or phase > len(config):
651        raise MBErr('Phase %d out of bounds for %s on %s' %
652                    (phase, self.args.builder, self.args.master))
653      return config[phase-1]
654
655    if self.args.phase is not None:
656      raise MBErr('Must not specify a build --phase for %s on %s' %
657                  (self.args.builder, self.args.master))
658    return config
659
660  def FlattenConfig(self, config):
661    mixins = self.configs[config]
662    vals = {
663      'args_file': '',
664      'cros_passthrough': False,
665      'gn_args': [],
666      'gyp_defines': '',
667      'gyp_crosscompile': False,
668      'type': None,
669    }
670
671    visited = []
672    self.FlattenMixins(mixins, vals, visited)
673    return vals
674
675  def FlattenMixins(self, mixins, vals, visited):
676    for m in mixins:
677      if m not in self.mixins:
678        raise MBErr('Unknown mixin "%s"' % m)
679
680      visited.append(m)
681
682      mixin_vals = self.mixins[m]
683
684      if 'cros_passthrough' in mixin_vals:
685        vals['cros_passthrough'] = mixin_vals['cros_passthrough']
686      if 'gn_args' in mixin_vals:
687        if vals['gn_args']:
688          vals['gn_args'] += ' ' + mixin_vals['gn_args']
689        else:
690          vals['gn_args'] = mixin_vals['gn_args']
691      if 'gyp_crosscompile' in mixin_vals:
692        vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
693      if 'gyp_defines' in mixin_vals:
694        if vals['gyp_defines']:
695          vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
696        else:
697          vals['gyp_defines'] = mixin_vals['gyp_defines']
698      if 'type' in mixin_vals:
699        vals['type'] = mixin_vals['type']
700
701      if 'mixins' in mixin_vals:
702        self.FlattenMixins(mixin_vals['mixins'], vals, visited)
703    return vals
704
705  def ClobberIfNeeded(self, vals):
706    path = self.args.path[0]
707    build_dir = self.ToAbsPath(path)
708    mb_type_path = self.PathJoin(build_dir, 'mb_type')
709    needs_clobber = False
710    new_mb_type = vals['type']
711    if self.Exists(build_dir):
712      if self.Exists(mb_type_path):
713        old_mb_type = self.ReadFile(mb_type_path)
714        if old_mb_type != new_mb_type:
715          self.Print("Build type mismatch: was %s, will be %s, clobbering %s" %
716                     (old_mb_type, new_mb_type, path))
717          needs_clobber = True
718      else:
719        # There is no 'mb_type' file in the build directory, so this probably
720        # means that the prior build(s) were not done through mb, and we
721        # have no idea if this was a GYP build or a GN build. Clobber it
722        # to be safe.
723        self.Print("%s/mb_type missing, clobbering to be safe" % path)
724        needs_clobber = True
725
726    if self.args.dryrun:
727      return
728
729    if needs_clobber:
730      self.RemoveDirectory(build_dir)
731
732    self.MaybeMakeDirectory(build_dir)
733    self.WriteFile(mb_type_path, new_mb_type)
734
735  def RunGNGen(self, vals):
736    build_dir = self.args.path[0]
737
738    cmd = self.GNCmd('gen', build_dir, '--check')
739    gn_args = self.GNArgs(vals)
740
741    # Since GN hasn't run yet, the build directory may not even exist.
742    self.MaybeMakeDirectory(self.ToAbsPath(build_dir))
743
744    gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
745    self.WriteFile(gn_args_path, gn_args, force_verbose=True)
746
747    swarming_targets = []
748    if getattr(self.args, 'swarming_targets_file', None):
749      # We need GN to generate the list of runtime dependencies for
750      # the compile targets listed (one per line) in the file so
751      # we can run them via swarming. We use ninja_to_gn.pyl to convert
752      # the compile targets to the matching GN labels.
753      path = self.args.swarming_targets_file
754      if not self.Exists(path):
755        self.WriteFailureAndRaise('"%s" does not exist' % path,
756                                  output_path=None)
757      contents = self.ReadFile(path)
758      swarming_targets = set(contents.splitlines())
759      gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
760          self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
761      gn_labels = []
762      err = ''
763      for target in swarming_targets:
764        target_name = self.GNTargetName(target)
765        if not target_name in gn_isolate_map:
766          err += ('test target "%s" not found\n' % target_name)
767        elif gn_isolate_map[target_name]['type'] == 'unknown':
768          err += ('test target "%s" type is unknown\n' % target_name)
769        else:
770          gn_labels.append(gn_isolate_map[target_name]['label'])
771
772      if err:
773          raise MBErr('Error: Failed to match swarming targets to %s:\n%s' %
774                      ('//testing/buildbot/gn_isolate_map.pyl', err))
775
776      gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
777      self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
778      cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
779
780    ret, _, _ = self.Run(cmd)
781    if ret:
782        # If `gn gen` failed, we should exit early rather than trying to
783        # generate isolates. Run() will have already logged any error output.
784        self.Print('GN gen failed: %d' % ret)
785        return ret
786
787    android = 'target_os="android"' in vals['gn_args']
788    for target in swarming_targets:
789      if android:
790        # Android targets may be either android_apk or executable. The former
791        # will result in runtime_deps associated with the stamp file, while the
792        # latter will result in runtime_deps associated with the executable.
793        target_name = self.GNTargetName(target)
794        label = gn_isolate_map[target_name]['label']
795        runtime_deps_targets = [
796            target_name + '.runtime_deps',
797            'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
798      elif gn_isolate_map[target]['type'] == 'gpu_browser_test':
799        if self.platform == 'win32':
800          runtime_deps_targets = ['browser_tests.exe.runtime_deps']
801        else:
802          runtime_deps_targets = ['browser_tests.runtime_deps']
803      elif (gn_isolate_map[target]['type'] == 'script' or
804            gn_isolate_map[target].get('label_type') == 'group'):
805        # For script targets, the build target is usually a group,
806        # for which gn generates the runtime_deps next to the stamp file
807        # for the label, which lives under the obj/ directory.
808        label = gn_isolate_map[target]['label']
809        runtime_deps_targets = [
810            'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
811      elif self.platform == 'win32':
812        runtime_deps_targets = [target + '.exe.runtime_deps']
813      else:
814        runtime_deps_targets = [target + '.runtime_deps']
815
816      for r in runtime_deps_targets:
817        runtime_deps_path = self.ToAbsPath(build_dir, r)
818        if self.Exists(runtime_deps_path):
819          break
820      else:
821        raise MBErr('did not generate any of %s' %
822                    ', '.join(runtime_deps_targets))
823
824      command, extra_files = self.GetIsolateCommand(target, vals,
825                                                    gn_isolate_map)
826
827      runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
828
829      self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
830                             extra_files)
831
832    return 0
833
834  def RunGNIsolate(self, vals):
835    gn_isolate_map = ast.literal_eval(self.ReadFile(self.PathJoin(
836        self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
837
838    build_dir = self.args.path[0]
839    target = self.args.target[0]
840    target_name = self.GNTargetName(target)
841    command, extra_files = self.GetIsolateCommand(target, vals, gn_isolate_map)
842
843    label = gn_isolate_map[target_name]['label']
844    cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
845    ret, out, _ = self.Call(cmd)
846    if ret:
847      if out:
848        self.Print(out)
849      return ret
850
851    runtime_deps = out.splitlines()
852
853    self.WriteIsolateFiles(build_dir, command, target, runtime_deps,
854                           extra_files)
855
856    ret, _, _ = self.Run([
857        self.executable,
858        self.PathJoin('tools', 'swarming_client', 'isolate.py'),
859        'check',
860        '-i',
861        self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
862        '-s',
863        self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target))],
864        buffer_output=False)
865
866    return ret
867
868  def WriteIsolateFiles(self, build_dir, command, target, runtime_deps,
869                        extra_files):
870    isolate_path = self.ToAbsPath(build_dir, target + '.isolate')
871    self.WriteFile(isolate_path,
872      pprint.pformat({
873        'variables': {
874          'command': command,
875          'files': sorted(runtime_deps + extra_files),
876        }
877      }) + '\n')
878
879    self.WriteJSON(
880      {
881        'args': [
882          '--isolated',
883          self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
884          '--isolate',
885          self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
886        ],
887        'dir': self.chromium_src_dir,
888        'version': 1,
889      },
890      isolate_path + 'd.gen.json',
891    )
892
893  def GNCmd(self, subcommand, path, *args):
894    if self.platform == 'linux2':
895      subdir, exe = 'linux64', 'gn'
896    elif self.platform == 'darwin':
897      subdir, exe = 'mac', 'gn'
898    else:
899      subdir, exe = 'win', 'gn.exe'
900
901    gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe)
902
903    return [gn_path, subcommand, path] + list(args)
904
905  def GNArgs(self, vals):
906    if vals['cros_passthrough']:
907      if not 'GN_ARGS' in os.environ:
908        raise MBErr('MB is expecting GN_ARGS to be in the environment')
909      gn_args = os.environ['GN_ARGS']
910      if not re.search('target_os.*=.*"chromeos"', gn_args):
911        raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' %
912                    gn_args)
913    else:
914      gn_args = vals['gn_args']
915
916    if self.args.goma_dir:
917      gn_args += ' goma_dir="%s"' % self.args.goma_dir
918
919    android_version_code = self.args.android_version_code
920    if android_version_code:
921      gn_args += ' android_default_version_code="%s"' % android_version_code
922
923    android_version_name = self.args.android_version_name
924    if android_version_name:
925      gn_args += ' android_default_version_name="%s"' % android_version_name
926
927    # Canonicalize the arg string into a sorted, newline-separated list
928    # of key-value pairs, and de-dup the keys if need be so that only
929    # the last instance of each arg is listed.
930    gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args))
931
932    args_file = vals.get('args_file', None)
933    if args_file:
934      gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
935    return gn_args
936
937  def RunGYPGen(self, vals):
938    path = self.args.path[0]
939
940    output_dir = self.ParseGYPConfigPath(path)
941    cmd, env = self.GYPCmd(output_dir, vals)
942    ret, _, _ = self.Run(cmd, env=env)
943    return ret
944
945  def RunGYPAnalyze(self, vals):
946    output_dir = self.ParseGYPConfigPath(self.args.path[0])
947    if self.args.verbose:
948      inp = self.ReadInputJSON(['files', 'test_targets',
949                                'additional_compile_targets'])
950      self.Print()
951      self.Print('analyze input:')
952      self.PrintJSON(inp)
953      self.Print()
954
955    cmd, env = self.GYPCmd(output_dir, vals)
956    cmd.extend(['-f', 'analyzer',
957                '-G', 'config_path=%s' % self.args.input_path[0],
958                '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
959    ret, _, _ = self.Run(cmd, env=env)
960    if not ret and self.args.verbose:
961      outp = json.loads(self.ReadFile(self.args.output_path[0]))
962      self.Print()
963      self.Print('analyze output:')
964      self.PrintJSON(outp)
965      self.Print()
966
967    return ret
968
969  def GetIsolateCommand(self, target, vals, gn_isolate_map):
970    android = 'target_os="android"' in vals['gn_args']
971
972    # This needs to mirror the settings in //build/config/ui.gni:
973    # use_x11 = is_linux && !use_ozone.
974    use_x11 = (self.platform == 'linux2' and
975               not android and
976               not 'use_ozone=true' in vals['gn_args'])
977
978    asan = 'is_asan=true' in vals['gn_args']
979    msan = 'is_msan=true' in vals['gn_args']
980    tsan = 'is_tsan=true' in vals['gn_args']
981
982    target_name = self.GNTargetName(target)
983    test_type = gn_isolate_map[target_name]['type']
984
985    executable = gn_isolate_map[target_name].get('executable', target_name)
986    executable_suffix = '.exe' if self.platform == 'win32' else ''
987
988    cmdline = []
989    extra_files = []
990
991    if android and test_type != "script":
992      logdog_command = [
993          '--logdog-bin-cmd', './../../bin/logdog_butler',
994          '--project', 'chromium',
995          '--service-account-json',
996          '/creds/service_accounts/service-account-luci-logdog-publisher.json',
997          '--prefix', 'android/swarming/logcats/${SWARMING_TASK_ID}',
998          '--source', '${ISOLATED_OUTDIR}/logcats',
999          '--name', 'unified_logcats',
1000      ]
1001      test_cmdline = [
1002          self.PathJoin('bin', 'run_%s' % target_name),
1003          '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats',
1004          '--target-devices-file', '${SWARMING_BOT_FILE}',
1005          '-v'
1006      ]
1007      cmdline = (['./../../build/android/test_wrapper/logdog_wrapper.py']
1008                 + logdog_command + test_cmdline)
1009    elif use_x11 and test_type == 'windowed_test_launcher':
1010      extra_files = [
1011          'xdisplaycheck',
1012          '../../testing/test_env.py',
1013          '../../testing/xvfb.py',
1014      ]
1015      cmdline = [
1016        '../../testing/xvfb.py',
1017        '.',
1018        './' + str(executable) + executable_suffix,
1019        '--brave-new-test-launcher',
1020        '--test-launcher-bot-mode',
1021        '--asan=%d' % asan,
1022        '--msan=%d' % msan,
1023        '--tsan=%d' % tsan,
1024      ]
1025    elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
1026      extra_files = [
1027          '../../testing/test_env.py'
1028      ]
1029      cmdline = [
1030          '../../testing/test_env.py',
1031          './' + str(executable) + executable_suffix,
1032          '--brave-new-test-launcher',
1033          '--test-launcher-bot-mode',
1034          '--asan=%d' % asan,
1035          '--msan=%d' % msan,
1036          '--tsan=%d' % tsan,
1037      ]
1038    elif test_type == 'gpu_browser_test':
1039      extra_files = [
1040          '../../testing/test_env.py'
1041      ]
1042      gtest_filter = gn_isolate_map[target]['gtest_filter']
1043      cmdline = [
1044          '../../testing/test_env.py',
1045          './browser_tests' + executable_suffix,
1046          '--test-launcher-bot-mode',
1047          '--enable-gpu',
1048          '--test-launcher-jobs=1',
1049          '--gtest_filter=%s' % gtest_filter,
1050      ]
1051    elif test_type == 'script':
1052      extra_files = [
1053          '../../testing/test_env.py'
1054      ]
1055      cmdline = [
1056          '../../testing/test_env.py',
1057          '../../' + self.ToSrcRelPath(gn_isolate_map[target]['script'])
1058      ]
1059    elif test_type in ('raw'):
1060      extra_files = []
1061      cmdline = [
1062          './' + str(target) + executable_suffix,
1063      ]
1064
1065    else:
1066      self.WriteFailureAndRaise('No command line for %s found (test type %s).'
1067                                % (target, test_type), output_path=None)
1068
1069    cmdline += gn_isolate_map[target_name].get('args', [])
1070
1071    return cmdline, extra_files
1072
1073  def ToAbsPath(self, build_path, *comps):
1074    return self.PathJoin(self.chromium_src_dir,
1075                         self.ToSrcRelPath(build_path),
1076                         *comps)
1077
1078  def ToSrcRelPath(self, path):
1079    """Returns a relative path from the top of the repo."""
1080    if path.startswith('//'):
1081      return path[2:].replace('/', self.sep)
1082    return self.RelPath(path, self.chromium_src_dir)
1083
1084  def ParseGYPConfigPath(self, path):
1085    rpath = self.ToSrcRelPath(path)
1086    output_dir, _, _ = rpath.rpartition(self.sep)
1087    return output_dir
1088
1089  def GYPCmd(self, output_dir, vals):
1090    if vals['cros_passthrough']:
1091      if not 'GYP_DEFINES' in os.environ:
1092        raise MBErr('MB is expecting GYP_DEFINES to be in the environment')
1093      gyp_defines = os.environ['GYP_DEFINES']
1094      if not 'chromeos=1' in gyp_defines:
1095        raise MBErr('GYP_DEFINES is missing chromeos=1: (GYP_DEFINES=%s)' %
1096                    gyp_defines)
1097    else:
1098      gyp_defines = vals['gyp_defines']
1099
1100    goma_dir = self.args.goma_dir
1101
1102    # GYP uses shlex.split() to split the gyp defines into separate arguments,
1103    # so we can support backslashes and and spaces in arguments by quoting
1104    # them, even on Windows, where this normally wouldn't work.
1105    if goma_dir and ('\\' in goma_dir or ' ' in goma_dir):
1106      goma_dir = "'%s'" % goma_dir
1107
1108    if goma_dir:
1109      gyp_defines += ' gomadir=%s' % goma_dir
1110
1111    android_version_code = self.args.android_version_code
1112    if android_version_code:
1113      gyp_defines += ' app_manifest_version_code=%s' % android_version_code
1114
1115    android_version_name = self.args.android_version_name
1116    if android_version_name:
1117      gyp_defines += ' app_manifest_version_name=%s' % android_version_name
1118
1119    cmd = [
1120        self.executable,
1121        self.args.gyp_script,
1122        '-G',
1123        'output_dir=' + output_dir,
1124    ]
1125
1126    # Ensure that we have an environment that only contains
1127    # the exact values of the GYP variables we need.
1128    env = os.environ.copy()
1129
1130    # This is a terrible hack to work around the fact that
1131    # //tools/clang/scripts/update.py is invoked by GYP and GN but
1132    # currently relies on an environment variable to figure out
1133    # what revision to embed in the command line #defines.
1134    # For GN, we've made this work via a gn arg that will cause update.py
1135    # to get an additional command line arg, but getting that to work
1136    # via GYP_DEFINES has proven difficult, so we rewrite the GYP_DEFINES
1137    # to get rid of the arg and add the old var in, instead.
1138    # See crbug.com/582737 for more on this. This can hopefully all
1139    # go away with GYP.
1140    m = re.search('llvm_force_head_revision=1\s*', gyp_defines)
1141    if m:
1142      env['LLVM_FORCE_HEAD_REVISION'] = '1'
1143      gyp_defines = gyp_defines.replace(m.group(0), '')
1144
1145    # This is another terrible hack to work around the fact that
1146    # GYP sets the link concurrency to use via the GYP_LINK_CONCURRENCY
1147    # environment variable, and not via a proper GYP_DEFINE. See
1148    # crbug.com/611491 for more on this.
1149    m = re.search('gyp_link_concurrency=(\d+)(\s*)', gyp_defines)
1150    if m:
1151      env['GYP_LINK_CONCURRENCY'] = m.group(1)
1152      gyp_defines = gyp_defines.replace(m.group(0), '')
1153
1154    env['GYP_GENERATORS'] = 'ninja'
1155    if 'GYP_CHROMIUM_NO_ACTION' in env:
1156      del env['GYP_CHROMIUM_NO_ACTION']
1157    if 'GYP_CROSSCOMPILE' in env:
1158      del env['GYP_CROSSCOMPILE']
1159    env['GYP_DEFINES'] = gyp_defines
1160    if vals['gyp_crosscompile']:
1161      env['GYP_CROSSCOMPILE'] = '1'
1162    return cmd, env
1163
1164  def RunGNAnalyze(self, vals):
1165    # analyze runs before 'gn gen' now, so we need to run gn gen
1166    # in order to ensure that we have a build directory.
1167    ret = self.RunGNGen(vals)
1168    if ret:
1169      return ret
1170
1171    inp = self.ReadInputJSON(['files', 'test_targets',
1172                              'additional_compile_targets'])
1173    if self.args.verbose:
1174      self.Print()
1175      self.Print('analyze input:')
1176      self.PrintJSON(inp)
1177      self.Print()
1178
1179    # TODO(crbug.com/555273) - currently GN treats targets and
1180    # additional_compile_targets identically since we can't tell the
1181    # difference between a target that is a group in GN and one that isn't.
1182    # We should eventually fix this and treat the two types differently.
1183    targets = (set(inp['test_targets']) |
1184               set(inp['additional_compile_targets']))
1185
1186    output_path = self.args.output_path[0]
1187
1188    # Bail out early if a GN file was modified, since 'gn refs' won't know
1189    # what to do about it. Also, bail out early if 'all' was asked for,
1190    # since we can't deal with it yet.
1191    if (any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']) or
1192        'all' in targets):
1193      self.WriteJSON({
1194            'status': 'Found dependency (all)',
1195            'compile_targets': sorted(targets),
1196            'test_targets': sorted(targets & set(inp['test_targets'])),
1197          }, output_path)
1198      return 0
1199
1200    # This shouldn't normally happen, but could due to unusual race conditions,
1201    # like a try job that gets scheduled before a patch lands but runs after
1202    # the patch has landed.
1203    if not inp['files']:
1204      self.Print('Warning: No files modified in patch, bailing out early.')
1205      self.WriteJSON({
1206            'status': 'No dependency',
1207            'compile_targets': [],
1208            'test_targets': [],
1209          }, output_path)
1210      return 0
1211
1212    ret = 0
1213    response_file = self.TempFile()
1214    response_file.write('\n'.join(inp['files']) + '\n')
1215    response_file.close()
1216
1217    matching_targets = set()
1218    try:
1219      cmd = self.GNCmd('refs',
1220                       self.args.path[0],
1221                       '@%s' % response_file.name,
1222                       '--all',
1223                       '--as=output')
1224      ret, out, _ = self.Run(cmd, force_verbose=False)
1225      if ret and not 'The input matches no targets' in out:
1226        self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
1227                                  output_path)
1228      build_dir = self.ToSrcRelPath(self.args.path[0]) + self.sep
1229      for output in out.splitlines():
1230        build_output = output.replace(build_dir, '')
1231        if build_output in targets:
1232          matching_targets.add(build_output)
1233
1234      cmd = self.GNCmd('refs',
1235                       self.args.path[0],
1236                       '@%s' % response_file.name,
1237                       '--all')
1238      ret, out, _ = self.Run(cmd, force_verbose=False)
1239      if ret and not 'The input matches no targets' in out:
1240        self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
1241                                  output_path)
1242      for label in out.splitlines():
1243        build_target = label[2:]
1244        # We want to accept 'chrome/android:chrome_public_apk' and
1245        # just 'chrome_public_apk'. This may result in too many targets
1246        # getting built, but we can adjust that later if need be.
1247        for input_target in targets:
1248          if (input_target == build_target or
1249              build_target.endswith(':' + input_target)):
1250            matching_targets.add(input_target)
1251    finally:
1252      self.RemoveFile(response_file.name)
1253
1254    if matching_targets:
1255      self.WriteJSON({
1256            'status': 'Found dependency',
1257            'compile_targets': sorted(matching_targets),
1258            'test_targets': sorted(matching_targets &
1259                                   set(inp['test_targets'])),
1260          }, output_path)
1261    else:
1262      self.WriteJSON({
1263          'status': 'No dependency',
1264          'compile_targets': [],
1265          'test_targets': [],
1266      }, output_path)
1267
1268    if self.args.verbose:
1269      outp = json.loads(self.ReadFile(output_path))
1270      self.Print()
1271      self.Print('analyze output:')
1272      self.PrintJSON(outp)
1273      self.Print()
1274
1275    return 0
1276
1277  def ReadInputJSON(self, required_keys):
1278    path = self.args.input_path[0]
1279    output_path = self.args.output_path[0]
1280    if not self.Exists(path):
1281      self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
1282
1283    try:
1284      inp = json.loads(self.ReadFile(path))
1285    except Exception as e:
1286      self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
1287                                (path, e), output_path)
1288
1289    for k in required_keys:
1290      if not k in inp:
1291        self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
1292                                  output_path)
1293
1294    return inp
1295
1296  def WriteFailureAndRaise(self, msg, output_path):
1297    if output_path:
1298      self.WriteJSON({'error': msg}, output_path, force_verbose=True)
1299    raise MBErr(msg)
1300
1301  def WriteJSON(self, obj, path, force_verbose=False):
1302    try:
1303      self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
1304                     force_verbose=force_verbose)
1305    except Exception as e:
1306      raise MBErr('Error %s writing to the output path "%s"' %
1307                 (e, path))
1308
1309  def CheckCompile(self, master, builder):
1310    url_template = self.args.url_template + '/{builder}/builds/_all?as_text=1'
1311    url = urllib2.quote(url_template.format(master=master, builder=builder),
1312                        safe=':/()?=')
1313    try:
1314      builds = json.loads(self.Fetch(url))
1315    except Exception as e:
1316      return str(e)
1317    successes = sorted(
1318        [int(x) for x in builds.keys() if "text" in builds[x] and
1319          cmp(builds[x]["text"][:2], ["build", "successful"]) == 0],
1320        reverse=True)
1321    if not successes:
1322      return "no successful builds"
1323    build = builds[str(successes[0])]
1324    step_names = set([step["name"] for step in build["steps"]])
1325    compile_indicators = set(["compile", "compile (with patch)", "analyze"])
1326    if compile_indicators & step_names:
1327      return "compiles"
1328    return "does not compile"
1329
1330  def PrintCmd(self, cmd, env):
1331    if self.platform == 'win32':
1332      env_prefix = 'set '
1333      env_quoter = QuoteForSet
1334      shell_quoter = QuoteForCmd
1335    else:
1336      env_prefix = ''
1337      env_quoter = pipes.quote
1338      shell_quoter = pipes.quote
1339
1340    def print_env(var):
1341      if env and var in env:
1342        self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var])))
1343
1344    print_env('GYP_CROSSCOMPILE')
1345    print_env('GYP_DEFINES')
1346    print_env('GYP_LINK_CONCURRENCY')
1347    print_env('LLVM_FORCE_HEAD_REVISION')
1348
1349    if cmd[0] == self.executable:
1350      cmd = ['python'] + cmd[1:]
1351    self.Print(*[shell_quoter(arg) for arg in cmd])
1352
1353  def PrintJSON(self, obj):
1354    self.Print(json.dumps(obj, indent=2, sort_keys=True))
1355
1356  def GNTargetName(self, target):
1357    return target
1358
1359  def Build(self, target):
1360    build_dir = self.ToSrcRelPath(self.args.path[0])
1361    ninja_cmd = ['ninja', '-C', build_dir]
1362    if self.args.jobs:
1363      ninja_cmd.extend(['-j', '%d' % self.args.jobs])
1364    ninja_cmd.append(target)
1365    ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False)
1366    return ret
1367
1368  def Run(self, cmd, env=None, force_verbose=True, buffer_output=True):
1369    # This function largely exists so it can be overridden for testing.
1370    if self.args.dryrun or self.args.verbose or force_verbose:
1371      self.PrintCmd(cmd, env)
1372    if self.args.dryrun:
1373      return 0, '', ''
1374
1375    ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output)
1376    if self.args.verbose or force_verbose:
1377      if ret:
1378        self.Print('  -> returned %d' % ret)
1379      if out:
1380        self.Print(out, end='')
1381      if err:
1382        self.Print(err, end='', file=sys.stderr)
1383    return ret, out, err
1384
1385  def Call(self, cmd, env=None, buffer_output=True):
1386    if buffer_output:
1387      p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1388                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1389                           env=env)
1390      out, err = p.communicate()
1391    else:
1392      p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1393                           env=env)
1394      p.wait()
1395      out = err = ''
1396    return p.returncode, out, err
1397
1398  def ExpandUser(self, path):
1399    # This function largely exists so it can be overridden for testing.
1400    return os.path.expanduser(path)
1401
1402  def Exists(self, path):
1403    # This function largely exists so it can be overridden for testing.
1404    return os.path.exists(path)
1405
1406  def Fetch(self, url):
1407    # This function largely exists so it can be overridden for testing.
1408    f = urllib2.urlopen(url)
1409    contents = f.read()
1410    f.close()
1411    return contents
1412
1413  def MaybeMakeDirectory(self, path):
1414    try:
1415      os.makedirs(path)
1416    except OSError, e:
1417      if e.errno != errno.EEXIST:
1418        raise
1419
1420  def PathJoin(self, *comps):
1421    # This function largely exists so it can be overriden for testing.
1422    return os.path.join(*comps)
1423
1424  def Print(self, *args, **kwargs):
1425    # This function largely exists so it can be overridden for testing.
1426    print(*args, **kwargs)
1427    if kwargs.get('stream', sys.stdout) == sys.stdout:
1428      sys.stdout.flush()
1429
1430  def ReadFile(self, path):
1431    # This function largely exists so it can be overriden for testing.
1432    with open(path) as fp:
1433      return fp.read()
1434
1435  def RelPath(self, path, start='.'):
1436    # This function largely exists so it can be overriden for testing.
1437    return os.path.relpath(path, start)
1438
1439  def RemoveFile(self, path):
1440    # This function largely exists so it can be overriden for testing.
1441    os.remove(path)
1442
1443  def RemoveDirectory(self, abs_path):
1444    if self.platform == 'win32':
1445      # In other places in chromium, we often have to retry this command
1446      # because we're worried about other processes still holding on to
1447      # file handles, but when MB is invoked, it will be early enough in the
1448      # build that their should be no other processes to interfere. We
1449      # can change this if need be.
1450      self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
1451    else:
1452      shutil.rmtree(abs_path, ignore_errors=True)
1453
1454  def TempFile(self, mode='w'):
1455    # This function largely exists so it can be overriden for testing.
1456    return tempfile.NamedTemporaryFile(mode=mode, delete=False)
1457
1458  def WriteFile(self, path, contents, force_verbose=False):
1459    # This function largely exists so it can be overriden for testing.
1460    if self.args.dryrun or self.args.verbose or force_verbose:
1461      self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
1462    with open(path, 'w') as fp:
1463      return fp.write(contents)
1464
1465
1466class MBErr(Exception):
1467  pass
1468
1469
1470# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful
1471# details of this next section, which handles escaping command lines
1472# so that they can be copied and pasted into a cmd window.
1473UNSAFE_FOR_SET = set('^<>&|')
1474UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%'))
1475ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"'))
1476
1477
1478def QuoteForSet(arg):
1479  if any(a in UNSAFE_FOR_SET for a in arg):
1480    arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg)
1481  return arg
1482
1483
1484def QuoteForCmd(arg):
1485  # First, escape the arg so that CommandLineToArgvW will parse it properly.
1486  # From //tools/gyp/pylib/gyp/msvs_emulation.py:23.
1487  if arg == '' or ' ' in arg or '"' in arg:
1488    quote_re = re.compile(r'(\\*)"')
1489    arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
1490
1491  # Then check to see if the arg contains any metacharacters other than
1492  # double quotes; if it does, quote everything (including the double
1493  # quotes) for safety.
1494  if any(a in UNSAFE_FOR_CMD for a in arg):
1495    arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
1496  return arg
1497
1498
1499if __name__ == '__main__':
1500  sys.exit(main(sys.argv[1:]))
1501