1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Copyright 2015 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""MB - the Meta-Build wrapper around GN.
8
9MB is a wrapper script for GN that can be used to generate build files
10for sets of canned configurations and analyze them.
11"""
12
13from __future__ import print_function
14
15import argparse
16import ast
17import errno
18import json
19import os
20import pipes
21import platform
22import pprint
23import re
24import shutil
25import sys
26import subprocess
27import tempfile
28import traceback
29import urllib2
30
31from collections import OrderedDict
32
33CHROMIUM_SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(
34    os.path.abspath(__file__))))
35sys.path = [os.path.join(CHROMIUM_SRC_DIR, 'build')] + sys.path
36
37import gn_helpers
38
39
40def main(args):
41  mbw = MetaBuildWrapper()
42  return mbw.Main(args)
43
44
45class MetaBuildWrapper(object):
46  def __init__(self):
47    self.chromium_src_dir = CHROMIUM_SRC_DIR
48    self.default_config = os.path.join(self.chromium_src_dir, 'infra', 'mb',
49                                       'mb_config.pyl')
50    self.default_isolate_map = os.path.join(self.chromium_src_dir, 'infra',
51                                            'mb', 'gn_isolate_map.pyl')
52    self.executable = sys.executable
53    self.platform = sys.platform
54    self.sep = os.sep
55    self.args = argparse.Namespace()
56    self.configs = {}
57    self.luci_tryservers = {}
58    self.masters = {}
59    self.mixins = {}
60
61  def Main(self, args):
62    self.ParseArgs(args)
63    try:
64      ret = self.args.func()
65      if ret:
66        self.DumpInputFiles()
67      return ret
68    except KeyboardInterrupt:
69      self.Print('interrupted, exiting')
70      return 130
71    except Exception:
72      self.DumpInputFiles()
73      s = traceback.format_exc()
74      for l in s.splitlines():
75        self.Print(l)
76      return 1
77
78  def ParseArgs(self, argv):
79    def AddCommonOptions(subp):
80      subp.add_argument('-b', '--builder',
81                        help='builder name to look up config from')
82      subp.add_argument('-m', '--master',
83                        help='master name to look up config from')
84      subp.add_argument('-c', '--config',
85                        help='configuration to analyze')
86      subp.add_argument('--phase',
87                        help='optional phase name (used when builders '
88                             'do multiple compiles with different '
89                             'arguments in a single build)')
90      subp.add_argument('-f', '--config-file', metavar='PATH',
91                        default=self.default_config,
92                        help='path to config file '
93                             '(default is %(default)s)')
94      subp.add_argument('-i', '--isolate-map-file', metavar='PATH',
95                        help='path to isolate map file '
96                             '(default is %(default)s)',
97                        default=[],
98                        action='append',
99                        dest='isolate_map_files')
100      subp.add_argument('-g', '--goma-dir',
101                        help='path to goma directory')
102      subp.add_argument('--android-version-code',
103                        help='Sets GN arg android_default_version_code')
104      subp.add_argument('--android-version-name',
105                        help='Sets GN arg android_default_version_name')
106      subp.add_argument('-n', '--dryrun', action='store_true',
107                        help='Do a dry run (i.e., do nothing, just print '
108                             'the commands that will run)')
109      subp.add_argument('-v', '--verbose', action='store_true',
110                        help='verbose logging')
111
112    parser = argparse.ArgumentParser(prog='mb')
113    subps = parser.add_subparsers()
114
115    subp = subps.add_parser('analyze',
116                            help='analyze whether changes to a set of files '
117                                 'will cause a set of binaries to be rebuilt.')
118    AddCommonOptions(subp)
119    subp.add_argument('path', nargs=1,
120                      help='path build was generated into.')
121    subp.add_argument('input_path', nargs=1,
122                      help='path to a file containing the input arguments '
123                           'as a JSON object.')
124    subp.add_argument('output_path', nargs=1,
125                      help='path to a file containing the output arguments '
126                           'as a JSON object.')
127    subp.set_defaults(func=self.CmdAnalyze)
128
129    subp = subps.add_parser('export',
130                            help='print out the expanded configuration for'
131                                 'each builder as a JSON object')
132    subp.add_argument('-f', '--config-file', metavar='PATH',
133                      default=self.default_config,
134                      help='path to config file (default is %(default)s)')
135    subp.add_argument('-g', '--goma-dir',
136                      help='path to goma directory')
137    subp.set_defaults(func=self.CmdExport)
138
139    subp = subps.add_parser('gen',
140                            help='generate a new set of build files')
141    AddCommonOptions(subp)
142    subp.add_argument('--swarming-targets-file',
143                      help='save runtime dependencies for targets listed '
144                           'in file.')
145    subp.add_argument('path', nargs=1,
146                      help='path to generate build into')
147    subp.set_defaults(func=self.CmdGen)
148
149    subp = subps.add_parser('isolate',
150                            help='generate the .isolate files for a given'
151                                 'binary')
152    AddCommonOptions(subp)
153    subp.add_argument('path', nargs=1,
154                      help='path build was generated into')
155    subp.add_argument('target', nargs=1,
156                      help='ninja target to generate the isolate for')
157    subp.set_defaults(func=self.CmdIsolate)
158
159    subp = subps.add_parser('lookup',
160                            help='look up the command for a given config or '
161                                 'builder')
162    AddCommonOptions(subp)
163    subp.set_defaults(func=self.CmdLookup)
164
165    subp = subps.add_parser(
166        'run',
167        help='build and run the isolated version of a '
168             'binary',
169        formatter_class=argparse.RawDescriptionHelpFormatter)
170    subp.description = (
171        'Build, isolate, and run the given binary with the command line\n'
172        'listed in the isolate. You may pass extra arguments after the\n'
173        'target; use "--" if the extra arguments need to include switches.\n'
174        '\n'
175        'Examples:\n'
176        '\n'
177        '  % tools/mb/mb.py run -m chromium.linux -b "Linux Builder" \\\n'
178        '    //out/Default content_browsertests\n'
179        '\n'
180        '  % tools/mb/mb.py run out/Default content_browsertests\n'
181        '\n'
182        '  % tools/mb/mb.py run out/Default content_browsertests -- \\\n'
183        '    --test-launcher-retry-limit=0'
184        '\n'
185    )
186    AddCommonOptions(subp)
187    subp.add_argument('-j', '--jobs', dest='jobs', type=int,
188                      help='Number of jobs to pass to ninja')
189    subp.add_argument('--no-build', dest='build', default=True,
190                      action='store_false',
191                      help='Do not build, just isolate and run')
192    subp.add_argument('path', nargs=1,
193                      help=('path to generate build into (or use).'
194                            ' This can be either a regular path or a '
195                            'GN-style source-relative path like '
196                            '//out/Default.'))
197    subp.add_argument('-s', '--swarmed', action='store_true',
198                      help='Run under swarming with the default dimensions')
199    subp.add_argument('-d', '--dimension', default=[], action='append', nargs=2,
200                      dest='dimensions', metavar='FOO bar',
201                      help='dimension to filter on')
202    subp.add_argument('--no-default-dimensions', action='store_false',
203                      dest='default_dimensions', default=True,
204                      help='Do not automatically add dimensions to the task')
205    subp.add_argument('target', nargs=1,
206                      help='ninja target to build and run')
207    subp.add_argument('extra_args', nargs='*',
208                      help=('extra args to pass to the isolate to run. Use '
209                            '"--" as the first arg if you need to pass '
210                            'switches'))
211    subp.set_defaults(func=self.CmdRun)
212
213    subp = subps.add_parser('validate',
214                            help='validate the config file')
215    subp.add_argument('-f', '--config-file', metavar='PATH',
216                      default=self.default_config,
217                      help='path to config file (default is %(default)s)')
218    subp.set_defaults(func=self.CmdValidate)
219
220    subp = subps.add_parser('gerrit-buildbucket-config',
221                            help='Print buildbucket.config for gerrit '
222                            '(see MB user guide)')
223    subp.add_argument('-f', '--config-file', metavar='PATH',
224                      default=self.default_config,
225                      help='path to config file (default is %(default)s)')
226    subp.set_defaults(func=self.CmdBuildbucket)
227
228    subp = subps.add_parser('help',
229                            help='Get help on a subcommand.')
230    subp.add_argument(nargs='?', action='store', dest='subcommand',
231                      help='The command to get help for.')
232    subp.set_defaults(func=self.CmdHelp)
233
234    self.args = parser.parse_args(argv)
235
236  def DumpInputFiles(self):
237
238    def DumpContentsOfFilePassedTo(arg_name, path):
239      if path and self.Exists(path):
240        self.Print("\n# To recreate the file passed to %s:" % arg_name)
241        self.Print("%% cat > %s <<EOF" % path)
242        contents = self.ReadFile(path)
243        self.Print(contents)
244        self.Print("EOF\n%\n")
245
246    if getattr(self.args, 'input_path', None):
247      DumpContentsOfFilePassedTo(
248          'argv[0] (input_path)', self.args.input_path[0])
249    if getattr(self.args, 'swarming_targets_file', None):
250      DumpContentsOfFilePassedTo(
251          '--swarming-targets-file', self.args.swarming_targets_file)
252
253  def CmdAnalyze(self):
254    vals = self.Lookup()
255    return self.RunGNAnalyze(vals)
256
257  def CmdExport(self):
258    self.ReadConfigFile()
259    obj = {}
260    for master, builders in self.masters.items():
261      obj[master] = {}
262      for builder in builders:
263        config = self.masters[master][builder]
264        if not config:
265          continue
266
267        if isinstance(config, dict):
268          args = {k: self.FlattenConfig(v)['gn_args']
269                  for k, v in config.items()}
270        elif config.startswith('//'):
271          args = config
272        else:
273          args = self.FlattenConfig(config)['gn_args']
274          if 'error' in args:
275            continue
276
277        obj[master][builder] = args
278
279    # Dump object and trim trailing whitespace.
280    s = '\n'.join(l.rstrip() for l in
281                  json.dumps(obj, sort_keys=True, indent=2).splitlines())
282    self.Print(s)
283    return 0
284
285  def CmdGen(self):
286    vals = self.Lookup()
287    return self.RunGNGen(vals)
288
289  def CmdHelp(self):
290    if self.args.subcommand:
291      self.ParseArgs([self.args.subcommand, '--help'])
292    else:
293      self.ParseArgs(['--help'])
294
295  def CmdIsolate(self):
296    vals = self.GetConfig()
297    if not vals:
298      return 1
299    return self.RunGNIsolate()
300
301  def CmdLookup(self):
302    vals = self.Lookup()
303    cmd = self.GNCmd('gen', '_path_')
304    gn_args = self.GNArgs(vals)
305    self.Print('\nWriting """\\\n%s""" to _path_/args.gn.\n' % gn_args)
306    env = None
307
308    self.PrintCmd(cmd, env)
309    return 0
310
311  def CmdRun(self):
312    vals = self.GetConfig()
313    if not vals:
314      return 1
315
316    build_dir = self.args.path[0]
317    target = self.args.target[0]
318
319    if self.args.build:
320      ret = self.Build(target)
321      if ret:
322        return ret
323    ret = self.RunGNIsolate()
324    if ret:
325      return ret
326
327    if self.args.swarmed:
328      return self._RunUnderSwarming(build_dir, target)
329    else:
330      return self._RunLocallyIsolated(build_dir, target)
331
332  def _RunUnderSwarming(self, build_dir, target):
333    # TODO(dpranke): Look up the information for the target in
334    # the //testing/buildbot.json file, if possible, so that we
335    # can determine the isolate target, command line, and additional
336    # swarming parameters, if possible.
337    #
338    # TODO(dpranke): Also, add support for sharding and merging results.
339    dimensions = []
340    for k, v in self._DefaultDimensions() + self.args.dimensions:
341      dimensions += ['-d', k, v]
342
343    cmd = [
344        self.executable,
345        self.PathJoin('tools', 'swarming_client', 'isolate.py'),
346        'archive',
347        '-s',
348        self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
349        '-I', 'isolateserver.appspot.com',
350      ]
351    ret, out, _ = self.Run(cmd, force_verbose=False)
352    if ret:
353      return ret
354
355    isolated_hash = out.splitlines()[0].split()[0]
356    cmd = [
357        self.executable,
358        self.PathJoin('tools', 'swarming_client', 'swarming.py'),
359          'run',
360          '-s', isolated_hash,
361          '-I', 'isolateserver.appspot.com',
362          '-S', 'chromium-swarm.appspot.com',
363      ] + dimensions
364    if self.args.extra_args:
365      cmd += ['--'] + self.args.extra_args
366    ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
367    return ret
368
369  def _RunLocallyIsolated(self, build_dir, target):
370    cmd = [
371        self.executable,
372        self.PathJoin('tools', 'swarming_client', 'isolate.py'),
373        'run',
374        '-s',
375        self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
376      ]
377    if self.args.extra_args:
378      cmd += ['--'] + self.args.extra_args
379    ret, _, _ = self.Run(cmd, force_verbose=True, buffer_output=False)
380    return ret
381
382  def _DefaultDimensions(self):
383    if not self.args.default_dimensions:
384      return []
385
386    # This code is naive and just picks reasonable defaults per platform.
387    if self.platform == 'darwin':
388      os_dim = ('os', 'Mac-10.12')
389    elif self.platform.startswith('linux'):
390      os_dim = ('os', 'Ubuntu-14.04')
391    elif self.platform == 'win32':
392      os_dim = ('os', 'Windows-10')
393    else:
394      raise MBErr('unrecognized platform string "%s"' % self.platform)
395
396    return [('pool', 'Chrome'),
397            ('cpu', 'x86-64'),
398            os_dim]
399
400  def CmdBuildbucket(self):
401    self.ReadConfigFile()
402
403    self.Print('# This file was generated using '
404               '"tools/mb/mb.py gerrit-buildbucket-config".')
405
406    for luci_tryserver in sorted(self.luci_tryservers):
407      self.Print('[bucket "luci.%s"]' % luci_tryserver)
408      for bot in sorted(self.luci_tryservers[luci_tryserver]):
409        self.Print('\tbuilder = %s' % bot)
410
411    for master in sorted(self.masters):
412      if master.startswith('tryserver.'):
413        self.Print('[bucket "master.%s"]' % master)
414        for bot in sorted(self.masters[master]):
415          self.Print('\tbuilder = %s' % bot)
416
417    return 0
418
419  def CmdValidate(self, print_ok=True):
420    errs = []
421
422    # Read the file to make sure it parses.
423    self.ReadConfigFile()
424
425    # Build a list of all of the configs referenced by builders.
426    all_configs = {}
427    for master in self.masters:
428      for config in self.masters[master].values():
429        if isinstance(config, dict):
430          for c in config.values():
431            all_configs[c] = master
432        else:
433          all_configs[config] = master
434
435    # Check that every referenced args file or config actually exists.
436    for config, loc in all_configs.items():
437      if config.startswith('//'):
438        if not self.Exists(self.ToAbsPath(config)):
439          errs.append('Unknown args file "%s" referenced from "%s".' %
440                      (config, loc))
441      elif not config in self.configs:
442        errs.append('Unknown config "%s" referenced from "%s".' %
443                    (config, loc))
444
445    # Check that every actual config is actually referenced.
446    for config in self.configs:
447      if not config in all_configs:
448        errs.append('Unused config "%s".' % config)
449
450    # Figure out the whole list of mixins, and check that every mixin
451    # listed by a config or another mixin actually exists.
452    referenced_mixins = set()
453    for config, mixins in self.configs.items():
454      for mixin in mixins:
455        if not mixin in self.mixins:
456          errs.append('Unknown mixin "%s" referenced by config "%s".' %
457                      (mixin, config))
458        referenced_mixins.add(mixin)
459
460    for mixin in self.mixins:
461      for sub_mixin in self.mixins[mixin].get('mixins', []):
462        if not sub_mixin in self.mixins:
463          errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
464                      (sub_mixin, mixin))
465        referenced_mixins.add(sub_mixin)
466
467    # Check that every mixin defined is actually referenced somewhere.
468    for mixin in self.mixins:
469      if not mixin in referenced_mixins:
470        errs.append('Unreferenced mixin "%s".' % mixin)
471
472    if errs:
473      raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
474                    '\n  ' + '\n  '.join(errs))
475
476    if print_ok:
477      self.Print('mb config file %s looks ok.' % self.args.config_file)
478    return 0
479
480  def GetConfig(self):
481    build_dir = self.args.path[0]
482
483    vals = self.DefaultVals()
484    if self.args.builder or self.args.master or self.args.config:
485      vals = self.Lookup()
486      # Re-run gn gen in order to ensure the config is consistent with the
487      # build dir.
488      self.RunGNGen(vals)
489      return vals
490
491    toolchain_path = self.PathJoin(self.ToAbsPath(build_dir),
492                                   'toolchain.ninja')
493    if not self.Exists(toolchain_path):
494      self.Print('Must either specify a path to an existing GN build dir '
495                 'or pass in a -m/-b pair or a -c flag to specify the '
496                 'configuration')
497      return {}
498
499    vals['gn_args'] = self.GNArgsFromDir(build_dir)
500    return vals
501
502  def GNArgsFromDir(self, build_dir):
503    args_contents = ""
504    gn_args_path = self.PathJoin(self.ToAbsPath(build_dir), 'args.gn')
505    if self.Exists(gn_args_path):
506      args_contents = self.ReadFile(gn_args_path)
507    gn_args = []
508    for l in args_contents.splitlines():
509      fields = l.split(' ')
510      name = fields[0]
511      val = ' '.join(fields[2:])
512      gn_args.append('%s=%s' % (name, val))
513
514    return ' '.join(gn_args)
515
516  def Lookup(self):
517    vals = self.ReadIOSBotConfig()
518    if not vals:
519      self.ReadConfigFile()
520      config = self.ConfigFromArgs()
521      if config.startswith('//'):
522        if not self.Exists(self.ToAbsPath(config)):
523          raise MBErr('args file "%s" not found' % config)
524        vals = self.DefaultVals()
525        vals['args_file'] = config
526      else:
527        if not config in self.configs:
528          raise MBErr('Config "%s" not found in %s' %
529                      (config, self.args.config_file))
530        vals = self.FlattenConfig(config)
531    return vals
532
533  def ReadIOSBotConfig(self):
534    if not self.args.master or not self.args.builder:
535      return {}
536    path = self.PathJoin(self.chromium_src_dir, 'ios', 'build', 'bots',
537                         self.args.master, self.args.builder + '.json')
538    if not self.Exists(path):
539      return {}
540
541    contents = json.loads(self.ReadFile(path))
542    gn_args = ' '.join(contents.get('gn_args', []))
543
544    vals = self.DefaultVals()
545    vals['gn_args'] = gn_args
546    return vals
547
548  def ReadConfigFile(self):
549    if not self.Exists(self.args.config_file):
550      raise MBErr('config file not found at %s' % self.args.config_file)
551
552    try:
553      contents = ast.literal_eval(self.ReadFile(self.args.config_file))
554    except SyntaxError as e:
555      raise MBErr('Failed to parse config file "%s": %s' %
556                 (self.args.config_file, e))
557
558    self.configs = contents['configs']
559    self.luci_tryservers = contents.get('luci_tryservers', {})
560    self.masters = contents['masters']
561    self.mixins = contents['mixins']
562
563  def ReadIsolateMap(self):
564    if not self.args.isolate_map_files:
565      self.args.isolate_map_files = [self.default_isolate_map]
566
567    for f in self.args.isolate_map_files:
568      if not self.Exists(f):
569        raise MBErr('isolate map file not found at %s' % f)
570    isolate_maps = {}
571    for isolate_map in self.args.isolate_map_files:
572      try:
573        isolate_map = ast.literal_eval(self.ReadFile(isolate_map))
574        duplicates = set(isolate_map).intersection(isolate_maps)
575        if duplicates:
576          raise MBErr(
577              'Duplicate targets in isolate map files: %s.' %
578              ', '.join(duplicates))
579        isolate_maps.update(isolate_map)
580      except SyntaxError as e:
581        raise MBErr(
582            'Failed to parse isolate map file "%s": %s' % (isolate_map, e))
583    return isolate_maps
584
585  def ConfigFromArgs(self):
586    if self.args.config:
587      if self.args.master or self.args.builder:
588        raise MBErr('Can not specific both -c/--config and -m/--master or '
589                    '-b/--builder')
590
591      return self.args.config
592
593    if not self.args.master or not self.args.builder:
594      raise MBErr('Must specify either -c/--config or '
595                  '(-m/--master and -b/--builder)')
596
597    if not self.args.master in self.masters:
598      raise MBErr('Master name "%s" not found in "%s"' %
599                  (self.args.master, self.args.config_file))
600
601    if not self.args.builder in self.masters[self.args.master]:
602      raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
603                  (self.args.builder, self.args.master, self.args.config_file))
604
605    config = self.masters[self.args.master][self.args.builder]
606    if isinstance(config, dict):
607      if self.args.phase is None:
608        raise MBErr('Must specify a build --phase for %s on %s' %
609                    (self.args.builder, self.args.master))
610      phase = str(self.args.phase)
611      if phase not in config:
612        raise MBErr('Phase %s doesn\'t exist for %s on %s' %
613                    (phase, self.args.builder, self.args.master))
614      return config[phase]
615
616    if self.args.phase is not None:
617      raise MBErr('Must not specify a build --phase for %s on %s' %
618                  (self.args.builder, self.args.master))
619    return config
620
621  def FlattenConfig(self, config):
622    mixins = self.configs[config]
623    vals = self.DefaultVals()
624
625    visited = []
626    self.FlattenMixins(mixins, vals, visited)
627    return vals
628
629  def DefaultVals(self):
630    return {
631      'args_file': '',
632      'cros_passthrough': False,
633      'gn_args': '',
634    }
635
636  def FlattenMixins(self, mixins, vals, visited):
637    for m in mixins:
638      if m not in self.mixins:
639        raise MBErr('Unknown mixin "%s"' % m)
640
641      visited.append(m)
642
643      mixin_vals = self.mixins[m]
644
645      if 'cros_passthrough' in mixin_vals:
646        vals['cros_passthrough'] = mixin_vals['cros_passthrough']
647      if 'args_file' in mixin_vals:
648        if vals['args_file']:
649            raise MBErr('args_file specified multiple times in mixins '
650                        'for %s on %s' % (self.args.builder, self.args.master))
651        vals['args_file'] = mixin_vals['args_file']
652      if 'gn_args' in mixin_vals:
653        if vals['gn_args']:
654          vals['gn_args'] += ' ' + mixin_vals['gn_args']
655        else:
656          vals['gn_args'] = mixin_vals['gn_args']
657
658      if 'mixins' in mixin_vals:
659        self.FlattenMixins(mixin_vals['mixins'], vals, visited)
660    return vals
661
662  def RunGNGen(self, vals, compute_grit_inputs_for_analyze=False):
663    build_dir = self.args.path[0]
664
665    cmd = self.GNCmd('gen', build_dir, '--check')
666    gn_args = self.GNArgs(vals)
667    if compute_grit_inputs_for_analyze:
668      gn_args += ' compute_grit_inputs_for_analyze=true'
669
670    # Since GN hasn't run yet, the build directory may not even exist.
671    self.MaybeMakeDirectory(self.ToAbsPath(build_dir))
672
673    gn_args_path = self.ToAbsPath(build_dir, 'args.gn')
674    self.WriteFile(gn_args_path, gn_args, force_verbose=True)
675
676    swarming_targets = []
677    if getattr(self.args, 'swarming_targets_file', None):
678      # We need GN to generate the list of runtime dependencies for
679      # the compile targets listed (one per line) in the file so
680      # we can run them via swarming. We use gn_isolate_map.pyl to convert
681      # the compile targets to the matching GN labels.
682      path = self.args.swarming_targets_file
683      if not self.Exists(path):
684        self.WriteFailureAndRaise('"%s" does not exist' % path,
685                                  output_path=None)
686      contents = self.ReadFile(path)
687      swarming_targets = set(contents.splitlines())
688
689      isolate_map = self.ReadIsolateMap()
690      err, labels = self.MapTargetsToLabels(isolate_map, swarming_targets)
691      if err:
692          raise MBErr(err)
693
694      gn_runtime_deps_path = self.ToAbsPath(build_dir, 'runtime_deps')
695      self.WriteFile(gn_runtime_deps_path, '\n'.join(labels) + '\n')
696      cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
697
698    ret, _, _ = self.Run(cmd)
699    if ret:
700        # If `gn gen` failed, we should exit early rather than trying to
701        # generate isolates. Run() will have already logged any error output.
702        self.Print('GN gen failed: %d' % ret)
703        return ret
704
705    android = 'target_os="android"' in vals['gn_args']
706    fuchsia = 'target_os="fuchsia"' in vals['gn_args']
707    for target in swarming_targets:
708      if android:
709        # Android targets may be either android_apk or executable. The former
710        # will result in runtime_deps associated with the stamp file, while the
711        # latter will result in runtime_deps associated with the executable.
712        label = isolate_map[target]['label']
713        runtime_deps_targets = [
714            target + '.runtime_deps',
715            'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
716      elif fuchsia:
717        # Only emit a runtime deps file for the group() target on Fuchsia.
718        label = isolate_map[target]['label']
719        runtime_deps_targets = [
720          'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
721      elif (isolate_map[target]['type'] == 'script' or
722            isolate_map[target].get('label_type') == 'group'):
723        # For script targets, the build target is usually a group,
724        # for which gn generates the runtime_deps next to the stamp file
725        # for the label, which lives under the obj/ directory, but it may
726        # also be an executable.
727        label = isolate_map[target]['label']
728        runtime_deps_targets = [
729            'obj/%s.stamp.runtime_deps' % label.replace(':', '/')]
730        if self.platform == 'win32':
731          runtime_deps_targets += [ target + '.exe.runtime_deps' ]
732        else:
733          runtime_deps_targets += [ target + '.runtime_deps' ]
734      elif self.platform == 'win32':
735        runtime_deps_targets = [target + '.exe.runtime_deps']
736      else:
737        runtime_deps_targets = [target + '.runtime_deps']
738
739      for r in runtime_deps_targets:
740        runtime_deps_path = self.ToAbsPath(build_dir, r)
741        if self.Exists(runtime_deps_path):
742          break
743      else:
744        raise MBErr('did not generate any of %s' %
745                    ', '.join(runtime_deps_targets))
746
747      runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
748
749      self.WriteIsolateFiles(build_dir, target, runtime_deps)
750
751    return 0
752
753  def RunGNIsolate(self):
754    target = self.args.target[0]
755    isolate_map = self.ReadIsolateMap()
756    err, labels = self.MapTargetsToLabels(isolate_map, [target])
757    if err:
758      raise MBErr(err)
759    label = labels[0]
760
761    build_dir = self.args.path[0]
762
763    cmd = self.GNCmd('desc', build_dir, label, 'runtime_deps')
764    ret, out, _ = self.Call(cmd)
765    if ret:
766      if out:
767        self.Print(out)
768      return ret
769
770    runtime_deps = out.splitlines()
771
772    self.WriteIsolateFiles(build_dir, target, runtime_deps)
773
774    ret, _, _ = self.Run([
775        self.executable,
776        self.PathJoin('tools', 'swarming_client', 'isolate.py'),
777        'check',
778        '-i',
779        self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
780        '-s',
781        self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target))],
782        buffer_output=False)
783
784    return ret
785
786  def WriteIsolateFiles(self, build_dir, target, runtime_deps):
787    isolate_path = self.ToAbsPath(build_dir, target + '.isolate')
788    self.WriteFile(isolate_path,
789      pprint.pformat({
790        'variables': {
791          'files': sorted(runtime_deps),
792        }
793      }) + '\n')
794
795    self.WriteJSON(
796      {
797        'args': [
798          '--isolated',
799          self.ToSrcRelPath('%s/%s.isolated' % (build_dir, target)),
800          '--isolate',
801          self.ToSrcRelPath('%s/%s.isolate' % (build_dir, target)),
802        ],
803        'dir': self.chromium_src_dir,
804        'version': 1,
805      },
806      isolate_path + 'd.gen.json',
807    )
808
809  def MapTargetsToLabels(self, isolate_map, targets):
810    labels = []
811    err = ''
812
813    for target in targets:
814      if target == 'all':
815        labels.append(target)
816      elif target.startswith('//'):
817        labels.append(target)
818      else:
819        if target in isolate_map:
820          if isolate_map[target]['type'] == 'unknown':
821            err += ('test target "%s" type is unknown\n' % target)
822          else:
823            labels.append(isolate_map[target]['label'])
824        else:
825          err += ('target "%s" not found in '
826                  '//infra/mb/gn_isolate_map.pyl\n' % target)
827
828    return err, labels
829
830  def GNCmd(self, subcommand, path, *args):
831    if self.platform == 'linux2':
832      subdir, exe = 'linux64', 'gn'
833    elif self.platform == 'darwin':
834      subdir, exe = 'mac', 'gn'
835    else:
836      subdir, exe = 'win', 'gn.exe'
837
838    arch = platform.machine()
839    if (arch.startswith('s390') or arch.startswith('ppc') or
840        self.platform.startswith('aix')):
841      # use gn in PATH
842      gn_path = 'gn'
843    else:
844      gn_path = self.PathJoin(self.chromium_src_dir, 'buildtools', subdir, exe)
845    return [gn_path, subcommand, path] + list(args)
846
847
848  def GNArgs(self, vals):
849    if vals['cros_passthrough']:
850      if not 'GN_ARGS' in os.environ:
851        raise MBErr('MB is expecting GN_ARGS to be in the environment')
852      gn_args = os.environ['GN_ARGS']
853      if not re.search('target_os.*=.*"chromeos"', gn_args):
854        raise MBErr('GN_ARGS is missing target_os = "chromeos": (GN_ARGS=%s)' %
855                    gn_args)
856    else:
857      gn_args = vals['gn_args']
858
859    if self.args.goma_dir:
860      gn_args += ' goma_dir="%s"' % self.args.goma_dir
861
862    android_version_code = self.args.android_version_code
863    if android_version_code:
864      gn_args += ' android_default_version_code="%s"' % android_version_code
865
866    android_version_name = self.args.android_version_name
867    if android_version_name:
868      gn_args += ' android_default_version_name="%s"' % android_version_name
869
870    # Canonicalize the arg string into a sorted, newline-separated list
871    # of key-value pairs, and de-dup the keys if need be so that only
872    # the last instance of each arg is listed.
873    gn_args = gn_helpers.ToGNString(gn_helpers.FromGNArgs(gn_args))
874
875    args_file = vals.get('args_file', None)
876    if args_file:
877      gn_args = ('import("%s")\n' % vals['args_file']) + gn_args
878    return gn_args
879
880  def ToAbsPath(self, build_path, *comps):
881    return self.PathJoin(self.chromium_src_dir,
882                         self.ToSrcRelPath(build_path),
883                         *comps)
884
885  def ToSrcRelPath(self, path):
886    """Returns a relative path from the top of the repo."""
887    if path.startswith('//'):
888      return path[2:].replace('/', self.sep)
889    return self.RelPath(path, self.chromium_src_dir)
890
891  def RunGNAnalyze(self, vals):
892    # Analyze runs before 'gn gen' now, so we need to run gn gen
893    # in order to ensure that we have a build directory.
894    ret = self.RunGNGen(vals, compute_grit_inputs_for_analyze=True)
895    if ret:
896      return ret
897
898    build_path = self.args.path[0]
899    input_path = self.args.input_path[0]
900    gn_input_path = input_path + '.gn'
901    output_path = self.args.output_path[0]
902    gn_output_path = output_path + '.gn'
903
904    inp = self.ReadInputJSON(['files', 'test_targets',
905                              'additional_compile_targets'])
906    if self.args.verbose:
907      self.Print()
908      self.Print('analyze input:')
909      self.PrintJSON(inp)
910      self.Print()
911
912
913    # This shouldn't normally happen, but could due to unusual race conditions,
914    # like a try job that gets scheduled before a patch lands but runs after
915    # the patch has landed.
916    if not inp['files']:
917      self.Print('Warning: No files modified in patch, bailing out early.')
918      self.WriteJSON({
919            'status': 'No dependency',
920            'compile_targets': [],
921            'test_targets': [],
922          }, output_path)
923      return 0
924
925    gn_inp = {}
926    gn_inp['files'] = ['//' + f for f in inp['files'] if not f.startswith('//')]
927
928    isolate_map = self.ReadIsolateMap()
929    err, gn_inp['additional_compile_targets'] = self.MapTargetsToLabels(
930        isolate_map, inp['additional_compile_targets'])
931    if err:
932      raise MBErr(err)
933
934    err, gn_inp['test_targets'] = self.MapTargetsToLabels(
935        isolate_map, inp['test_targets'])
936    if err:
937      raise MBErr(err)
938    labels_to_targets = {}
939    for i, label in enumerate(gn_inp['test_targets']):
940      labels_to_targets[label] = inp['test_targets'][i]
941
942    try:
943      self.WriteJSON(gn_inp, gn_input_path)
944      cmd = self.GNCmd('analyze', build_path, gn_input_path, gn_output_path)
945      ret, _, _ = self.Run(cmd, force_verbose=True)
946      if ret:
947        return ret
948
949      gn_outp_str = self.ReadFile(gn_output_path)
950      try:
951        gn_outp = json.loads(gn_outp_str)
952      except Exception as e:
953        self.Print("Failed to parse the JSON string GN returned: %s\n%s"
954                   % (repr(gn_outp_str), str(e)))
955        raise
956
957      outp = {}
958      if 'status' in gn_outp:
959        outp['status'] = gn_outp['status']
960      if 'error' in gn_outp:
961        outp['error'] = gn_outp['error']
962      if 'invalid_targets' in gn_outp:
963        outp['invalid_targets'] = gn_outp['invalid_targets']
964      if 'compile_targets' in gn_outp:
965        all_input_compile_targets = sorted(
966            set(inp['test_targets'] + inp['additional_compile_targets']))
967
968        # If we're building 'all', we can throw away the rest of the targets
969        # since they're redundant.
970        if 'all' in gn_outp['compile_targets']:
971          outp['compile_targets'] = ['all']
972        else:
973          outp['compile_targets'] = gn_outp['compile_targets']
974
975        # crbug.com/736215: When GN returns targets back, for targets in
976        # the default toolchain, GN will have generated a phony ninja
977        # target matching the label, and so we can safely (and easily)
978        # transform any GN label into the matching ninja target. For
979        # targets in other toolchains, though, GN doesn't generate the
980        # phony targets, and we don't know how to turn the labels into
981        # compile targets. In this case, we also conservatively give up
982        # and build everything. Probably the right thing to do here is
983        # to have GN return the compile targets directly.
984        if any("(" in target for target in outp['compile_targets']):
985          self.Print('WARNING: targets with non-default toolchains were '
986                     'found, building everything instead.')
987          outp['compile_targets'] = all_input_compile_targets
988        else:
989          outp['compile_targets'] = [
990              label.replace('//', '') for label in outp['compile_targets']]
991
992        # Windows has a maximum command line length of 8k; even Linux
993        # maxes out at 128k; if analyze returns a *really long* list of
994        # targets, we just give up and conservatively build everything instead.
995        # Probably the right thing here is for ninja to support response
996        # files as input on the command line
997        # (see https://github.com/ninja-build/ninja/issues/1355).
998        if len(' '.join(outp['compile_targets'])) > 7*1024:
999          self.Print('WARNING: Too many compile targets were affected.')
1000          self.Print('WARNING: Building everything instead to avoid '
1001                     'command-line length issues.')
1002          outp['compile_targets'] = all_input_compile_targets
1003
1004
1005      if 'test_targets' in gn_outp:
1006        outp['test_targets'] = [
1007          labels_to_targets[label] for label in gn_outp['test_targets']]
1008
1009      if self.args.verbose:
1010        self.Print()
1011        self.Print('analyze output:')
1012        self.PrintJSON(outp)
1013        self.Print()
1014
1015      self.WriteJSON(outp, output_path)
1016
1017    finally:
1018      if self.Exists(gn_input_path):
1019        self.RemoveFile(gn_input_path)
1020      if self.Exists(gn_output_path):
1021        self.RemoveFile(gn_output_path)
1022
1023    return 0
1024
1025  def ReadInputJSON(self, required_keys):
1026    path = self.args.input_path[0]
1027    output_path = self.args.output_path[0]
1028    if not self.Exists(path):
1029      self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
1030
1031    try:
1032      inp = json.loads(self.ReadFile(path))
1033    except Exception as e:
1034      self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
1035                                (path, e), output_path)
1036
1037    for k in required_keys:
1038      if not k in inp:
1039        self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
1040                                  output_path)
1041
1042    return inp
1043
1044  def WriteFailureAndRaise(self, msg, output_path):
1045    if output_path:
1046      self.WriteJSON({'error': msg}, output_path, force_verbose=True)
1047    raise MBErr(msg)
1048
1049  def WriteJSON(self, obj, path, force_verbose=False):
1050    try:
1051      self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n',
1052                     force_verbose=force_verbose)
1053    except Exception as e:
1054      raise MBErr('Error %s writing to the output path "%s"' %
1055                 (e, path))
1056
1057  def CheckCompile(self, master, builder):
1058    url_template = self.args.url_template + '/{builder}/builds/_all?as_text=1'
1059    url = urllib2.quote(url_template.format(master=master, builder=builder),
1060                        safe=':/()?=')
1061    try:
1062      builds = json.loads(self.Fetch(url))
1063    except Exception as e:
1064      return str(e)
1065    successes = sorted(
1066        [int(x) for x in builds.keys() if "text" in builds[x] and
1067          cmp(builds[x]["text"][:2], ["build", "successful"]) == 0],
1068        reverse=True)
1069    if not successes:
1070      return "no successful builds"
1071    build = builds[str(successes[0])]
1072    step_names = set([step["name"] for step in build["steps"]])
1073    compile_indicators = set(["compile", "compile (with patch)", "analyze"])
1074    if compile_indicators & step_names:
1075      return "compiles"
1076    return "does not compile"
1077
1078  def PrintCmd(self, cmd, env):
1079    if self.platform == 'win32':
1080      env_prefix = 'set '
1081      env_quoter = QuoteForSet
1082      shell_quoter = QuoteForCmd
1083    else:
1084      env_prefix = ''
1085      env_quoter = pipes.quote
1086      shell_quoter = pipes.quote
1087
1088    def print_env(var):
1089      if env and var in env:
1090        self.Print('%s%s=%s' % (env_prefix, var, env_quoter(env[var])))
1091
1092    print_env('LLVM_FORCE_HEAD_REVISION')
1093
1094    if cmd[0] == self.executable:
1095      cmd = ['python'] + cmd[1:]
1096    self.Print(*[shell_quoter(arg) for arg in cmd])
1097
1098  def PrintJSON(self, obj):
1099    self.Print(json.dumps(obj, indent=2, sort_keys=True))
1100
1101  def Build(self, target):
1102    build_dir = self.ToSrcRelPath(self.args.path[0])
1103    ninja_cmd = ['ninja', '-C', build_dir]
1104    if self.args.jobs:
1105      ninja_cmd.extend(['-j', '%d' % self.args.jobs])
1106    ninja_cmd.append(target)
1107    ret, _, _ = self.Run(ninja_cmd, force_verbose=False, buffer_output=False)
1108    return ret
1109
1110  def Run(self, cmd, env=None, force_verbose=True, buffer_output=True):
1111    # This function largely exists so it can be overridden for testing.
1112    if self.args.dryrun or self.args.verbose or force_verbose:
1113      self.PrintCmd(cmd, env)
1114    if self.args.dryrun:
1115      return 0, '', ''
1116
1117    ret, out, err = self.Call(cmd, env=env, buffer_output=buffer_output)
1118    if self.args.verbose or force_verbose:
1119      if ret:
1120        self.Print('  -> returned %d' % ret)
1121      if out:
1122        self.Print(out, end='')
1123      if err:
1124        self.Print(err, end='', file=sys.stderr)
1125    return ret, out, err
1126
1127  def Call(self, cmd, env=None, buffer_output=True):
1128    if buffer_output:
1129      p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1130                           stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1131                           env=env)
1132      out, err = p.communicate()
1133    else:
1134      p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
1135                           env=env)
1136      p.wait()
1137      out = err = ''
1138    return p.returncode, out, err
1139
1140  def ExpandUser(self, path):
1141    # This function largely exists so it can be overridden for testing.
1142    return os.path.expanduser(path)
1143
1144  def Exists(self, path):
1145    # This function largely exists so it can be overridden for testing.
1146    return os.path.exists(path)
1147
1148  def Fetch(self, url):
1149    # This function largely exists so it can be overridden for testing.
1150    f = urllib2.urlopen(url)
1151    contents = f.read()
1152    f.close()
1153    return contents
1154
1155  def MaybeMakeDirectory(self, path):
1156    try:
1157      os.makedirs(path)
1158    except OSError, e:
1159      if e.errno != errno.EEXIST:
1160        raise
1161
1162  def PathJoin(self, *comps):
1163    # This function largely exists so it can be overriden for testing.
1164    return os.path.join(*comps)
1165
1166  def Print(self, *args, **kwargs):
1167    # This function largely exists so it can be overridden for testing.
1168    print(*args, **kwargs)
1169    if kwargs.get('stream', sys.stdout) == sys.stdout:
1170      sys.stdout.flush()
1171
1172  def ReadFile(self, path):
1173    # This function largely exists so it can be overriden for testing.
1174    with open(path) as fp:
1175      return fp.read()
1176
1177  def RelPath(self, path, start='.'):
1178    # This function largely exists so it can be overriden for testing.
1179    return os.path.relpath(path, start)
1180
1181  def RemoveFile(self, path):
1182    # This function largely exists so it can be overriden for testing.
1183    os.remove(path)
1184
1185  def RemoveDirectory(self, abs_path):
1186    if self.platform == 'win32':
1187      # In other places in chromium, we often have to retry this command
1188      # because we're worried about other processes still holding on to
1189      # file handles, but when MB is invoked, it will be early enough in the
1190      # build that their should be no other processes to interfere. We
1191      # can change this if need be.
1192      self.Run(['cmd.exe', '/c', 'rmdir', '/q', '/s', abs_path])
1193    else:
1194      shutil.rmtree(abs_path, ignore_errors=True)
1195
1196  def TempFile(self, mode='w'):
1197    # This function largely exists so it can be overriden for testing.
1198    return tempfile.NamedTemporaryFile(mode=mode, delete=False)
1199
1200  def WriteFile(self, path, contents, force_verbose=False):
1201    # This function largely exists so it can be overriden for testing.
1202    if self.args.dryrun or self.args.verbose or force_verbose:
1203      self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
1204    with open(path, 'w') as fp:
1205      return fp.write(contents)
1206
1207
1208class MBErr(Exception):
1209  pass
1210
1211
1212# See http://goo.gl/l5NPDW and http://goo.gl/4Diozm for the painful
1213# details of this next section, which handles escaping command lines
1214# so that they can be copied and pasted into a cmd window.
1215UNSAFE_FOR_SET = set('^<>&|')
1216UNSAFE_FOR_CMD = UNSAFE_FOR_SET.union(set('()%'))
1217ALL_META_CHARS = UNSAFE_FOR_CMD.union(set('"'))
1218
1219
1220def QuoteForSet(arg):
1221  if any(a in UNSAFE_FOR_SET for a in arg):
1222    arg = ''.join('^' + a if a in UNSAFE_FOR_SET else a for a in arg)
1223  return arg
1224
1225
1226def QuoteForCmd(arg):
1227  # First, escape the arg so that CommandLineToArgvW will parse it properly.
1228  if arg == '' or ' ' in arg or '"' in arg:
1229    quote_re = re.compile(r'(\\*)"')
1230    arg = '"%s"' % (quote_re.sub(lambda mo: 2 * mo.group(1) + '\\"', arg))
1231
1232  # Then check to see if the arg contains any metacharacters other than
1233  # double quotes; if it does, quote everything (including the double
1234  # quotes) for safety.
1235  if any(a in UNSAFE_FOR_CMD for a in arg):
1236    arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
1237  return arg
1238
1239
1240if __name__ == '__main__':
1241  sys.exit(main(sys.argv[1:]))
1242