• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1#!/usr/bin/env python
2# Copyright 2017 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5"""\
6Convenience wrapper for compiling V8 with gn/ninja and running tests.
7Sets up build output directories if they don't exist.
8Produces simulator builds for non-Intel target architectures.
9Uses Goma by default if it is detected (at output directory setup time).
10Expects to be run from the root of a V8 checkout.
11
12Usage:
13    gm.py [<arch>].[<mode>].[<target>] [testname...]
14
15All arguments are optional. Most combinations should work, e.g.:
16    gm.py ia32.debug x64.release d8
17    gm.py x64 mjsunit/foo cctest/test-bar/*
18"""
19# See HELP below for additional documentation.
20
21from __future__ import print_function
22import errno
23import os
24import pty
25import re
26import subprocess
27import sys
28
29BUILD_TARGETS_TEST = ["d8", "cctest", "unittests"]
30BUILD_TARGETS_ALL = ["all"]
31
32# All arches that this script understands.
33ARCHES = ["ia32", "x64", "arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
34          "s390", "s390x"]
35# Arches that get built/run when you don't specify any.
36DEFAULT_ARCHES = ["ia32", "x64", "arm", "arm64"]
37# Modes that this script understands.
38MODES = ["release", "debug", "optdebug"]
39# Modes that get built/run when you don't specify any.
40DEFAULT_MODES = ["release", "debug"]
41# Build targets that can be manually specified.
42TARGETS = ["d8", "cctest", "unittests", "v8_fuzzers", "mkgrokdump",
43           "generate-bytecode-expectations", "inspector-test"]
44# Build targets that get built when you don't specify any (and specified tests
45# don't imply any other targets).
46DEFAULT_TARGETS = ["d8"]
47# Tests that run-tests.py would run by default that can be run with
48# BUILD_TARGETS_TESTS.
49DEFAULT_TESTS = ["cctest", "debugger", "intl", "message", "mjsunit",
50                 "preparser", "unittests"]
51# These can be suffixed to any <arch>.<mode> combo, or used standalone,
52# or used as global modifiers (affecting all <arch>.<mode> combos).
53ACTIONS = {
54  "all": {"targets": BUILD_TARGETS_ALL, "tests": []},
55  "tests": {"targets": BUILD_TARGETS_TEST, "tests": []},
56  "check": {"targets": BUILD_TARGETS_TEST, "tests": DEFAULT_TESTS},
57  "checkall": {"targets": BUILD_TARGETS_ALL, "tests": ["ALL"]},
58}
59
60HELP = """<arch> can be any of: %(arches)s
61<mode> can be any of: %(modes)s
62<target> can be any of:
63 - cctest, d8, unittests, v8_fuzzers (build respective binary)
64 - all (build all binaries)
65 - tests (build test binaries)
66 - check (build test binaries, run most tests)
67 - checkall (build all binaries, run more tests)
68""" % {"arches": " ".join(ARCHES),
69       "modes": " ".join(MODES)}
70
71TESTSUITES_TARGETS = {"benchmarks": "d8",
72              "cctest": "cctest",
73              "debugger": "d8",
74              "fuzzer": "v8_fuzzers",
75              "inspector": "inspector-test",
76              "intl": "d8",
77              "message": "d8",
78              "mjsunit": "d8",
79              "mozilla": "d8",
80              "preparser": "d8",
81              "test262": "d8",
82              "unittests": "unittests",
83              "webkit": "d8"}
84
85OUTDIR = "out"
86
87def DetectGoma():
88  home_goma = os.path.expanduser("~/goma")
89  if os.path.exists(home_goma):
90    return home_goma
91  if os.environ.get("GOMA_DIR"):
92    return os.environ.get("GOMA_DIR")
93  if os.environ.get("GOMADIR"):
94    return os.environ.get("GOMADIR")
95  return None
96
97GOMADIR = DetectGoma()
98IS_GOMA_MACHINE = GOMADIR is not None
99
100USE_GOMA = "true" if IS_GOMA_MACHINE else "false"
101
102RELEASE_ARGS_TEMPLATE = """\
103is_component_build = false
104is_debug = false
105%s
106use_goma = {GOMA}
107goma_dir = \"{GOMA_DIR}\"
108v8_enable_backtrace = true
109v8_enable_disassembler = true
110v8_enable_object_print = true
111v8_enable_verify_heap = true
112""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
113
114DEBUG_ARGS_TEMPLATE = """\
115is_component_build = true
116is_debug = true
117symbol_level = 2
118%s
119use_goma = {GOMA}
120goma_dir = \"{GOMA_DIR}\"
121v8_enable_backtrace = true
122v8_enable_fast_mksnapshot = true
123v8_enable_slow_dchecks = true
124v8_optimized_debug = false
125""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
126
127OPTDEBUG_ARGS_TEMPLATE = """\
128is_component_build = true
129is_debug = true
130symbol_level = 1
131%s
132use_goma = {GOMA}
133goma_dir = \"{GOMA_DIR}\"
134v8_enable_backtrace = true
135v8_enable_fast_mksnapshot = true
136v8_enable_verify_heap = true
137v8_optimized_debug = true
138""".replace("{GOMA}", USE_GOMA).replace("{GOMA_DIR}", str(GOMADIR))
139
140ARGS_TEMPLATES = {
141  "release": RELEASE_ARGS_TEMPLATE,
142  "debug": DEBUG_ARGS_TEMPLATE,
143  "optdebug": OPTDEBUG_ARGS_TEMPLATE
144}
145
146def PrintHelpAndExit():
147  print(__doc__)
148  print(HELP)
149  sys.exit(0)
150
151def _Call(cmd, silent=False):
152  if not silent: print("# %s" % cmd)
153  return subprocess.call(cmd, shell=True)
154
155def _CallWithOutputNoTerminal(cmd):
156  return subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=True)
157
158def _CallWithOutput(cmd):
159  print("# %s" % cmd)
160  # The following trickery is required so that the 'cmd' thinks it's running
161  # in a real terminal, while this script gets to intercept its output.
162  master, slave = pty.openpty()
163  p = subprocess.Popen(cmd, shell=True, stdin=slave, stdout=slave, stderr=slave)
164  os.close(slave)
165  output = []
166  try:
167    while True:
168      try:
169        data = os.read(master, 512)
170      except OSError as e:
171        if e.errno != errno.EIO: raise
172        break # EIO means EOF on some systems
173      else:
174        if not data: # EOF
175          break
176        print(data, end="")
177        sys.stdout.flush()
178        output.append(data)
179  finally:
180    os.close(master)
181    p.wait()
182  return p.returncode, "".join(output)
183
184def _Which(cmd):
185  for path in os.environ["PATH"].split(os.pathsep):
186    if os.path.exists(os.path.join(path, cmd)):
187      return os.path.join(path, cmd)
188  return None
189
190def _Write(filename, content):
191  print("# echo > %s << EOF\n%sEOF" % (filename, content))
192  with open(filename, "w") as f:
193    f.write(content)
194
195def _Notify(summary, body):
196  if _Which('notify-send') is not None:
197    _Call("notify-send '{}' '{}'".format(summary, body), silent=True)
198  else:
199    print("{} - {}".format(summary, body))
200
201def GetPath(arch, mode):
202  subdir = "%s.%s" % (arch, mode)
203  return os.path.join(OUTDIR, subdir)
204
205class Config(object):
206  def __init__(self, arch, mode, targets, tests=[]):
207    self.arch = arch
208    self.mode = mode
209    self.targets = set(targets)
210    self.tests = set(tests)
211
212  def Extend(self, targets, tests=[]):
213    self.targets.update(targets)
214    self.tests.update(tests)
215
216  def GetTargetCpu(self):
217    cpu = "x86"
218    if "64" in self.arch or self.arch == "s390x":
219      cpu = "x64"
220    return "target_cpu = \"%s\"" % cpu
221
222  def GetV8TargetCpu(self):
223    if self.arch in ("arm", "arm64", "mipsel", "mips64el", "ppc", "ppc64",
224                     "s390", "s390x"):
225      return "\nv8_target_cpu = \"%s\"" % self.arch
226    return ""
227
228  def GetGnArgs(self):
229    template = ARGS_TEMPLATES[self.mode]
230    arch_specific = self.GetTargetCpu() + self.GetV8TargetCpu()
231    return template % arch_specific
232
233  def Build(self):
234    path = GetPath(self.arch, self.mode)
235    args_gn = os.path.join(path, "args.gn")
236    if not os.path.exists(path):
237      print("# mkdir -p %s" % path)
238      os.makedirs(path)
239    if not os.path.exists(args_gn):
240      _Write(args_gn, self.GetGnArgs())
241      code = _Call("gn gen %s" % path)
242      if code != 0: return code
243    targets = " ".join(self.targets)
244    # The implementation of mksnapshot failure detection relies on
245    # the "pty" module and GDB presence, so skip it on non-Linux.
246    if "linux" not in sys.platform:
247      return _Call("autoninja -C %s %s" % (path, targets))
248
249    return_code, output = _CallWithOutput("autoninja -C %s %s" %
250                                          (path, targets))
251    if return_code != 0 and "FAILED: snapshot_blob.bin" in output:
252      csa_trap = re.compile("Specify option( --csa-trap-on-node=[^ ]*)")
253      match = csa_trap.search(output)
254      extra_opt = match.group(1) if match else ""
255      _Notify("V8 build requires your attention",
256              "Detected mksnapshot failure, re-running in GDB...")
257      _Call("gdb -args %(path)s/mksnapshot "
258            "--startup_src %(path)s/gen/snapshot.cc "
259            "--random-seed 314159265 "
260            "--startup-blob %(path)s/snapshot_blob.bin"
261            "%(extra)s"% {"path": path, "extra": extra_opt})
262    return return_code
263
264  def RunTests(self):
265    if not self.tests: return 0
266    if "ALL" in self.tests:
267      tests = ""
268    else:
269      tests = " ".join(self.tests)
270    return _Call("tools/run-tests.py --arch=%s --mode=%s %s" %
271                 (self.arch, self.mode, tests))
272
273def GetTestBinary(argstring):
274  for suite in TESTSUITES_TARGETS:
275    if argstring.startswith(suite): return TESTSUITES_TARGETS[suite]
276  return None
277
278class ArgumentParser(object):
279  def __init__(self):
280    self.global_targets = set()
281    self.global_tests = set()
282    self.global_actions = set()
283    self.configs = {}
284
285  def PopulateConfigs(self, arches, modes, targets, tests):
286    for a in arches:
287      for m in modes:
288        path = GetPath(a, m)
289        if path not in self.configs:
290          self.configs[path] = Config(a, m, targets, tests)
291        else:
292          self.configs[path].Extend(targets, tests)
293
294  def ProcessGlobalActions(self):
295    have_configs = len(self.configs) > 0
296    for action in self.global_actions:
297      impact = ACTIONS[action]
298      if (have_configs):
299        for c in self.configs:
300          self.configs[c].Extend(**impact)
301      else:
302        self.PopulateConfigs(DEFAULT_ARCHES, DEFAULT_MODES, **impact)
303
304  def ParseArg(self, argstring):
305    if argstring in ("-h", "--help", "help"):
306      PrintHelpAndExit()
307    arches = []
308    modes = []
309    targets = []
310    actions = []
311    tests = []
312    # Specifying a single unit test looks like "unittests/Foo.Bar".
313    if argstring.startswith("unittests/"):
314      words = [argstring]
315    else:
316      words = argstring.split('.')
317    if len(words) == 1:
318      word = words[0]
319      if word in ACTIONS:
320        self.global_actions.add(word)
321        return
322      if word in TARGETS:
323        self.global_targets.add(word)
324        return
325      maybe_target = GetTestBinary(word)
326      if maybe_target is not None:
327        self.global_tests.add(word)
328        self.global_targets.add(maybe_target)
329        return
330    for word in words:
331      if word in ARCHES:
332        arches.append(word)
333      elif word in MODES:
334        modes.append(word)
335      elif word in TARGETS:
336        targets.append(word)
337      elif word in ACTIONS:
338        actions.append(word)
339      else:
340        print("Didn't understand: %s" % word)
341        sys.exit(1)
342    # Process actions.
343    for action in actions:
344      impact = ACTIONS[action]
345      targets += impact["targets"]
346      tests += impact["tests"]
347    # Fill in defaults for things that weren't specified.
348    arches = arches or DEFAULT_ARCHES
349    modes = modes or DEFAULT_MODES
350    targets = targets or DEFAULT_TARGETS
351    # Produce configs.
352    self.PopulateConfigs(arches, modes, targets, tests)
353
354  def ParseArguments(self, argv):
355    if len(argv) == 0:
356      PrintHelpAndExit()
357    for argstring in argv:
358      self.ParseArg(argstring)
359    self.ProcessGlobalActions()
360    for c in self.configs:
361      self.configs[c].Extend(self.global_targets, self.global_tests)
362    return self.configs
363
364def Main(argv):
365  parser = ArgumentParser()
366  configs = parser.ParseArguments(argv[1:])
367  return_code = 0
368  # If we have Goma but it is not running, start it.
369  if (GOMADIR is not None and
370      _Call("ps -e | grep compiler_proxy > /dev/null", silent=True) != 0):
371    _Call("%s/goma_ctl.py ensure_start" % GOMADIR)
372  for c in configs:
373    return_code += configs[c].Build()
374  if return_code == 0:
375    for c in configs:
376      return_code += configs[c].RunTests()
377  if return_code == 0:
378    _Notify('Done!', 'V8 compilation finished successfully.')
379  else:
380    _Notify('Error!', 'V8 compilation finished with errors.')
381  return return_code
382
383if __name__ == "__main__":
384  sys.exit(Main(sys.argv))
385