1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31from collections import OrderedDict
32import itertools
33import json
34import multiprocessing
35import optparse
36import os
37from os.path import getmtime, isdir, join
38import platform
39import random
40import shlex
41import subprocess
42import sys
43import time
44
45from testrunner.local import execution
46from testrunner.local import progress
47from testrunner.local import testsuite
48from testrunner.local.variants import ALL_VARIANTS
49from testrunner.local import utils
50from testrunner.local import verbose
51from testrunner.network import network_execution
52from testrunner.objects import context
53
54
55# Base dir of the v8 checkout to be used as cwd.
56BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
57
58DEFAULT_OUT_GN = "out.gn"
59
60ARCH_GUESS = utils.DefaultArch()
61
62# Map of test name synonyms to lists of test suites. Should be ordered by
63# expected runtimes (suites with slow test cases first). These groups are
64# invoked in seperate steps on the bots.
65TEST_MAP = {
66  # This needs to stay in sync with test/bot_default.isolate.
67  "bot_default": [
68    "mjsunit",
69    "cctest",
70    "debugger",
71    "inspector",
72    "webkit",
73    "fuzzer",
74    "message",
75    "preparser",
76    "intl",
77    "unittests",
78  ],
79  # This needs to stay in sync with test/default.isolate.
80  "default": [
81    "mjsunit",
82    "cctest",
83    "debugger",
84    "inspector",
85    "fuzzer",
86    "message",
87    "preparser",
88    "intl",
89    "unittests",
90  ],
91  # This needs to stay in sync with test/optimize_for_size.isolate.
92  "optimize_for_size": [
93    "mjsunit",
94    "cctest",
95    "debugger",
96    "inspector",
97    "webkit",
98    "intl",
99  ],
100  "unittests": [
101    "unittests",
102  ],
103}
104
105TIMEOUT_DEFAULT = 60
106
107VARIANTS = ["default", "turbofan", "ignition_staging"]
108
109MORE_VARIANTS = [
110  "ignition",
111  "stress",
112  "turbofan_opt",
113  "asm_wasm",
114]
115
116EXHAUSTIVE_VARIANTS = VARIANTS + MORE_VARIANTS
117
118VARIANT_ALIASES = {
119  # The default for developer workstations.
120  "dev": VARIANTS,
121  # Additional variants, run on all bots.
122  "more": MORE_VARIANTS,
123  # Additional variants, run on a subset of bots.
124  "extra": ["nocrankshaft"],
125}
126
127DEBUG_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
128               "--nofold-constants", "--enable-slow-asserts",
129               "--debug-code", "--verify-heap"]
130RELEASE_FLAGS = ["--nohard-abort", "--nodead-code-elimination",
131                 "--nofold-constants"]
132
133MODES = {
134  "debug": {
135    "flags": DEBUG_FLAGS,
136    "timeout_scalefactor": 4,
137    "status_mode": "debug",
138    "execution_mode": "debug",
139    "output_folder": "debug",
140  },
141  "optdebug": {
142    "flags": DEBUG_FLAGS,
143    "timeout_scalefactor": 4,
144    "status_mode": "debug",
145    "execution_mode": "debug",
146    "output_folder": "optdebug",
147  },
148  "release": {
149    "flags": RELEASE_FLAGS,
150    "timeout_scalefactor": 1,
151    "status_mode": "release",
152    "execution_mode": "release",
153    "output_folder": "release",
154  },
155  # Normal trybot release configuration. There, dchecks are always on which
156  # implies debug is set. Hence, the status file needs to assume debug-like
157  # behavior/timeouts.
158  "tryrelease": {
159    "flags": RELEASE_FLAGS,
160    "timeout_scalefactor": 1,
161    "status_mode": "debug",
162    "execution_mode": "release",
163    "output_folder": "release",
164  },
165  # This mode requires v8 to be compiled with dchecks and slow dchecks.
166  "slowrelease": {
167    "flags": RELEASE_FLAGS + ["--enable-slow-asserts"],
168    "timeout_scalefactor": 2,
169    "status_mode": "debug",
170    "execution_mode": "release",
171    "output_folder": "release",
172  },
173}
174
175GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
176                   "--concurrent-recompilation-queue-length=64",
177                   "--concurrent-recompilation-delay=500",
178                   "--concurrent-recompilation"]
179
180SUPPORTED_ARCHS = ["android_arm",
181                   "android_arm64",
182                   "android_ia32",
183                   "android_x64",
184                   "arm",
185                   "ia32",
186                   "x87",
187                   "mips",
188                   "mipsel",
189                   "mips64",
190                   "mips64el",
191                   "s390",
192                   "s390x",
193                   "ppc",
194                   "ppc64",
195                   "x64",
196                   "x32",
197                   "arm64"]
198# Double the timeout for these:
199SLOW_ARCHS = ["android_arm",
200              "android_arm64",
201              "android_ia32",
202              "android_x64",
203              "arm",
204              "mips",
205              "mipsel",
206              "mips64",
207              "mips64el",
208              "s390",
209              "s390x",
210              "x87",
211              "arm64"]
212
213
214def BuildOptions():
215  result = optparse.OptionParser()
216  result.usage = '%prog [options] [tests]'
217  result.description = """TESTS: %s""" % (TEST_MAP["default"])
218  result.add_option("--arch",
219                    help=("The architecture to run tests for, "
220                          "'auto' or 'native' for auto-detect: %s" % SUPPORTED_ARCHS),
221                    default="ia32,x64,arm")
222  result.add_option("--arch-and-mode",
223                    help="Architecture and mode in the format 'arch.mode'",
224                    default=None)
225  result.add_option("--asan",
226                    help="Regard test expectations for ASAN",
227                    default=False, action="store_true")
228  result.add_option("--sancov-dir",
229                    help="Directory where to collect coverage data")
230  result.add_option("--cfi-vptr",
231                    help="Run tests with UBSAN cfi_vptr option.",
232                    default=False, action="store_true")
233  result.add_option("--buildbot",
234                    help="Adapt to path structure used on buildbots",
235                    default=False, action="store_true")
236  result.add_option("--dcheck-always-on",
237                    help="Indicates that V8 was compiled with DCHECKs enabled",
238                    default=False, action="store_true")
239  result.add_option("--novfp3",
240                    help="Indicates that V8 was compiled without VFP3 support",
241                    default=False, action="store_true")
242  result.add_option("--cat", help="Print the source of the tests",
243                    default=False, action="store_true")
244  result.add_option("--slow-tests",
245                    help="Regard slow tests (run|skip|dontcare)",
246                    default="dontcare")
247  result.add_option("--pass-fail-tests",
248                    help="Regard pass|fail tests (run|skip|dontcare)",
249                    default="dontcare")
250  result.add_option("--gc-stress",
251                    help="Switch on GC stress mode",
252                    default=False, action="store_true")
253  result.add_option("--gcov-coverage",
254                    help="Uses executables instrumented for gcov coverage",
255                    default=False, action="store_true")
256  result.add_option("--command-prefix",
257                    help="Prepended to each shell command used to run a test",
258                    default="")
259  result.add_option("--download-data", help="Download missing test suite data",
260                    default=False, action="store_true")
261  result.add_option("--download-data-only",
262                    help="Deprecated",
263                    default=False, action="store_true")
264  result.add_option("--enable-inspector",
265                    help="Indicates a build with inspector support",
266                    default=False, action="store_true")
267  result.add_option("--extra-flags",
268                    help="Additional flags to pass to each test command",
269                    default="")
270  result.add_option("--isolates", help="Whether to test isolates",
271                    default=False, action="store_true")
272  result.add_option("-j", help="The number of parallel tasks to run",
273                    default=0, type="int")
274  result.add_option("-m", "--mode",
275                    help="The test modes in which to run (comma-separated,"
276                    " uppercase for ninja and buildbot builds): %s" % MODES.keys(),
277                    default="release,debug")
278  result.add_option("--no-harness", "--noharness",
279                    help="Run without test harness of a given suite",
280                    default=False, action="store_true")
281  result.add_option("--no-i18n", "--noi18n",
282                    help="Skip internationalization tests",
283                    default=False, action="store_true")
284  result.add_option("--no-network", "--nonetwork",
285                    help="Don't distribute tests on the network",
286                    default=(utils.GuessOS() != "linux"),
287                    dest="no_network", action="store_true")
288  result.add_option("--no-presubmit", "--nopresubmit",
289                    help='Skip presubmit checks (deprecated)',
290                    default=False, dest="no_presubmit", action="store_true")
291  result.add_option("--no-snap", "--nosnap",
292                    help='Test a build compiled without snapshot.',
293                    default=False, dest="no_snap", action="store_true")
294  result.add_option("--no-sorting", "--nosorting",
295                    help="Don't sort tests according to duration of last run.",
296                    default=False, dest="no_sorting", action="store_true")
297  result.add_option("--no-variants", "--novariants",
298                    help="Don't run any testing variants",
299                    default=False, dest="no_variants", action="store_true")
300  result.add_option("--variants",
301                    help="Comma-separated list of testing variants;"
302                    " default: \"%s\"" % ",".join(VARIANTS))
303  result.add_option("--exhaustive-variants",
304                    default=False, action="store_true",
305                    help="Use exhaustive set of default variants:"
306                    " \"%s\"" % ",".join(EXHAUSTIVE_VARIANTS))
307  result.add_option("--outdir", help="Base directory with compile output",
308                    default="out")
309  result.add_option("--gn", help="Scan out.gn for the last built configuration",
310                    default=False, action="store_true")
311  result.add_option("--predictable",
312                    help="Compare output of several reruns of each test",
313                    default=False, action="store_true")
314  result.add_option("-p", "--progress",
315                    help=("The style of progress indicator"
316                          " (verbose, dots, color, mono)"),
317                    choices=progress.PROGRESS_INDICATORS.keys(), default="mono")
318  result.add_option("--quickcheck", default=False, action="store_true",
319                    help=("Quick check mode (skip slow tests)"))
320  result.add_option("--report", help="Print a summary of the tests to be run",
321                    default=False, action="store_true")
322  result.add_option("--json-test-results",
323                    help="Path to a file for storing json results.")
324  result.add_option("--rerun-failures-count",
325                    help=("Number of times to rerun each failing test case. "
326                          "Very slow tests will be rerun only once."),
327                    default=0, type="int")
328  result.add_option("--rerun-failures-max",
329                    help="Maximum number of failing test cases to rerun.",
330                    default=100, type="int")
331  result.add_option("--shard-count",
332                    help="Split testsuites into this number of shards",
333                    default=1, type="int")
334  result.add_option("--shard-run",
335                    help="Run this shard from the split up tests.",
336                    default=1, type="int")
337  result.add_option("--shell", help="DEPRECATED! use --shell-dir", default="")
338  result.add_option("--shell-dir", help="Directory containing executables",
339                    default="")
340  result.add_option("--dont-skip-slow-simulator-tests",
341                    help="Don't skip more slow tests when using a simulator.",
342                    default=False, action="store_true",
343                    dest="dont_skip_simulator_slow_tests")
344  result.add_option("--swarming",
345                    help="Indicates running test driver on swarming.",
346                    default=False, action="store_true")
347  result.add_option("--time", help="Print timing information after running",
348                    default=False, action="store_true")
349  result.add_option("-t", "--timeout", help="Timeout in seconds",
350                    default=TIMEOUT_DEFAULT, type="int")
351  result.add_option("--tsan",
352                    help="Regard test expectations for TSAN",
353                    default=False, action="store_true")
354  result.add_option("-v", "--verbose", help="Verbose output",
355                    default=False, action="store_true")
356  result.add_option("--valgrind", help="Run tests through valgrind",
357                    default=False, action="store_true")
358  result.add_option("--warn-unused", help="Report unused rules",
359                    default=False, action="store_true")
360  result.add_option("--junitout", help="File name of the JUnit output")
361  result.add_option("--junittestsuite",
362                    help="The testsuite name in the JUnit output file",
363                    default="v8tests")
364  result.add_option("--random-seed", default=0, dest="random_seed", type="int",
365                    help="Default seed for initializing random generator")
366  result.add_option("--random-seed-stress-count", default=1, type="int",
367                    dest="random_seed_stress_count",
368                    help="Number of runs with different random seeds")
369  result.add_option("--msan",
370                    help="Regard test expectations for MSAN",
371                    default=False, action="store_true")
372  return result
373
374
375def RandomSeed():
376  seed = 0
377  while not seed:
378    seed = random.SystemRandom().randint(-2147483648, 2147483647)
379  return seed
380
381
382def BuildbotToV8Mode(config):
383  """Convert buildbot build configs to configs understood by the v8 runner.
384
385  V8 configs are always lower case and without the additional _x64 suffix for
386  64 bit builds on windows with ninja.
387  """
388  mode = config[:-4] if config.endswith('_x64') else config
389  return mode.lower()
390
391def SetupEnvironment(options):
392  """Setup additional environment variables."""
393
394  # Many tests assume an English interface.
395  os.environ['LANG'] = 'en_US.UTF-8'
396
397  symbolizer = 'external_symbolizer_path=%s' % (
398      os.path.join(
399          BASE_DIR, 'third_party', 'llvm-build', 'Release+Asserts', 'bin',
400          'llvm-symbolizer',
401      )
402  )
403
404  if options.asan:
405    os.environ['ASAN_OPTIONS'] = symbolizer
406
407  if options.sancov_dir:
408    assert os.path.exists(options.sancov_dir)
409    os.environ['ASAN_OPTIONS'] = ":".join([
410      'coverage=1',
411      'coverage_dir=%s' % options.sancov_dir,
412      symbolizer,
413    ])
414
415  if options.cfi_vptr:
416    os.environ['UBSAN_OPTIONS'] = ":".join([
417      'print_stacktrace=1',
418      'print_summary=1',
419      'symbolize=1',
420      symbolizer,
421    ])
422
423  if options.msan:
424    os.environ['MSAN_OPTIONS'] = symbolizer
425
426  if options.tsan:
427    suppressions_file = os.path.join(
428        BASE_DIR, 'tools', 'sanitizers', 'tsan_suppressions.txt')
429    os.environ['TSAN_OPTIONS'] = " ".join([
430      symbolizer,
431      'suppressions=%s' % suppressions_file,
432      'exit_code=0',
433      'report_thread_leaks=0',
434      'history_size=7',
435      'report_destroy_locked=0',
436    ])
437
438def ProcessOptions(options):
439  global VARIANTS
440
441  # First try to auto-detect configurations based on the build if GN was
442  # used. This can't be overridden by cmd-line arguments.
443  options.auto_detect = False
444  if options.gn:
445    gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN)
446    latest_timestamp = -1
447    latest_config = None
448    for gn_config in os.listdir(gn_out_dir):
449      gn_config_dir = os.path.join(gn_out_dir, gn_config)
450      if not isdir(gn_config_dir):
451        continue
452      if os.path.getmtime(gn_config_dir) > latest_timestamp:
453        latest_timestamp = os.path.getmtime(gn_config_dir)
454        latest_config = gn_config
455    if latest_config:
456      print(">>> Latest GN build found is %s" % latest_config)
457      options.outdir = os.path.join(DEFAULT_OUT_GN, latest_config)
458
459  if options.buildbot:
460    build_config_path = os.path.join(
461        BASE_DIR, options.outdir, options.mode, "v8_build_config.json")
462  else:
463    build_config_path = os.path.join(
464        BASE_DIR, options.outdir, "v8_build_config.json")
465
466  if os.path.exists(build_config_path):
467    try:
468      with open(build_config_path) as f:
469        build_config = json.load(f)
470    except Exception:
471      print ("%s exists but contains invalid json. Is your build up-to-date?" %
472             build_config_path)
473      return False
474    options.auto_detect = True
475
476    # In auto-detect mode the outdir is always where we found the build config.
477    # This ensures that we'll also take the build products from there.
478    options.outdir = os.path.dirname(build_config_path)
479
480    options.arch_and_mode = None
481    options.arch = build_config["v8_target_cpu"]
482    if options.arch == 'x86':
483      # TODO(machenbach): Transform all to x86 eventually.
484      options.arch = 'ia32'
485    options.asan = build_config["is_asan"]
486    options.dcheck_always_on = build_config["dcheck_always_on"]
487    options.enable_inspector = build_config["v8_enable_inspector"]
488    options.mode = 'debug' if build_config["is_debug"] else 'release'
489    options.msan = build_config["is_msan"]
490    options.no_i18n = not build_config["v8_enable_i18n_support"]
491    options.no_snap = not build_config["v8_use_snapshot"]
492    options.tsan = build_config["is_tsan"]
493
494  # Architecture and mode related stuff.
495  if options.arch_and_mode:
496    options.arch_and_mode = [arch_and_mode.split(".")
497        for arch_and_mode in options.arch_and_mode.split(",")]
498    options.arch = ",".join([tokens[0] for tokens in options.arch_and_mode])
499    options.mode = ",".join([tokens[1] for tokens in options.arch_and_mode])
500  options.mode = options.mode.split(",")
501  for mode in options.mode:
502    if not BuildbotToV8Mode(mode) in MODES:
503      print "Unknown mode %s" % mode
504      return False
505  if options.arch in ["auto", "native"]:
506    options.arch = ARCH_GUESS
507  options.arch = options.arch.split(",")
508  for arch in options.arch:
509    if not arch in SUPPORTED_ARCHS:
510      print "Unknown architecture %s" % arch
511      return False
512
513  # Store the final configuration in arch_and_mode list. Don't overwrite
514  # predefined arch_and_mode since it is more expressive than arch and mode.
515  if not options.arch_and_mode:
516    options.arch_and_mode = itertools.product(options.arch, options.mode)
517
518  # Special processing of other options, sorted alphabetically.
519
520  if options.buildbot:
521    options.no_network = True
522  if options.command_prefix:
523    print("Specifying --command-prefix disables network distribution, "
524          "running tests locally.")
525    options.no_network = True
526  options.command_prefix = shlex.split(options.command_prefix)
527  options.extra_flags = shlex.split(options.extra_flags)
528
529  if options.gc_stress:
530    options.extra_flags += GC_STRESS_FLAGS
531
532  if options.asan:
533    options.extra_flags.append("--invoke-weak-callbacks")
534    options.extra_flags.append("--omit-quit")
535
536  if options.novfp3:
537    options.extra_flags.append("--noenable-vfp3")
538
539  if options.exhaustive_variants:
540    # This is used on many bots. It includes a larger set of default variants.
541    # Other options for manipulating variants still apply afterwards.
542    VARIANTS = EXHAUSTIVE_VARIANTS
543
544  # TODO(machenbach): Figure out how to test a bigger subset of variants on
545  # msan and tsan.
546  if options.msan:
547    VARIANTS = ["default"]
548
549  if options.tsan:
550    VARIANTS = ["default"]
551
552  if options.j == 0:
553    options.j = multiprocessing.cpu_count()
554
555  if options.random_seed_stress_count <= 1 and options.random_seed == 0:
556    options.random_seed = RandomSeed()
557
558  def excl(*args):
559    """Returns true if zero or one of multiple arguments are true."""
560    return reduce(lambda x, y: x + y, args) <= 1
561
562  if not excl(options.no_variants, bool(options.variants)):
563    print("Use only one of --no-variants or --variants.")
564    return False
565  if options.quickcheck:
566    VARIANTS = ["default", "stress"]
567    options.slow_tests = "skip"
568    options.pass_fail_tests = "skip"
569  if options.no_variants:
570    VARIANTS = ["default"]
571  if options.variants:
572    VARIANTS = options.variants.split(",")
573
574    # Resolve variant aliases.
575    VARIANTS = reduce(
576        list.__add__,
577        (VARIANT_ALIASES.get(v, [v]) for v in VARIANTS),
578        [],
579    )
580
581    if not set(VARIANTS).issubset(ALL_VARIANTS):
582      print "All variants must be in %s" % str(ALL_VARIANTS)
583      return False
584  if options.predictable:
585    VARIANTS = ["default"]
586    options.extra_flags.append("--predictable")
587    options.extra_flags.append("--verify_predictable")
588    options.extra_flags.append("--no-inline-new")
589
590  # Dedupe.
591  VARIANTS = list(set(VARIANTS))
592
593  if not options.shell_dir:
594    if options.shell:
595      print "Warning: --shell is deprecated, use --shell-dir instead."
596      options.shell_dir = os.path.dirname(options.shell)
597  if options.valgrind:
598    run_valgrind = os.path.join("tools", "run-valgrind.py")
599    # This is OK for distributed running, so we don't need to set no_network.
600    options.command_prefix = (["python", "-u", run_valgrind] +
601                              options.command_prefix)
602  def CheckTestMode(name, option):
603    if not option in ["run", "skip", "dontcare"]:
604      print "Unknown %s mode %s" % (name, option)
605      return False
606    return True
607  if not CheckTestMode("slow test", options.slow_tests):
608    return False
609  if not CheckTestMode("pass|fail test", options.pass_fail_tests):
610    return False
611  if options.no_i18n:
612    TEST_MAP["bot_default"].remove("intl")
613    TEST_MAP["default"].remove("intl")
614  if not options.enable_inspector:
615    TEST_MAP["default"].remove("inspector")
616    TEST_MAP["bot_default"].remove("inspector")
617    TEST_MAP["optimize_for_size"].remove("inspector")
618    TEST_MAP["default"].remove("debugger")
619    TEST_MAP["bot_default"].remove("debugger")
620    TEST_MAP["optimize_for_size"].remove("debugger")
621  return True
622
623
624def ShardTests(tests, options):
625  # Read gtest shard configuration from environment (e.g. set by swarming).
626  # If none is present, use values passed on the command line.
627  shard_count = int(os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count))
628  shard_run = os.environ.get('GTEST_SHARD_INDEX')
629  if shard_run is not None:
630    # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0.
631    shard_run = int(shard_run) + 1
632  else:
633    shard_run = options.shard_run
634
635  if options.shard_count > 1:
636    # Log if a value was passed on the cmd line and it differs from the
637    # environment variables.
638    if options.shard_count != shard_count:
639      print("shard_count from cmd line differs from environment variable "
640            "GTEST_TOTAL_SHARDS")
641    if options.shard_run > 1 and options.shard_run != shard_run:
642      print("shard_run from cmd line differs from environment variable "
643            "GTEST_SHARD_INDEX")
644
645  if shard_count < 2:
646    return tests
647  if shard_run < 1 or shard_run > shard_count:
648    print "shard-run not a valid number, should be in [1:shard-count]"
649    print "defaulting back to running all tests"
650    return tests
651  count = 0
652  shard = []
653  for test in tests:
654    if count % shard_count == shard_run - 1:
655      shard.append(test)
656    count += 1
657  return shard
658
659
660def Main():
661  # Use the v8 root as cwd as some test cases use "load" with relative paths.
662  os.chdir(BASE_DIR)
663
664  parser = BuildOptions()
665  (options, args) = parser.parse_args()
666  if not ProcessOptions(options):
667    parser.print_help()
668    return 1
669  SetupEnvironment(options)
670
671  if options.swarming:
672    # Swarming doesn't print how isolated commands are called. Lets make this
673    # less cryptic by printing it ourselves.
674    print ' '.join(sys.argv)
675
676  exit_code = 0
677
678  suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
679
680  # Use default tests if no test configuration was provided at the cmd line.
681  if len(args) == 0:
682    args = ["default"]
683
684  # Expand arguments with grouped tests. The args should reflect the list of
685  # suites as otherwise filters would break.
686  def ExpandTestGroups(name):
687    if name in TEST_MAP:
688      return [suite for suite in TEST_MAP[name]]
689    else:
690      return [name]
691  args = reduce(lambda x, y: x + y,
692         [ExpandTestGroups(arg) for arg in args],
693         [])
694
695  args_suites = OrderedDict() # Used as set
696  for arg in args:
697    args_suites[arg.split('/')[0]] = True
698  suite_paths = [ s for s in args_suites if s in suite_paths ]
699
700  suites = []
701  for root in suite_paths:
702    suite = testsuite.TestSuite.LoadTestSuite(
703        os.path.join(BASE_DIR, "test", root))
704    if suite:
705      suites.append(suite)
706
707  if options.download_data or options.download_data_only:
708    for s in suites:
709      s.DownloadData()
710
711  if options.download_data_only:
712    return exit_code
713
714  for s in suites:
715    s.PrepareSources()
716
717  for (arch, mode) in options.arch_and_mode:
718    try:
719      code = Execute(arch, mode, args, options, suites)
720    except KeyboardInterrupt:
721      return 2
722    exit_code = exit_code or code
723  return exit_code
724
725
726def Execute(arch, mode, args, options, suites):
727  print(">>> Running tests for %s.%s" % (arch, mode))
728
729  shell_dir = options.shell_dir
730  if not shell_dir:
731    if options.auto_detect:
732      # If an output dir with a build was passed, test directly in that
733      # directory.
734      shell_dir = os.path.join(BASE_DIR, options.outdir)
735    elif options.buildbot:
736      # TODO(machenbach): Get rid of different output folder location on
737      # buildbot. Currently this is capitalized Release and Debug.
738      shell_dir = os.path.join(BASE_DIR, options.outdir, mode)
739      mode = BuildbotToV8Mode(mode)
740    else:
741      shell_dir = os.path.join(
742          BASE_DIR,
743          options.outdir,
744          "%s.%s" % (arch, MODES[mode]["output_folder"]),
745      )
746  if not os.path.exists(shell_dir):
747      raise Exception('Could not find shell_dir: "%s"' % shell_dir)
748
749  # Populate context object.
750  mode_flags = MODES[mode]["flags"]
751
752  # Simulators are slow, therefore allow a longer timeout.
753  if arch in SLOW_ARCHS:
754    options.timeout *= 2
755
756  options.timeout *= MODES[mode]["timeout_scalefactor"]
757
758  if options.predictable:
759    # Predictable mode is slower.
760    options.timeout *= 2
761
762  ctx = context.Context(arch, MODES[mode]["execution_mode"], shell_dir,
763                        mode_flags, options.verbose,
764                        options.timeout,
765                        options.isolates,
766                        options.command_prefix,
767                        options.extra_flags,
768                        options.no_i18n,
769                        options.random_seed,
770                        options.no_sorting,
771                        options.rerun_failures_count,
772                        options.rerun_failures_max,
773                        options.predictable,
774                        options.no_harness,
775                        use_perf_data=not options.swarming,
776                        sancov_dir=options.sancov_dir)
777
778  # TODO(all): Combine "simulator" and "simulator_run".
779  # TODO(machenbach): In GN we can derive simulator run from
780  # target_arch != v8_target_arch in the dumped build config.
781  simulator_run = not options.dont_skip_simulator_slow_tests and \
782      arch in ['arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', \
783               'ppc', 'ppc64'] and \
784      ARCH_GUESS and arch != ARCH_GUESS
785  # Find available test suites and read test cases from them.
786  variables = {
787    "arch": arch,
788    "asan": options.asan,
789    "deopt_fuzzer": False,
790    "gc_stress": options.gc_stress,
791    "gcov_coverage": options.gcov_coverage,
792    "isolates": options.isolates,
793    "mode": MODES[mode]["status_mode"],
794    "no_i18n": options.no_i18n,
795    "no_snap": options.no_snap,
796    "simulator_run": simulator_run,
797    "simulator": utils.UseSimulator(arch),
798    "system": utils.GuessOS(),
799    "tsan": options.tsan,
800    "msan": options.msan,
801    "dcheck_always_on": options.dcheck_always_on,
802    "novfp3": options.novfp3,
803    "predictable": options.predictable,
804    "byteorder": sys.byteorder,
805  }
806  all_tests = []
807  num_tests = 0
808  for s in suites:
809    s.ReadStatusFile(variables)
810    s.ReadTestCases(ctx)
811    if len(args) > 0:
812      s.FilterTestCasesByArgs(args)
813    all_tests += s.tests
814
815    # First filtering by status applying the generic rules (independent of
816    # variants).
817    s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
818                              options.pass_fail_tests)
819
820    if options.cat:
821      verbose.PrintTestSource(s.tests)
822      continue
823    variant_gen = s.CreateVariantGenerator(VARIANTS)
824    variant_tests = [ t.CopyAddingFlags(v, flags)
825                      for t in s.tests
826                      for v in variant_gen.FilterVariantsByTest(t)
827                      for flags in variant_gen.GetFlagSets(t, v) ]
828
829    if options.random_seed_stress_count > 1:
830      # Duplicate test for random seed stress mode.
831      def iter_seed_flags():
832        for i in range(0, options.random_seed_stress_count):
833          # Use given random seed for all runs (set by default in execution.py)
834          # or a new random seed if none is specified.
835          if options.random_seed:
836            yield []
837          else:
838            yield ["--random-seed=%d" % RandomSeed()]
839      s.tests = [
840        t.CopyAddingFlags(t.variant, flags)
841        for t in variant_tests
842        for flags in iter_seed_flags()
843      ]
844    else:
845      s.tests = variant_tests
846
847    # Second filtering by status applying the variant-dependent rules.
848    s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
849                              options.pass_fail_tests, variants=True)
850
851    s.tests = ShardTests(s.tests, options)
852    num_tests += len(s.tests)
853
854  if options.cat:
855    return 0  # We're done here.
856
857  if options.report:
858    verbose.PrintReport(all_tests)
859
860  # Run the tests, either locally or distributed on the network.
861  start_time = time.time()
862  progress_indicator = progress.IndicatorNotifier()
863  progress_indicator.Register(progress.PROGRESS_INDICATORS[options.progress]())
864  if options.junitout:
865    progress_indicator.Register(progress.JUnitTestProgressIndicator(
866        options.junitout, options.junittestsuite))
867  if options.json_test_results:
868    progress_indicator.Register(progress.JsonTestProgressIndicator(
869        options.json_test_results, arch, MODES[mode]["execution_mode"],
870        ctx.random_seed))
871
872  run_networked = not options.no_network
873  if not run_networked:
874    if options.verbose:
875      print("Network distribution disabled, running tests locally.")
876  elif utils.GuessOS() != "linux":
877    print("Network distribution is only supported on Linux, sorry!")
878    run_networked = False
879  peers = []
880  if run_networked:
881    peers = network_execution.GetPeers()
882    if not peers:
883      print("No connection to distribution server; running tests locally.")
884      run_networked = False
885    elif len(peers) == 1:
886      print("No other peers on the network; running tests locally.")
887      run_networked = False
888    elif num_tests <= 100:
889      print("Less than 100 tests, running them locally.")
890      run_networked = False
891
892  if run_networked:
893    runner = network_execution.NetworkedRunner(suites, progress_indicator,
894                                               ctx, peers, BASE_DIR)
895  else:
896    runner = execution.Runner(suites, progress_indicator, ctx)
897
898  exit_code = runner.Run(options.j)
899  overall_duration = time.time() - start_time
900
901  if options.time:
902    verbose.PrintTestDurations(suites, overall_duration)
903
904  if num_tests == 0:
905    print("Warning: no tests were run!")
906
907  if exit_code == 1 and options.json_test_results:
908    print("Force exit code 0 after failures. Json test results file generated "
909          "with failure information.")
910    exit_code = 0
911
912  if options.sancov_dir:
913    # If tests ran with sanitizer coverage, merge coverage files in the end.
914    try:
915      print "Merging sancov files."
916      subprocess.check_call([
917        sys.executable,
918        join(BASE_DIR, "tools", "sanitizers", "sancov_merger.py"),
919        "--coverage-dir=%s" % options.sancov_dir])
920    except:
921      print >> sys.stderr, "Error: Merging sancov files failed."
922      exit_code = 1
923
924  return exit_code
925
926
927if __name__ == "__main__":
928  sys.exit(Main())
929