1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30
31import json
32import math
33import multiprocessing
34import optparse
35import os
36from os.path import join
37import random
38import shlex
39import subprocess
40import sys
41import time
42
43from testrunner.local import execution
44from testrunner.local import progress
45from testrunner.local import testsuite
46from testrunner.local import utils
47from testrunner.local import verbose
48from testrunner.objects import context
49
50
51# Base dir of the v8 checkout to be used as cwd.
52BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
53
54ARCH_GUESS = utils.DefaultArch()
55DEFAULT_TESTS = ["mjsunit", "webkit"]
56TIMEOUT_DEFAULT = 60
57TIMEOUT_SCALEFACTOR = {"debug"   : 4,
58                       "release" : 1 }
59
60MODE_FLAGS = {
61    "debug"   : ["--nohard-abort", "--nodead-code-elimination",
62                 "--nofold-constants", "--enable-slow-asserts",
63                 "--debug-code", "--verify-heap",
64                 "--noconcurrent-recompilation"],
65    "release" : ["--nohard-abort", "--nodead-code-elimination",
66                 "--nofold-constants", "--noconcurrent-recompilation"]}
67
68SUPPORTED_ARCHS = ["android_arm",
69                   "android_ia32",
70                   "arm",
71                   "ia32",
72                   "ppc",
73                   "ppc64",
74                   "s390",
75                   "s390x",
76                   "mipsel",
77                   "x64"]
78# Double the timeout for these:
79SLOW_ARCHS = ["android_arm",
80              "android_ia32",
81              "arm",
82              "mipsel"]
83MAX_DEOPT = 1000000000
84DISTRIBUTION_MODES = ["smooth", "random"]
85
86
87class RandomDistribution:
88  def __init__(self, seed=None):
89    seed = seed or random.randint(1, sys.maxint)
90    print "Using random distribution with seed %d" % seed
91    self._random = random.Random(seed)
92
93  def Distribute(self, n, m):
94    if n > m:
95      n = m
96    return self._random.sample(xrange(1, m + 1), n)
97
98
99class SmoothDistribution:
100  """Distribute n numbers into the interval [1:m].
101  F1: Factor of the first derivation of the distribution function.
102  F2: Factor of the second derivation of the distribution function.
103  With F1 and F2 set to 0, the distribution will be equal.
104  """
105  def __init__(self, factor1=2.0, factor2=0.2):
106    self._factor1 = factor1
107    self._factor2 = factor2
108
109  def Distribute(self, n, m):
110    if n > m:
111      n = m
112    if n <= 1:
113      return [ 1 ]
114
115    result = []
116    x = 0.0
117    dx = 1.0
118    ddx = self._factor1
119    dddx = self._factor2
120    for i in range(0, n):
121      result += [ x ]
122      x += dx
123      dx += ddx
124      ddx += dddx
125
126    # Project the distribution into the interval [0:M].
127    result = [ x * m / result[-1] for x in result ]
128
129    # Equalize by n. The closer n is to m, the more equal will be the
130    # distribution.
131    for (i, x) in enumerate(result):
132      # The value of x if it was equally distributed.
133      equal_x = i / float(n - 1) * float(m - 1) + 1
134
135      # Difference factor between actual and equal distribution.
136      diff = 1 - (x / equal_x)
137
138      # Equalize x dependent on the number of values to distribute.
139      result[i] = int(x + (i + 1) * diff)
140    return result
141
142
143def Distribution(options):
144  if options.distribution_mode == "random":
145    return RandomDistribution(options.seed)
146  if options.distribution_mode == "smooth":
147    return SmoothDistribution(options.distribution_factor1,
148                              options.distribution_factor2)
149
150
151def BuildOptions():
152  result = optparse.OptionParser()
153  result.add_option("--arch",
154                    help=("The architecture to run tests for, "
155                          "'auto' or 'native' for auto-detect"),
156                    default="ia32,x64,arm")
157  result.add_option("--arch-and-mode",
158                    help="Architecture and mode in the format 'arch.mode'",
159                    default=None)
160  result.add_option("--asan",
161                    help="Regard test expectations for ASAN",
162                    default=False, action="store_true")
163  result.add_option("--buildbot",
164                    help="Adapt to path structure used on buildbots",
165                    default=False, action="store_true")
166  result.add_option("--dcheck-always-on",
167                    help="Indicates that V8 was compiled with DCHECKs enabled",
168                    default=False, action="store_true")
169  result.add_option("--command-prefix",
170                    help="Prepended to each shell command used to run a test",
171                    default="")
172  result.add_option("--coverage", help=("Exponential test coverage "
173                    "(range 0.0, 1.0) -- 0.0: one test, 1.0 all tests (slow)"),
174                    default=0.4, type="float")
175  result.add_option("--coverage-lift", help=("Lifts test coverage for tests "
176                    "with a small number of deopt points (range 0, inf)"),
177                    default=20, type="int")
178  result.add_option("--download-data", help="Download missing test suite data",
179                    default=False, action="store_true")
180  result.add_option("--distribution-factor1", help=("Factor of the first "
181                    "derivation of the distribution function"), default=2.0,
182                    type="float")
183  result.add_option("--distribution-factor2", help=("Factor of the second "
184                    "derivation of the distribution function"), default=0.7,
185                    type="float")
186  result.add_option("--distribution-mode", help=("How to select deopt points "
187                    "for a given test (smooth|random)"),
188                    default="smooth")
189  result.add_option("--dump-results-file", help=("Dump maximum number of "
190                    "deopt points per test to a file"))
191  result.add_option("--extra-flags",
192                    help="Additional flags to pass to each test command",
193                    default="")
194  result.add_option("--isolates", help="Whether to test isolates",
195                    default=False, action="store_true")
196  result.add_option("-j", help="The number of parallel tasks to run",
197                    default=0, type="int")
198  result.add_option("-m", "--mode",
199                    help="The test modes in which to run (comma-separated)",
200                    default="release,debug")
201  result.add_option("--outdir", help="Base directory with compile output",
202                    default="out")
203  result.add_option("-p", "--progress",
204                    help=("The style of progress indicator"
205                          " (verbose, dots, color, mono)"),
206                    choices=progress.PROGRESS_INDICATORS.keys(),
207                    default="mono")
208  result.add_option("--shard-count",
209                    help="Split testsuites into this number of shards",
210                    default=1, type="int")
211  result.add_option("--shard-run",
212                    help="Run this shard from the split up tests.",
213                    default=1, type="int")
214  result.add_option("--shell-dir", help="Directory containing executables",
215                    default="")
216  result.add_option("--seed", help="The seed for the random distribution",
217                    type="int")
218  result.add_option("-t", "--timeout", help="Timeout in seconds",
219                    default= -1, type="int")
220  result.add_option("-v", "--verbose", help="Verbose output",
221                    default=False, action="store_true")
222  result.add_option("--random-seed", default=0, dest="random_seed",
223                    help="Default seed for initializing random generator")
224  return result
225
226
227def ProcessOptions(options):
228  global VARIANT_FLAGS
229
230  # Architecture and mode related stuff.
231  if options.arch_and_mode:
232    tokens = options.arch_and_mode.split(".")
233    options.arch = tokens[0]
234    options.mode = tokens[1]
235  options.mode = options.mode.split(",")
236  for mode in options.mode:
237    if not mode.lower() in ["debug", "release"]:
238      print "Unknown mode %s" % mode
239      return False
240  if options.arch in ["auto", "native"]:
241    options.arch = ARCH_GUESS
242  options.arch = options.arch.split(",")
243  for arch in options.arch:
244    if not arch in SUPPORTED_ARCHS:
245      print "Unknown architecture %s" % arch
246      return False
247
248  # Special processing of other options, sorted alphabetically.
249  options.command_prefix = shlex.split(options.command_prefix)
250  options.extra_flags = shlex.split(options.extra_flags)
251  if options.j == 0:
252    options.j = multiprocessing.cpu_count()
253  while options.random_seed == 0:
254    options.random_seed = random.SystemRandom().randint(-2147483648, 2147483647)
255  if not options.distribution_mode in DISTRIBUTION_MODES:
256    print "Unknown distribution mode %s" % options.distribution_mode
257    return False
258  if options.distribution_factor1 < 0.0:
259    print ("Distribution factor1 %s is out of range. Defaulting to 0.0"
260        % options.distribution_factor1)
261    options.distribution_factor1 = 0.0
262  if options.distribution_factor2 < 0.0:
263    print ("Distribution factor2 %s is out of range. Defaulting to 0.0"
264        % options.distribution_factor2)
265    options.distribution_factor2 = 0.0
266  if options.coverage < 0.0 or options.coverage > 1.0:
267    print ("Coverage %s is out of range. Defaulting to 0.4"
268        % options.coverage)
269    options.coverage = 0.4
270  if options.coverage_lift < 0:
271    print ("Coverage lift %s is out of range. Defaulting to 0"
272        % options.coverage_lift)
273    options.coverage_lift = 0
274  return True
275
276
277def ShardTests(tests, shard_count, shard_run):
278  if shard_count < 2:
279    return tests
280  if shard_run < 1 or shard_run > shard_count:
281    print "shard-run not a valid number, should be in [1:shard-count]"
282    print "defaulting back to running all tests"
283    return tests
284  count = 0
285  shard = []
286  for test in tests:
287    if count % shard_count == shard_run - 1:
288      shard.append(test)
289    count += 1
290  return shard
291
292
293def Main():
294  # Use the v8 root as cwd as some test cases use "load" with relative paths.
295  os.chdir(BASE_DIR)
296
297  parser = BuildOptions()
298  (options, args) = parser.parse_args()
299  if not ProcessOptions(options):
300    parser.print_help()
301    return 1
302
303  exit_code = 0
304
305  suite_paths = utils.GetSuitePaths(join(BASE_DIR, "test"))
306
307  if len(args) == 0:
308    suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
309  else:
310    args_suites = set()
311    for arg in args:
312      suite = arg.split(os.path.sep)[0]
313      if not suite in args_suites:
314        args_suites.add(suite)
315    suite_paths = [ s for s in suite_paths if s in args_suites ]
316
317  suites = []
318  for root in suite_paths:
319    suite = testsuite.TestSuite.LoadTestSuite(
320        os.path.join(BASE_DIR, "test", root))
321    if suite:
322      suites.append(suite)
323
324  if options.download_data:
325    for s in suites:
326      s.DownloadData()
327
328  for mode in options.mode:
329    for arch in options.arch:
330      try:
331        code = Execute(arch, mode, args, options, suites, BASE_DIR)
332        exit_code = exit_code or code
333      except KeyboardInterrupt:
334        return 2
335  return exit_code
336
337
338def CalculateNTests(m, options):
339  """Calculates the number of tests from m deopt points with exponential
340  coverage.
341  The coverage is expected to be between 0.0 and 1.0.
342  The 'coverage lift' lifts the coverage for tests with smaller m values.
343  """
344  c = float(options.coverage)
345  l = float(options.coverage_lift)
346  return int(math.pow(m, (m * c + l) / (m + l)))
347
348
349def Execute(arch, mode, args, options, suites, workspace):
350  print(">>> Running tests for %s.%s" % (arch, mode))
351
352  dist = Distribution(options)
353
354  shell_dir = options.shell_dir
355  if not shell_dir:
356    if options.buildbot:
357      shell_dir = os.path.join(workspace, options.outdir, mode)
358      mode = mode.lower()
359    else:
360      shell_dir = os.path.join(workspace, options.outdir,
361                               "%s.%s" % (arch, mode))
362  shell_dir = os.path.relpath(shell_dir)
363
364  # Populate context object.
365  mode_flags = MODE_FLAGS[mode]
366  timeout = options.timeout
367  if timeout == -1:
368    # Simulators are slow, therefore allow a longer default timeout.
369    if arch in SLOW_ARCHS:
370      timeout = 2 * TIMEOUT_DEFAULT;
371    else:
372      timeout = TIMEOUT_DEFAULT;
373
374  timeout *= TIMEOUT_SCALEFACTOR[mode]
375  ctx = context.Context(arch, mode, shell_dir,
376                        mode_flags, options.verbose,
377                        timeout, options.isolates,
378                        options.command_prefix,
379                        options.extra_flags,
380                        False,  # Keep i18n on by default.
381                        options.random_seed,
382                        True,  # No sorting of test cases.
383                        0,  # Don't rerun failing tests.
384                        0,  # No use of a rerun-failing-tests maximum.
385                        False,  # No predictable mode.
386                        False,  # No no_harness mode.
387                        False,  # Don't use perf data.
388                        False)  # Coverage not supported.
389
390  # Find available test suites and read test cases from them.
391  variables = {
392    "arch": arch,
393    "asan": options.asan,
394    "deopt_fuzzer": True,
395    "gc_stress": False,
396    "gcov_coverage": False,
397    "isolates": options.isolates,
398    "mode": mode,
399    "no_i18n": False,
400    "no_snap": False,
401    "simulator": utils.UseSimulator(arch),
402    "system": utils.GuessOS(),
403    "tsan": False,
404    "msan": False,
405    "dcheck_always_on": options.dcheck_always_on,
406    "novfp3": False,
407    "predictable": False,
408    "byteorder": sys.byteorder,
409  }
410  all_tests = []
411  num_tests = 0
412  test_id = 0
413
414  # Remember test case prototypes for the fuzzing phase.
415  test_backup = dict((s, []) for s in suites)
416
417  for s in suites:
418    s.ReadStatusFile(variables)
419    s.ReadTestCases(ctx)
420    if len(args) > 0:
421      s.FilterTestCasesByArgs(args)
422    all_tests += s.tests
423    s.FilterTestCasesByStatus(False)
424    test_backup[s] = s.tests
425    analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
426                      "--print-deopt-stress"]
427    s.tests = [ t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests ]
428    num_tests += len(s.tests)
429    for t in s.tests:
430      t.id = test_id
431      test_id += 1
432
433  if num_tests == 0:
434    print "No tests to run."
435    return 0
436
437  print(">>> Collection phase")
438  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
439  runner = execution.Runner(suites, progress_indicator, ctx)
440
441  exit_code = runner.Run(options.j)
442
443  print(">>> Analysis phase")
444  num_tests = 0
445  test_id = 0
446  for s in suites:
447    test_results = {}
448    for t in s.tests:
449      for line in t.output.stdout.splitlines():
450        if line.startswith("=== Stress deopt counter: "):
451          test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
452    for t in s.tests:
453      if t.path not in test_results:
454        print "Missing results for %s" % t.path
455    if options.dump_results_file:
456      results_dict = dict((t.path, n) for (t, n) in test_results.iteritems())
457      with file("%s.%d.txt" % (dump_results_file, time.time()), "w") as f:
458        f.write(json.dumps(results_dict))
459
460    # Reset tests and redistribute the prototypes from the collection phase.
461    s.tests = []
462    if options.verbose:
463      print "Test distributions:"
464    for t in test_backup[s]:
465      max_deopt = test_results.get(t.path, 0)
466      if max_deopt == 0:
467        continue
468      n_deopt = CalculateNTests(max_deopt, options)
469      distribution = dist.Distribute(n_deopt, max_deopt)
470      if options.verbose:
471        print "%s %s" % (t.path, distribution)
472      for i in distribution:
473        fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
474        s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
475    num_tests += len(s.tests)
476    for t in s.tests:
477      t.id = test_id
478      test_id += 1
479
480  if num_tests == 0:
481    print "No tests to run."
482    return 0
483
484  print(">>> Deopt fuzzing phase (%d test cases)" % num_tests)
485  progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
486  runner = execution.Runner(suites, progress_indicator, ctx)
487
488  code = runner.Run(options.j)
489  return exit_code or code
490
491
492if __name__ == "__main__":
493  sys.exit(Main())
494