1#!/usr/bin/env python
2#
3# Copyright 2017 the V8 project authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7
8import random
9import sys
10
11# Adds testrunner to the path hence it has to be imported at the beggining.
12import base_runner
13
14from testrunner.local import utils
15
16from testrunner.testproc import fuzzer
17from testrunner.testproc.base import TestProcProducer
18from testrunner.testproc.combiner import CombinerProc
19from testrunner.testproc.execution import ExecutionProc
20from testrunner.testproc.expectation import ForgiveTimeoutProc
21from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
22from testrunner.testproc.loader import LoadProc
23from testrunner.testproc.progress import ResultsTracker, TestsCounter
24from testrunner.utils import random_utils
25
26
27DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]
28
29
30class NumFuzzer(base_runner.BaseTestRunner):
31  def __init__(self, *args, **kwargs):
32    super(NumFuzzer, self).__init__(*args, **kwargs)
33
34  def _add_parser_options(self, parser):
35    parser.add_option("--fuzzer-random-seed", default=0,
36                      help="Default seed for initializing fuzzer random "
37                      "generator")
38    parser.add_option("--tests-count", default=5, type="int",
39                      help="Number of tests to generate from each base test. "
40                           "Can be combined with --total-timeout-sec with "
41                           "value 0 to provide infinite number of subtests. "
42                           "When --combine-tests is set it indicates how many "
43                           "tests to create in total")
44
45    # Stress gc
46    parser.add_option("--stress-marking", default=0, type="int",
47                      help="probability [0-10] of adding --stress-marking "
48                           "flag to the test")
49    parser.add_option("--stress-scavenge", default=0, type="int",
50                      help="probability [0-10] of adding --stress-scavenge "
51                           "flag to the test")
52    parser.add_option("--stress-compaction", default=0, type="int",
53                      help="probability [0-10] of adding --stress-compaction "
54                           "flag to the test")
55    parser.add_option("--stress-gc", default=0, type="int",
56                      help="probability [0-10] of adding --random-gc-interval "
57                           "flag to the test")
58    parser.add_option("--stress-thread-pool-size", default=0, type="int",
59                      help="probability [0-10] of adding --thread-pool-size "
60                           "flag to the test")
61
62    # Stress deopt
63    parser.add_option("--stress-deopt", default=0, type="int",
64                      help="probability [0-10] of adding --deopt-every-n-times "
65                           "flag to the test")
66    parser.add_option("--stress-deopt-min", default=1, type="int",
67                      help="extends --stress-deopt to have minimum interval "
68                           "between deopt points")
69
70    # Stress interrupt budget
71    parser.add_option("--stress-interrupt-budget", default=0, type="int",
72                      help="probability [0-10] of adding --interrupt-budget "
73                           "flag to the test")
74
75    # Combine multiple tests
76    parser.add_option("--combine-tests", default=False, action="store_true",
77                      help="Combine multiple tests as one and run with "
78                           "try-catch wrapper")
79    parser.add_option("--combine-max", default=100, type="int",
80                      help="Maximum number of tests to combine")
81    parser.add_option("--combine-min", default=2, type="int",
82                      help="Minimum number of tests to combine")
83
84    # Miscellaneous
85    parser.add_option("--variants", default='default',
86                      help="Comma-separated list of testing variants")
87
88    return parser
89
90
91  def _process_options(self, options):
92    if not options.fuzzer_random_seed:
93      options.fuzzer_random_seed = random_utils.random_seed()
94
95    if options.total_timeout_sec:
96      options.tests_count = 0
97
98    if options.combine_tests:
99      if options.combine_min > options.combine_max:
100        print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
101               options.min_group_size, options.max_group_size)
102        raise base_runner.TestRunnerError()
103
104    if options.variants != 'default':
105      print ('Only default testing variant is supported with numfuzz')
106      raise base_runner.TestRunnerError()
107
108    return True
109
110  def _get_default_suite_names(self):
111    return DEFAULT_SUITES
112
113  def _timeout_scalefactor(self, options):
114    factor = super(NumFuzzer, self)._timeout_scalefactor(options)
115    if options.stress_interrupt_budget:
116      # TODO(machenbach): This should be moved to a more generic config.
117      # Fuzzers have too much timeout in debug mode.
118      factor = max(int(factor * 0.25), 1)
119    return factor
120
121  def _get_statusfile_variables(self, options):
122    variables = (
123        super(NumFuzzer, self)._get_statusfile_variables(options))
124    variables.update({
125      'deopt_fuzzer': bool(options.stress_deopt),
126      'endurance_fuzzer': bool(options.combine_tests),
127      'gc_stress': bool(options.stress_gc),
128      'gc_fuzzer': bool(max([options.stress_marking,
129                             options.stress_scavenge,
130                             options.stress_compaction,
131                             options.stress_gc,
132                             options.stress_thread_pool_size])),
133    })
134    return variables
135
136  def _do_execute(self, tests, args, options):
137    loader = LoadProc()
138    fuzzer_rng = random.Random(options.fuzzer_random_seed)
139
140    combiner = self._create_combiner(fuzzer_rng, options)
141    results = ResultsTracker()
142    execproc = ExecutionProc(options.j)
143    sigproc = self._create_signal_proc()
144    indicators = self._create_progress_indicators(options)
145    procs = [
146      loader,
147      NameFilterProc(args) if args else None,
148      StatusFileFilterProc(None, None),
149      # TODO(majeski): Improve sharding when combiner is present. Maybe select
150      # different random seeds for shards instead of splitting tests.
151      self._create_shard_proc(options),
152      ForgiveTimeoutProc(),
153      combiner,
154      self._create_fuzzer(fuzzer_rng, options),
155      sigproc,
156    ] + indicators + [
157      results,
158      self._create_timeout_proc(options),
159      self._create_rerun_proc(options),
160      execproc,
161    ]
162    self._prepare_procs(procs)
163    loader.load_tests(tests)
164
165    # TODO(majeski): maybe some notification from loader would be better?
166    if combiner:
167      combiner.generate_initial_tests(options.j * 4)
168
169    # This starts up worker processes and blocks until all tests are
170    # processed.
171    execproc.run()
172
173    for indicator in indicators:
174      indicator.finished()
175
176    print '>>> %d tests ran' % results.total
177    if results.failed:
178      return utils.EXIT_CODE_FAILURES
179
180    # Indicate if a SIGINT or SIGTERM happened.
181    return sigproc.exit_code
182
183  def _load_suites(self, names, options):
184    suites = super(NumFuzzer, self)._load_suites(names, options)
185    if options.combine_tests:
186      suites = [s for s in suites if s.test_combiner_available()]
187    if options.stress_interrupt_budget:
188      # Changing interrupt budget forces us to suppress certain test assertions.
189      for suite in suites:
190        suite.do_suppress_internals()
191    return suites
192
193  def _create_combiner(self, rng, options):
194    if not options.combine_tests:
195      return None
196    return CombinerProc(rng, options.combine_min, options.combine_max,
197                        options.tests_count)
198
199  def _create_fuzzer(self, rng, options):
200    return fuzzer.FuzzerProc(
201        rng,
202        self._tests_count(options),
203        self._create_fuzzer_configs(options),
204        self._disable_analysis(options),
205    )
206
207  def _tests_count(self, options):
208    if options.combine_tests:
209      return 1
210    return options.tests_count
211
212  def _disable_analysis(self, options):
213    """Disable analysis phase when options are used that don't support it."""
214    return options.combine_tests or options.stress_interrupt_budget
215
216  def _create_fuzzer_configs(self, options):
217    fuzzers = []
218    def add(name, prob, *args):
219      if prob:
220        fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))
221
222    add('compaction', options.stress_compaction)
223    add('marking', options.stress_marking)
224    add('scavenge', options.stress_scavenge)
225    add('gc_interval', options.stress_gc)
226    add('threads', options.stress_thread_pool_size)
227    add('interrupt_budget', options.stress_interrupt_budget)
228    add('deopt', options.stress_deopt, options.stress_deopt_min)
229    return fuzzers
230
231
232if __name__ == '__main__':
233  sys.exit(NumFuzzer().execute())
234