1#!/usr/bin/env python
2# Copyright 2016 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""
7V8 correctness fuzzer launcher script.
8"""
9
10import argparse
11import hashlib
12import itertools
13import json
14import os
15import random
16import re
17import sys
18import traceback
19
20import v8_commands
21import v8_suppressions
22
23CONFIGS = dict(
24  default=[],
25  ignition=[
26    '--turbo-filter=~',
27    '--noopt',
28    '--liftoff',
29    '--no-wasm-tier-up',
30  ],
31  ignition_asm=[
32    '--turbo-filter=~',
33    '--noopt',
34    '--validate-asm',
35    '--stress-validate-asm',
36  ],
37  ignition_eager=[
38    '--turbo-filter=~',
39    '--noopt',
40    '--no-lazy',
41    '--no-lazy-inner-functions',
42  ],
43  ignition_turbo=[],
44  ignition_turbo_opt=[
45    '--always-opt',
46    '--no-liftoff',
47    '--no-wasm-tier-up',
48  ],
49  ignition_turbo_opt_eager=[
50    '--always-opt',
51    '--no-lazy',
52    '--no-lazy-inner-functions',
53  ],
54  slow_path=[
55    '--force-slow-path',
56  ],
57  slow_path_opt=[
58    '--always-opt',
59    '--force-slow-path',
60  ],
61  trusted=[
62    '--no-untrusted-code-mitigations',
63  ],
64  trusted_opt=[
65    '--always-opt',
66    '--no-untrusted-code-mitigations',
67  ],
68)
69
70# Additional flag experiments. List of tuples like
71# (<likelihood to use flags in [0,1)>, <flag>).
72ADDITIONAL_FLAGS = [
73  (0.1, '--stress-marking=100'),
74  (0.1, '--stress-scavenge=100'),
75  (0.1, '--stress-compaction-random'),
76  (0.1, '--random-gc-interval=2000'),
77  (0.2, '--noanalyze-environment-liveness'),
78]
79
80# Timeout in seconds for one d8 run.
81TIMEOUT = 3
82
83# Return codes.
84RETURN_PASS = 0
85RETURN_FAIL = 2
86
87BASE_PATH = os.path.dirname(os.path.abspath(__file__))
88PREAMBLE = [
89  os.path.join(BASE_PATH, 'v8_mock.js'),
90  os.path.join(BASE_PATH, 'v8_suppressions.js'),
91]
92ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
93
94FLAGS = ['--abort_on_stack_or_string_length_overflow', '--expose-gc',
95         '--allow-natives-syntax', '--invoke-weak-callbacks', '--omit-quit',
96         '--es-staging', '--wasm-num-compilation-tasks=0',
97         '--suppress-asm-messages']
98
99SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']
100
101# Output for suppressed failure case.
102FAILURE_HEADER_TEMPLATE = """#
103# V8 correctness failure
104# V8 correctness configs: %(configs)s
105# V8 correctness sources: %(source_key)s
106# V8 correctness suppression: %(suppression)s
107"""
108
109# Extended output for failure case. The 'CHECK' is for the minimizer.
110FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
111# CHECK
112#
113# Compared %(first_config_label)s with %(second_config_label)s
114#
115# Flags of %(first_config_label)s:
116%(first_config_flags)s
117# Flags of %(second_config_label)s:
118%(second_config_flags)s
119#
120# Difference:
121%(difference)s
122#
123# Source file:
124%(source)s
125#
126### Start of configuration %(first_config_label)s:
127%(first_config_output)s
128### End of configuration %(first_config_label)s
129#
130### Start of configuration %(second_config_label)s:
131%(second_config_output)s
132### End of configuration %(second_config_label)s
133"""
134
135FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)')
136SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);')
137
138# The number of hex digits used from the hash of the original source file path.
139# Keep the number small to avoid duplicate explosion.
140ORIGINAL_SOURCE_HASH_LENGTH = 3
141
142# Placeholder string if no original source file could be determined.
143ORIGINAL_SOURCE_DEFAULT = 'none'
144
145
146def infer_arch(d8):
147  """Infer the V8 architecture from the build configuration next to the
148  executable.
149  """
150  with open(os.path.join(os.path.dirname(d8), 'v8_build_config.json')) as f:
151    arch = json.load(f)['v8_current_cpu']
152  return 'ia32' if arch == 'x86' else arch
153
154
155def parse_args():
156  parser = argparse.ArgumentParser()
157  parser.add_argument(
158    '--random-seed', type=int, required=True,
159    help='random seed passed to both runs')
160  parser.add_argument(
161      '--first-config', help='first configuration', default='ignition')
162  parser.add_argument(
163      '--second-config', help='second configuration', default='ignition_turbo')
164  parser.add_argument(
165      '--first-d8', default='d8',
166      help='optional path to first d8 executable, '
167           'default: bundled in the same directory as this script')
168  parser.add_argument(
169      '--second-d8',
170      help='optional path to second d8 executable, default: same as first')
171  parser.add_argument('testcase', help='path to test case')
172  options = parser.parse_args()
173
174  # Ensure we have a test case.
175  assert (os.path.exists(options.testcase) and
176          os.path.isfile(options.testcase)), (
177      'Test case %s doesn\'t exist' % options.testcase)
178
179  # Use first d8 as default for second d8.
180  options.second_d8 = options.second_d8 or options.first_d8
181
182  # Ensure absolute paths.
183  if not os.path.isabs(options.first_d8):
184    options.first_d8 = os.path.join(BASE_PATH, options.first_d8)
185  if not os.path.isabs(options.second_d8):
186    options.second_d8 = os.path.join(BASE_PATH, options.second_d8)
187
188  # Ensure executables exist.
189  assert os.path.exists(options.first_d8)
190  assert os.path.exists(options.second_d8)
191
192  # Infer architecture from build artifacts.
193  options.first_arch = infer_arch(options.first_d8)
194  options.second_arch = infer_arch(options.second_d8)
195
196  # Ensure we make a sane comparison.
197  if (options.first_arch == options.second_arch and
198      options.first_config == options.second_config):
199    parser.error('Need either arch or config difference.')
200  assert options.first_arch in SUPPORTED_ARCHS
201  assert options.second_arch in SUPPORTED_ARCHS
202  assert options.first_config in CONFIGS
203  assert options.second_config in CONFIGS
204
205  return options
206
207
208def get_meta_data(content):
209  """Extracts original-source-file paths from test case content."""
210  sources = []
211  for line in content.splitlines():
212    match = SOURCE_RE.match(line)
213    if match:
214      sources.append(match.group(1))
215  return {'sources': sources}
216
217
218def content_bailout(content, ignore_fun):
219  """Print failure state and return if ignore_fun matches content."""
220  bug = (ignore_fun(content) or '').strip()
221  if bug:
222    print FAILURE_HEADER_TEMPLATE % dict(
223        configs='', source_key='', suppression=bug)
224    return True
225  return False
226
227
228def pass_bailout(output, step_number):
229  """Print info and return if in timeout or crash pass states."""
230  if output.HasTimedOut():
231    # Dashed output, so that no other clusterfuzz tools can match the
232    # words timeout or crash.
233    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
234    return True
235  if output.HasCrashed():
236    print '# V8 correctness - C-R-A-S-H %d' % step_number
237    return True
238  return False
239
240
241def fail_bailout(output, ignore_by_output_fun):
242  """Print failure state and return if ignore_by_output_fun matches output."""
243  bug = (ignore_by_output_fun(output.stdout) or '').strip()
244  if bug:
245    print FAILURE_HEADER_TEMPLATE % dict(
246        configs='', source_key='', suppression=bug)
247    return True
248  return False
249
250
251def main():
252  options = parse_args()
253  rng = random.Random(options.random_seed)
254
255  # Suppressions are architecture and configuration specific.
256  suppress = v8_suppressions.get_suppression(
257      options.first_arch, options.first_config,
258      options.second_arch, options.second_config,
259  )
260
261  # Static bailout based on test case content or metadata.
262  with open(options.testcase) as f:
263    content = f.read()
264  if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
265    return RETURN_FAIL
266  if content_bailout(content, suppress.ignore_by_content):
267    return RETURN_FAIL
268
269  # Set up runtime arguments.
270  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
271  first_config_flags = common_flags + CONFIGS[options.first_config]
272  second_config_flags = common_flags + CONFIGS[options.second_config]
273
274  # Add additional flags to second config based on experiment percentages.
275  for p, flag in ADDITIONAL_FLAGS:
276    if rng.random() < p:
277      second_config_flags.append(flag)
278
279  def run_d8(d8, config_flags):
280    preamble = PREAMBLE[:]
281    if options.first_arch != options.second_arch:
282      preamble.append(ARCH_MOCKS)
283    args = [d8] + config_flags + preamble + [options.testcase]
284    print " ".join(args)
285    if d8.endswith('.py'):
286      # Wrap with python in tests.
287      args = [sys.executable] + args
288    return v8_commands.Execute(
289        args,
290        cwd=os.path.dirname(os.path.abspath(options.testcase)),
291        timeout=TIMEOUT,
292    )
293
294  first_config_output = run_d8(options.first_d8, first_config_flags)
295
296  # Early bailout based on first run's output.
297  if pass_bailout(first_config_output, 1):
298    return RETURN_PASS
299
300  second_config_output = run_d8(options.second_d8, second_config_flags)
301
302  # Bailout based on second run's output.
303  if pass_bailout(second_config_output, 2):
304    return RETURN_PASS
305
306  difference, source = suppress.diff(
307      first_config_output.stdout, second_config_output.stdout)
308
309  if source:
310    source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
311  else:
312    source = ORIGINAL_SOURCE_DEFAULT
313    source_key = ORIGINAL_SOURCE_DEFAULT
314
315  if difference:
316    # Only bail out due to suppressed output if there was a difference. If a
317    # suppression doesn't show up anymore in the statistics, we might want to
318    # remove it.
319    if fail_bailout(first_config_output, suppress.ignore_by_output1):
320      return RETURN_FAIL
321    if fail_bailout(second_config_output, suppress.ignore_by_output2):
322      return RETURN_FAIL
323
324    # The first three entries will be parsed by clusterfuzz. Format changes
325    # will require changes on the clusterfuzz side.
326    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
327    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
328    print (FAILURE_TEMPLATE % dict(
329        configs='%s:%s' % (first_config_label, second_config_label),
330        source_key=source_key,
331        suppression='', # We can't tie bugs to differences.
332        first_config_label=first_config_label,
333        second_config_label=second_config_label,
334        first_config_flags=' '.join(first_config_flags),
335        second_config_flags=' '.join(second_config_flags),
336        first_config_output=
337            first_config_output.stdout.decode('utf-8', 'replace'),
338        second_config_output=
339            second_config_output.stdout.decode('utf-8', 'replace'),
340        source=source,
341        difference=difference.decode('utf-8', 'replace'),
342    )).encode('utf-8', 'replace')
343    return RETURN_FAIL
344
345  # TODO(machenbach): Figure out if we could also return a bug in case there's
346  # no difference, but one of the line suppressions has matched - and without
347  # the match there would be a difference.
348
349  print '# V8 correctness - pass'
350  return RETURN_PASS
351
352
353if __name__ == "__main__":
354  try:
355    result = main()
356  except SystemExit:
357    # Make sure clusterfuzz reports internal errors and wrong usage.
358    # Use one label for all internal and usage errors.
359    print FAILURE_HEADER_TEMPLATE % dict(
360        configs='', source_key='', suppression='wrong_usage')
361    result = RETURN_FAIL
362  except MemoryError:
363    # Running out of memory happens occasionally but is not actionable.
364    print '# V8 correctness - pass'
365    result = RETURN_PASS
366  except Exception as e:
367    print FAILURE_HEADER_TEMPLATE % dict(
368        configs='', source_key='', suppression='internal_error')
369    print '# Internal error: %s' % e
370    traceback.print_exc(file=sys.stdout)
371    result = RETURN_FAIL
372
373  sys.exit(result)
374