1# Copyright 2016 the V8 project authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Suppressions for V8 correctness fuzzer failures.
7
8We support three types of suppressions:
91. Ignore test case by pattern.
10Map a regular expression to a bug entry. A new failure will be reported
11when the pattern matches a JS test case.
12Subsequent matches will be recoreded under the first failure.
13
142. Ignore test run by output pattern:
15Map a regular expression to a bug entry. A new failure will be reported
16when the pattern matches the output of a particular run.
17Subsequent matches will be recoreded under the first failure.
18
193. Relax line-to-line comparisons with expressions of lines to ignore and
20lines to be normalized (i.e. ignore only portions of lines).
21These are not tied to bugs, be careful to not silently switch off this tool!
22
23Alternatively, think about adding a behavior change to v8_suppressions.js
24to silence a particular class of problems.
25"""
26
27import itertools
28import re
29
30# Max line length for regular experessions checking for lines to ignore.
31MAX_LINE_LENGTH = 512
32
33# For ignoring lines before carets and to ignore caret positions.
34CARET_RE = re.compile(r'^\s*\^\s*$')
35
36# Ignore by original source files. Map from bug->list of relative file paths in
37# V8, e.g. '/v8/test/mjsunit/d8-performance-now.js' including /v8/. A test will
38# be suppressed if one of the files below was used to mutate the test.
39IGNORE_SOURCES = {
40  # This contains a usage of f.arguments that often fires.
41  'crbug.com/662424': [
42    '/v8/test/mjsunit/bugs/bug-222.js',
43    '/v8/test/mjsunit/bugs/bug-941049.js',
44    '/v8/test/mjsunit/regress/regress-crbug-668795.js',
45    '/v8/test/mjsunit/regress/regress-1079.js',
46    '/v8/test/mjsunit/regress/regress-2989.js',
47  ],
48
49  'crbug.com/688159': [
50    '/v8/test/mjsunit/es7/exponentiation-operator.js',
51  ],
52
53  # TODO(machenbach): Implement blacklisting files for particular configs only,
54  # here ignition_eager.
55  'crbug.com/691589': [
56    '/v8/test/mjsunit/regress/regress-1200351.js',
57  ],
58
59  'crbug.com/691587': [
60    '/v8/test/mjsunit/asm/regress-674089.js',
61  ],
62
63  'crbug.com/774805': [
64    '/v8/test/mjsunit/console.js',
65  ],
66}
67
68# Ignore by test case pattern. Map from config->bug->regexp. Config '' is used
69# to match all configurations. Otherwise use either a compiler configuration,
70# e.g. ignition or validate_asm or an architecture, e.g. x64 or ia32.
71# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
72# label.
73# Regular expressions are assumed to be compiled. We use regexp.search.
74IGNORE_TEST_CASES = {
75}
76
77# Ignore by output pattern. Map from config->bug->regexp. See IGNORE_TEST_CASES
78# on how to specify config keys.
79# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
80# label.
81# Regular expressions are assumed to be compiled. We use regexp.search.
82IGNORE_OUTPUT = {
83  '': {
84    'crbug.com/664068':
85        re.compile(r'RangeError(?!: byte length)', re.S),
86    'crbug.com/667678':
87        re.compile(r'\[native code\]', re.S),
88    'crbug.com/689877':
89        re.compile(r'^.*SyntaxError: .*Stack overflow$', re.M),
90  },
91}
92
93# Lines matching any of the following regular expressions will be ignored
94# if appearing on both sides. The capturing groups need to match exactly.
95# Use uncompiled regular expressions - they'll be compiled later.
96ALLOWED_LINE_DIFFS = [
97  # Ignore caret position in stack traces.
98  r'^\s*\^\s*$',
99
100  # Ignore some stack trace headers as messages might not match.
101  r'^(.*)TypeError: .* is not a function$',
102  r'^(.*)TypeError: .* is not a constructor$',
103  r'^(.*)TypeError: (.*) is not .*$',
104  r'^(.*)ReferenceError: .* is not defined$',
105  r'^(.*):\d+: ReferenceError: .* is not defined$',
106
107  # These are rarely needed. It includes some cases above.
108  r'^\w*Error: .* is not .*$',
109  r'^(.*) \w*Error: .* is not .*$',
110  r'^(.*):\d+: \w*Error: .* is not .*$',
111
112  # Some test cases just print the message.
113  r'^.* is not a function(.*)$',
114  r'^(.*) is not a .*$',
115
116  # crbug.com/680064. This subsumes one of the above expressions.
117  r'^(.*)TypeError: .* function$',
118
119  # crbug.com/664068
120  r'^(.*)(?:Array buffer allocation failed|Invalid array buffer length)(.*)$',
121]
122
123# Lines matching any of the following regular expressions will be ignored.
124# Use uncompiled regular expressions - they'll be compiled later.
125IGNORE_LINES = [
126  r'^Warning: unknown flag .*$',
127  r'^Warning: .+ is deprecated.*$',
128  r'^Try --help for options$',
129
130  # crbug.com/705962
131  r'^\s\[0x[0-9a-f]+\]$',
132]
133
134
135###############################################################################
136# Implementation - you should not need to change anything below this point.
137
138# Compile regular expressions.
139ALLOWED_LINE_DIFFS = [re.compile(exp) for exp in ALLOWED_LINE_DIFFS]
140IGNORE_LINES = [re.compile(exp) for exp in IGNORE_LINES]
141
142ORIGINAL_SOURCE_PREFIX = 'v8-foozzie source: '
143
144def line_pairs(lines):
145  return itertools.izip_longest(
146      lines, itertools.islice(lines, 1, None), fillvalue=None)
147
148
149def caret_match(line1, line2):
150  if (not line1 or
151      not line2 or
152      len(line1) > MAX_LINE_LENGTH or
153      len(line2) > MAX_LINE_LENGTH):
154    return False
155  return bool(CARET_RE.match(line1) and CARET_RE.match(line2))
156
157
158def short_line_output(line):
159  if len(line) <= MAX_LINE_LENGTH:
160    # Avoid copying.
161    return line
162  return line[0:MAX_LINE_LENGTH] + '...'
163
164
165def ignore_by_regexp(line1, line2, allowed):
166  if len(line1) > MAX_LINE_LENGTH or len(line2) > MAX_LINE_LENGTH:
167    return False
168  for exp in allowed:
169    match1 = exp.match(line1)
170    match2 = exp.match(line2)
171    if match1 and match2:
172      # If there are groups in the regexp, ensure the groups matched the same
173      # things.
174      if match1.groups() == match2.groups():  # tuple comparison
175        return True
176  return False
177
178
179def diff_output(output1, output2, allowed, ignore1, ignore2):
180  """Returns a tuple (difference, source).
181
182  The difference is None if there's no difference, otherwise a string
183  with a readable diff.
184
185  The source is the last source output within the test case, or None if no
186  such output existed.
187  """
188  def useful_line(ignore):
189    def fun(line):
190      return all(not e.match(line) for e in ignore)
191    return fun
192
193  lines1 = filter(useful_line(ignore1), output1)
194  lines2 = filter(useful_line(ignore2), output2)
195
196  # This keeps track where we are in the original source file of the fuzz
197  # test case.
198  source = None
199
200  for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest(
201      line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):
202
203    # Only one of the two iterators should run out.
204    assert not (line1 is None and line2 is None)
205
206    # One iterator ends earlier.
207    if line1 is None:
208      return '+ %s' % short_line_output(line2), source
209    if line2 is None:
210      return '- %s' % short_line_output(line1), source
211
212    # If lines are equal, no further checks are necessary.
213    if line1 == line2:
214      # Instrumented original-source-file output must be equal in both
215      # versions. It only makes sense to update it here when both lines
216      # are equal.
217      if line1.startswith(ORIGINAL_SOURCE_PREFIX):
218        source = line1[len(ORIGINAL_SOURCE_PREFIX):]
219      continue
220
221    # Look ahead. If next line is a caret, ignore this line.
222    if caret_match(lookahead1, lookahead2):
223      continue
224
225    # Check if a regexp allows these lines to be different.
226    if ignore_by_regexp(line1, line2, allowed):
227      continue
228
229    # Lines are different.
230    return (
231        '- %s\n+ %s' % (short_line_output(line1), short_line_output(line2)),
232        source,
233    )
234
235  # No difference found.
236  return None, source
237
238
239def get_suppression(arch1, config1, arch2, config2):
240  return V8Suppression(arch1, config1, arch2, config2)
241
242
243class Suppression(object):
244  def diff(self, output1, output2):
245    return None
246
247  def ignore_by_metadata(self, metadata):
248    return None
249
250  def ignore_by_content(self, testcase):
251    return None
252
253  def ignore_by_output1(self, output):
254    return None
255
256  def ignore_by_output2(self, output):
257    return None
258
259
260class V8Suppression(Suppression):
261  def __init__(self, arch1, config1, arch2, config2):
262    self.arch1 = arch1
263    self.config1 = config1
264    self.arch2 = arch2
265    self.config2 = config2
266
267  def diff(self, output1, output2):
268    return diff_output(
269        output1.splitlines(),
270        output2.splitlines(),
271        ALLOWED_LINE_DIFFS,
272        IGNORE_LINES,
273        IGNORE_LINES,
274    )
275
276  def ignore_by_content(self, testcase):
277    # Strip off test case preamble.
278    try:
279      lines = testcase.splitlines()
280      lines = lines[lines.index(
281          'print("js-mutation: start generated test case");'):]
282      content = '\n'.join(lines)
283    except ValueError:
284      # Search the whole test case if preamble can't be found. E.g. older
285      # already minimized test cases might have dropped the delimiter line.
286      content = testcase
287    for key in ['', self.arch1, self.arch2, self.config1, self.config2]:
288      for bug, exp in IGNORE_TEST_CASES.get(key, {}).iteritems():
289        if exp.search(content):
290          return bug
291    return None
292
293  def ignore_by_metadata(self, metadata):
294    for bug, sources in IGNORE_SOURCES.iteritems():
295      for source in sources:
296        if source in metadata['sources']:
297          return bug
298    return None
299
300  def ignore_by_output1(self, output):
301    return self.ignore_by_output(output, self.arch1, self.config1)
302
303  def ignore_by_output2(self, output):
304    return self.ignore_by_output(output, self.arch2, self.config2)
305
306  def ignore_by_output(self, output, arch, config):
307    def check(mapping):
308      for bug, exp in mapping.iteritems():
309        if exp.search(output):
310          return bug
311      return None
312    for key in ['', arch, config]:
313      bug = check(IGNORE_OUTPUT.get(key, {}))
314      if bug:
315        return bug
316    return None
317