1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import imp
30import os
31
32from . import commands
33from . import statusfile
34from . import utils
35from ..objects import testcase
36
37# Use this to run several variants of the tests.
38ALL_VARIANT_FLAGS = {
39  "default": [[]],
40  "stress": [["--stress-opt", "--always-opt"]],
41  "turbofan": [["--turbo"]],
42  "turbofan_opt": [["--turbo", "--always-opt"]],
43  "nocrankshaft": [["--nocrankshaft"]],
44  "ignition": [["--ignition", "--turbo", "--ignition-fake-try-catch",
45                "--ignition-fallback-on-eval-and-catch"]],
46  "preparser": [["--min-preparse-length=0"]],
47}
48
49# FAST_VARIANTS implies no --always-opt.
50FAST_VARIANT_FLAGS = {
51  "default": [[]],
52  "stress": [["--stress-opt"]],
53  "turbofan": [["--turbo"]],
54  "nocrankshaft": [["--nocrankshaft"]],
55  "ignition": [["--ignition", "--turbo", "--ignition-fake-try-catch",
56                "--ignition-fallback-on-eval-and-catch"]],
57  "preparser": [["--min-preparse-length=0"]],
58}
59
60ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
61                    "nocrankshaft", "ignition", "preparser"])
62FAST_VARIANTS = set(["default", "turbofan"])
63STANDARD_VARIANT = set(["default"])
64
65
66class VariantGenerator(object):
67  def __init__(self, suite, variants):
68    self.suite = suite
69    self.all_variants = ALL_VARIANTS & variants
70    self.fast_variants = FAST_VARIANTS & variants
71    self.standard_variant = STANDARD_VARIANT & variants
72
73  def FilterVariantsByTest(self, testcase):
74    if testcase.outcomes and statusfile.OnlyStandardVariant(
75        testcase.outcomes):
76      return self.standard_variant
77    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
78      return self.fast_variants
79    return self.all_variants
80
81  def GetFlagSets(self, testcase, variant):
82    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
83      return FAST_VARIANT_FLAGS[variant]
84    else:
85      return ALL_VARIANT_FLAGS[variant]
86
87
88class TestSuite(object):
89
90  @staticmethod
91  def LoadTestSuite(root, global_init=True):
92    name = root.split(os.path.sep)[-1]
93    f = None
94    try:
95      (f, pathname, description) = imp.find_module("testcfg", [root])
96      module = imp.load_module("testcfg", f, pathname, description)
97      return module.GetSuite(name, root)
98    except ImportError:
99      # Use default if no testcfg is present.
100      return GoogleTestSuite(name, root)
101    finally:
102      if f:
103        f.close()
104
105  def __init__(self, name, root):
106    # Note: This might be called concurrently from different processes.
107    # Changing harddisk state should be done in 'SetupWorkingDirectory' below.
108    self.name = name  # string
109    self.root = root  # string containing path
110    self.tests = None  # list of TestCase objects
111    self.rules = None  # dictionary mapping test path to list of outcomes
112    self.wildcards = None  # dictionary mapping test paths to list of outcomes
113    self.total_duration = None  # float, assigned on demand
114
115  def SetupWorkingDirectory(self):
116    # This is called once per test suite object in a multi-process setting.
117    # Multi-process-unsafe work-directory setup can go here.
118    pass
119
120  def shell(self):
121    return "d8"
122
123  def suffix(self):
124    return ".js"
125
126  def status_file(self):
127    return "%s/%s.status" % (self.root, self.name)
128
129  # Used in the status file and for stdout printing.
130  def CommonTestName(self, testcase):
131    if utils.IsWindows():
132      return testcase.path.replace("\\", "/")
133    else:
134      return testcase.path
135
136  def ListTests(self, context):
137    raise NotImplementedError
138
139  def _VariantGeneratorFactory(self):
140    """The variant generator class to be used."""
141    return VariantGenerator
142
143  def CreateVariantGenerator(self, variants):
144    """Return a generator for the testing variants of this suite.
145
146    Args:
147      variants: List of variant names to be run as specified by the test
148                runner.
149    Returns: An object of type VariantGenerator.
150    """
151    return self._VariantGeneratorFactory()(self, set(variants))
152
153  def DownloadData(self):
154    pass
155
156  def ReadStatusFile(self, variables):
157    (self.rules, self.wildcards) = \
158        statusfile.ReadStatusFile(self.status_file(), variables)
159
160  def ReadTestCases(self, context):
161    self.tests = self.ListTests(context)
162
163  @staticmethod
164  def _FilterFlaky(flaky, mode):
165    return (mode == "run" and not flaky) or (mode == "skip" and flaky)
166
167  @staticmethod
168  def _FilterSlow(slow, mode):
169    return (mode == "run" and not slow) or (mode == "skip" and slow)
170
171  @staticmethod
172  def _FilterPassFail(pass_fail, mode):
173    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)
174
175  def FilterTestCasesByStatus(self, warn_unused_rules,
176                              flaky_tests="dontcare",
177                              slow_tests="dontcare",
178                              pass_fail_tests="dontcare"):
179    filtered = []
180    used_rules = set()
181    for t in self.tests:
182      flaky = False
183      slow = False
184      pass_fail = False
185      testname = self.CommonTestName(t)
186      if testname in self.rules:
187        used_rules.add(testname)
188        # Even for skipped tests, as the TestCase object stays around and
189        # PrintReport() uses it.
190        t.outcomes = self.rules[testname]
191        if statusfile.DoSkip(t.outcomes):
192          continue  # Don't add skipped tests to |filtered|.
193        for outcome in t.outcomes:
194          if outcome.startswith('Flags: '):
195            t.flags += outcome[7:].split()
196        flaky = statusfile.IsFlaky(t.outcomes)
197        slow = statusfile.IsSlow(t.outcomes)
198        pass_fail = statusfile.IsPassOrFail(t.outcomes)
199      skip = False
200      for rule in self.wildcards:
201        assert rule[-1] == '*'
202        if testname.startswith(rule[:-1]):
203          used_rules.add(rule)
204          t.outcomes |= self.wildcards[rule]
205          if statusfile.DoSkip(t.outcomes):
206            skip = True
207            break  # "for rule in self.wildcards"
208          flaky = flaky or statusfile.IsFlaky(t.outcomes)
209          slow = slow or statusfile.IsSlow(t.outcomes)
210          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
211      if (skip or self._FilterFlaky(flaky, flaky_tests)
212          or self._FilterSlow(slow, slow_tests)
213          or self._FilterPassFail(pass_fail, pass_fail_tests)):
214        continue  # "for t in self.tests"
215      filtered.append(t)
216    self.tests = filtered
217
218    if not warn_unused_rules:
219      return
220
221    for rule in self.rules:
222      if rule not in used_rules:
223        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
224    for rule in self.wildcards:
225      if rule not in used_rules:
226        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
227
228  def FilterTestCasesByArgs(self, args):
229    """Filter test cases based on command-line arguments.
230
231    An argument with an asterisk in the end will match all test cases
232    that have the argument as a prefix. Without asterisk, only exact matches
233    will be used with the exeption of the test-suite name as argument.
234    """
235    filtered = []
236    globs = []
237    exact_matches = []
238    for a in args:
239      argpath = a.split('/')
240      if argpath[0] != self.name:
241        continue
242      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
243        return  # Don't filter, run all tests in this suite.
244      path = '/'.join(argpath[1:])
245      if path[-1] == '*':
246        path = path[:-1]
247        globs.append(path)
248      else:
249        exact_matches.append(path)
250    for t in self.tests:
251      for a in globs:
252        if t.path.startswith(a):
253          filtered.append(t)
254          break
255      for a in exact_matches:
256        if t.path == a:
257          filtered.append(t)
258          break
259    self.tests = filtered
260
261  def GetFlagsForTestCase(self, testcase, context):
262    raise NotImplementedError
263
264  def GetSourceForTest(self, testcase):
265    return "(no source available)"
266
267  def IsFailureOutput(self, output, testpath):
268    return output.exit_code != 0
269
270  def IsNegativeTest(self, testcase):
271    return False
272
273  def HasFailed(self, testcase):
274    execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
275    if self.IsNegativeTest(testcase):
276      return not execution_failed
277    else:
278      return execution_failed
279
280  def GetOutcome(self, testcase):
281    if testcase.output.HasCrashed():
282      return statusfile.CRASH
283    elif testcase.output.HasTimedOut():
284      return statusfile.TIMEOUT
285    elif self.HasFailed(testcase):
286      return statusfile.FAIL
287    else:
288      return statusfile.PASS
289
290  def HasUnexpectedOutput(self, testcase):
291    outcome = self.GetOutcome(testcase)
292    return not outcome in (testcase.outcomes or [statusfile.PASS])
293
294  def StripOutputForTransmit(self, testcase):
295    if not self.HasUnexpectedOutput(testcase):
296      testcase.output.stdout = ""
297      testcase.output.stderr = ""
298
299  def CalculateTotalDuration(self):
300    self.total_duration = 0.0
301    for t in self.tests:
302      self.total_duration += t.duration
303    return self.total_duration
304
305
306class StandardVariantGenerator(VariantGenerator):
307  def FilterVariantsByTest(self, testcase):
308    return self.standard_variant
309
310
311class GoogleTestSuite(TestSuite):
312  def __init__(self, name, root):
313    super(GoogleTestSuite, self).__init__(name, root)
314
315  def ListTests(self, context):
316    shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
317    if utils.IsWindows():
318      shell += ".exe"
319    output = commands.Execute(context.command_prefix +
320                              [shell, "--gtest_list_tests"] +
321                              context.extra_flags)
322    if output.exit_code != 0:
323      print output.stdout
324      print output.stderr
325      raise Exception("Test executable failed to list the tests.")
326    tests = []
327    test_case = ''
328    for line in output.stdout.splitlines():
329      test_desc = line.strip().split()[0]
330      if test_desc.endswith('.'):
331        test_case = test_desc
332      elif test_case and test_desc:
333        test = testcase.TestCase(self, test_case + test_desc, dependency=None)
334        tests.append(test)
335    tests.sort()
336    return tests
337
338  def GetFlagsForTestCase(self, testcase, context):
339    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
340            ["--gtest_random_seed=%s" % context.random_seed] +
341            ["--gtest_print_time=0"] +
342            context.mode_flags)
343
344  def _VariantGeneratorFactory(self):
345    return StandardVariantGenerator
346
347  def shell(self):
348    return self.name
349