1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import os
30import shutil
31import time
32
33from pool import Pool
34from . import commands
35from . import perfdata
36from . import utils
37
38
39class Job(object):
40  def __init__(self, command, dep_command, test_id, timeout, verbose):
41    self.command = command
42    self.dep_command = dep_command
43    self.id = test_id
44    self.timeout = timeout
45    self.verbose = verbose
46
47
48def RunTest(job):
49  start_time = time.time()
50  if job.dep_command is not None:
51    dep_output = commands.Execute(job.dep_command, job.verbose, job.timeout)
52    # TODO(jkummerow): We approximate the test suite specific function
53    # IsFailureOutput() by just checking the exit code here. Currently
54    # only cctests define dependencies, for which this simplification is
55    # correct.
56    if dep_output.exit_code != 0:
57      return (job.id, dep_output, time.time() - start_time)
58  output = commands.Execute(job.command, job.verbose, job.timeout)
59  return (job.id, output, time.time() - start_time)
60
61class Runner(object):
62
63  def __init__(self, suites, progress_indicator, context):
64    self.datapath = os.path.join("out", "testrunner_data")
65    self.perf_data_manager = perfdata.PerfDataManager(self.datapath)
66    self.perfdata = self.perf_data_manager.GetStore(context.arch, context.mode)
67    self.perf_failures = False
68    self.printed_allocations = False
69    self.tests = [ t for s in suites for t in s.tests ]
70    if not context.no_sorting:
71      for t in self.tests:
72        t.duration = self.perfdata.FetchPerfData(t) or 1.0
73      self.tests.sort(key=lambda t: t.duration, reverse=True)
74    self._CommonInit(len(self.tests), progress_indicator, context)
75
76  def _CommonInit(self, num_tests, progress_indicator, context):
77    self.indicator = progress_indicator
78    progress_indicator.runner = self
79    self.context = context
80    self.succeeded = 0
81    self.total = num_tests
82    self.remaining = num_tests
83    self.failed = []
84    self.crashed = 0
85    self.reran_tests = 0
86
87  def _RunPerfSafe(self, fun):
88    try:
89      fun()
90    except Exception, e:
91      print("PerfData exception: %s" % e)
92      self.perf_failures = True
93
94  def _GetJob(self, test):
95    command = self.GetCommand(test)
96    timeout = self.context.timeout
97    if ("--stress-opt" in test.flags or
98        "--stress-opt" in self.context.mode_flags or
99        "--stress-opt" in self.context.extra_flags):
100      timeout *= 4
101    if test.dependency is not None:
102      dep_command = [ c.replace(test.path, test.dependency) for c in command ]
103    else:
104      dep_command = None
105    return Job(command, dep_command, test.id, timeout, self.context.verbose)
106
107  def _MaybeRerun(self, pool, test):
108    if test.run <= self.context.rerun_failures_count:
109      # Possibly rerun this test if its run count is below the maximum per
110      # test. <= as the flag controls reruns not including the first run.
111      if test.run == 1:
112        # Count the overall number of reran tests on the first rerun.
113        if self.reran_tests < self.context.rerun_failures_max:
114          self.reran_tests += 1
115        else:
116          # Don't rerun this if the overall number of rerun tests has been
117          # reached.
118          return
119      if test.run >= 2 and test.duration > self.context.timeout / 20.0:
120        # Rerun slow tests at most once.
121        return
122
123      # Rerun this test.
124      test.duration = None
125      test.output = None
126      test.run += 1
127      pool.add([self._GetJob(test)])
128      self.remaining += 1
129
130  def _ProcessTestNormal(self, test, result, pool):
131    self.indicator.AboutToRun(test)
132    test.output = result[1]
133    test.duration = result[2]
134    has_unexpected_output = test.suite.HasUnexpectedOutput(test)
135    if has_unexpected_output:
136      self.failed.append(test)
137      if test.output.HasCrashed():
138        self.crashed += 1
139    else:
140      self.succeeded += 1
141    self.remaining -= 1
142    # For the indicator, everything that happens after the first run is treated
143    # as unexpected even if it flakily passes in order to include it in the
144    # output.
145    self.indicator.HasRun(test, has_unexpected_output or test.run > 1)
146    if has_unexpected_output:
147      # Rerun test failures after the indicator has processed the results.
148      self._MaybeRerun(pool, test)
149    # Update the perf database if the test succeeded.
150    return not has_unexpected_output
151
152  def _ProcessTestPredictable(self, test, result, pool):
153    def HasDifferentAllocations(output1, output2):
154      def AllocationStr(stdout):
155        for line in reversed((stdout or "").splitlines()):
156          if line.startswith("### Allocations = "):
157            self.printed_allocations = True
158            return line
159        return ""
160      return (AllocationStr(output1.stdout) != AllocationStr(output2.stdout))
161
162    # Always pass the test duration for the database update.
163    test.duration = result[2]
164    if test.run == 1 and result[1].HasTimedOut():
165      # If we get a timeout in the first run, we are already in an
166      # unpredictable state. Just report it as a failure and don't rerun.
167      self.indicator.AboutToRun(test)
168      test.output = result[1]
169      self.remaining -= 1
170      self.failed.append(test)
171      self.indicator.HasRun(test, True)
172    if test.run > 1 and HasDifferentAllocations(test.output, result[1]):
173      # From the second run on, check for different allocations. If a
174      # difference is found, call the indicator twice to report both tests.
175      # All runs of each test are counted as one for the statistic.
176      self.indicator.AboutToRun(test)
177      self.remaining -= 1
178      self.failed.append(test)
179      self.indicator.HasRun(test, True)
180      self.indicator.AboutToRun(test)
181      test.output = result[1]
182      self.indicator.HasRun(test, True)
183    elif test.run >= 3:
184      # No difference on the third run -> report a success.
185      self.indicator.AboutToRun(test)
186      self.remaining -= 1
187      self.succeeded += 1
188      test.output = result[1]
189      self.indicator.HasRun(test, False)
190    else:
191      # No difference yet and less than three runs -> add another run and
192      # remember the output for comparison.
193      test.run += 1
194      test.output = result[1]
195      pool.add([self._GetJob(test)])
196    # Always update the perf database.
197    return True
198
199  def Run(self, jobs):
200    self.indicator.Starting()
201    self._RunInternal(jobs)
202    self.indicator.Done()
203    if self.failed or self.remaining:
204      return 1
205    return 0
206
207  def _RunInternal(self, jobs):
208    pool = Pool(jobs)
209    test_map = {}
210    # TODO(machenbach): Instead of filling the queue completely before
211    # pool.imap_unordered, make this a generator that already starts testing
212    # while the queue is filled.
213    queue = []
214    queued_exception = None
215    for test in self.tests:
216      assert test.id >= 0
217      test_map[test.id] = test
218      try:
219        queue.append([self._GetJob(test)])
220      except Exception, e:
221        # If this failed, save the exception and re-raise it later (after
222        # all other tests have had a chance to run).
223        queued_exception = e
224        continue
225    try:
226      it = pool.imap_unordered(RunTest, queue)
227      for result in it:
228        test = test_map[result[0]]
229        if self.context.predictable:
230          update_perf = self._ProcessTestPredictable(test, result, pool)
231        else:
232          update_perf = self._ProcessTestNormal(test, result, pool)
233        if update_perf:
234          self._RunPerfSafe(lambda: self.perfdata.UpdatePerfData(test))
235    finally:
236      pool.terminate()
237      self._RunPerfSafe(lambda: self.perf_data_manager.close())
238      if self.perf_failures:
239        # Nuke perf data in case of failures. This might not work on windows as
240        # some files might still be open.
241        print "Deleting perf test data due to db corruption."
242        shutil.rmtree(self.datapath)
243    if queued_exception:
244      raise queued_exception
245
246    # Make sure that any allocations were printed in predictable mode.
247    assert not self.context.predictable or self.printed_allocations
248
249  def GetCommand(self, test):
250    d8testflag = []
251    shell = test.suite.shell()
252    if shell == "d8":
253      d8testflag = ["--test"]
254    if utils.IsWindows():
255      shell += ".exe"
256    cmd = (self.context.command_prefix +
257           [os.path.abspath(os.path.join(self.context.shell_dir, shell))] +
258           d8testflag +
259           ["--random-seed=%s" % self.context.random_seed] +
260           test.suite.GetFlagsForTestCase(test, self.context) +
261           self.context.extra_flags)
262    return cmd
263
264
265class BreakNowException(Exception):
266  def __init__(self, value):
267    self.value = value
268  def __str__(self):
269    return repr(self.value)
270