1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29from functools import wraps
30import json
31import os
32import sys
33import time
34
35from . import execution
36from . import junit_output
37from . import statusfile
38
39
40ABS_PATH_PREFIX = os.getcwd() + os.sep
41
42
43class ProgressIndicator(object):
44
45  def __init__(self):
46    self.runner = None
47
48  def SetRunner(self, runner):
49    self.runner = runner
50
51  def Starting(self):
52    pass
53
54  def Done(self):
55    pass
56
57  def HasRun(self, test, has_unexpected_output):
58    pass
59
60  def Heartbeat(self):
61    pass
62
63  def PrintFailureHeader(self, test):
64    if test.suite.IsNegativeTest(test):
65      negative_marker = '[negative] '
66    else:
67      negative_marker = ''
68    print "=== %(label)s %(negative)s===" % {
69      'label': test.GetLabel(),
70      'negative': negative_marker
71    }
72
73  def _EscapeCommand(self, test):
74    command = execution.GetCommand(test, self.runner.context)
75    parts = []
76    for part in command:
77      if ' ' in part:
78        # Escape spaces.  We may need to escape more characters for this
79        # to work properly.
80        parts.append('"%s"' % part)
81      else:
82        parts.append(part)
83    return " ".join(parts)
84
85
86class IndicatorNotifier(object):
87  """Holds a list of progress indicators and notifies them all on events."""
88  def __init__(self):
89    self.indicators = []
90
91  def Register(self, indicator):
92    self.indicators.append(indicator)
93
94
95# Forge all generic event-dispatching methods in IndicatorNotifier, which are
96# part of the ProgressIndicator interface.
97for func_name in ProgressIndicator.__dict__:
98  func = getattr(ProgressIndicator, func_name)
99  if callable(func) and not func.__name__.startswith('_'):
100    def wrap_functor(f):
101      @wraps(f)
102      def functor(self, *args, **kwargs):
103        """Generic event dispatcher."""
104        for indicator in self.indicators:
105          getattr(indicator, f.__name__)(*args, **kwargs)
106      return functor
107    setattr(IndicatorNotifier, func_name, wrap_functor(func))
108
109
110class SimpleProgressIndicator(ProgressIndicator):
111  """Abstract base class for {Verbose,Dots}ProgressIndicator"""
112
113  def Starting(self):
114    print 'Running %i tests' % self.runner.total
115
116  def Done(self):
117    print
118    for failed in self.runner.failed:
119      self.PrintFailureHeader(failed)
120      if failed.output.stderr:
121        print "--- stderr ---"
122        print failed.output.stderr.strip()
123      if failed.output.stdout:
124        print "--- stdout ---"
125        print failed.output.stdout.strip()
126      print "Command: %s" % self._EscapeCommand(failed)
127      if failed.output.HasCrashed():
128        print "exit code: %d" % failed.output.exit_code
129        print "--- CRASHED ---"
130      if failed.output.HasTimedOut():
131        print "--- TIMEOUT ---"
132    if len(self.runner.failed) == 0:
133      print "==="
134      print "=== All tests succeeded"
135      print "==="
136    else:
137      print
138      print "==="
139      print "=== %i tests failed" % len(self.runner.failed)
140      if self.runner.crashed > 0:
141        print "=== %i tests CRASHED" % self.runner.crashed
142      print "==="
143
144
145class VerboseProgressIndicator(SimpleProgressIndicator):
146
147  def HasRun(self, test, has_unexpected_output):
148    if has_unexpected_output:
149      if test.output.HasCrashed():
150        outcome = 'CRASH'
151      else:
152        outcome = 'FAIL'
153    else:
154      outcome = 'pass'
155    print 'Done running %s: %s' % (test.GetLabel(), outcome)
156    sys.stdout.flush()
157
158  def Heartbeat(self):
159    print 'Still working...'
160    sys.stdout.flush()
161
162
163class DotsProgressIndicator(SimpleProgressIndicator):
164
165  def HasRun(self, test, has_unexpected_output):
166    total = self.runner.succeeded + len(self.runner.failed)
167    if (total > 1) and (total % 50 == 1):
168      sys.stdout.write('\n')
169    if has_unexpected_output:
170      if test.output.HasCrashed():
171        sys.stdout.write('C')
172        sys.stdout.flush()
173      elif test.output.HasTimedOut():
174        sys.stdout.write('T')
175        sys.stdout.flush()
176      else:
177        sys.stdout.write('F')
178        sys.stdout.flush()
179    else:
180      sys.stdout.write('.')
181      sys.stdout.flush()
182
183
184class CompactProgressIndicator(ProgressIndicator):
185  """Abstract base class for {Color,Monochrome}ProgressIndicator"""
186
187  def __init__(self, templates):
188    super(CompactProgressIndicator, self).__init__()
189    self.templates = templates
190    self.last_status_length = 0
191    self.start_time = time.time()
192
193  def Done(self):
194    self.PrintProgress('Done')
195    print ""  # Line break.
196
197  def HasRun(self, test, has_unexpected_output):
198    self.PrintProgress(test.GetLabel())
199    if has_unexpected_output:
200      self.ClearLine(self.last_status_length)
201      self.PrintFailureHeader(test)
202      stdout = test.output.stdout.strip()
203      if len(stdout):
204        print self.templates['stdout'] % stdout
205      stderr = test.output.stderr.strip()
206      if len(stderr):
207        print self.templates['stderr'] % stderr
208      print "Command: %s" % self._EscapeCommand(test)
209      if test.output.HasCrashed():
210        print "exit code: %d" % test.output.exit_code
211        print "--- CRASHED ---"
212      if test.output.HasTimedOut():
213        print "--- TIMEOUT ---"
214
215  def Truncate(self, string, length):
216    if length and (len(string) > (length - 3)):
217      return string[:(length - 3)] + "..."
218    else:
219      return string
220
221  def PrintProgress(self, name):
222    self.ClearLine(self.last_status_length)
223    elapsed = time.time() - self.start_time
224    progress = 0 if not self.runner.total else (
225        ((self.runner.total - self.runner.remaining) * 100) //
226          self.runner.total)
227    status = self.templates['status_line'] % {
228      'passed': self.runner.succeeded,
229      'progress': progress,
230      'failed': len(self.runner.failed),
231      'test': name,
232      'mins': int(elapsed) / 60,
233      'secs': int(elapsed) % 60
234    }
235    status = self.Truncate(status, 78)
236    self.last_status_length = len(status)
237    print status,
238    sys.stdout.flush()
239
240
241class ColorProgressIndicator(CompactProgressIndicator):
242
243  def __init__(self):
244    templates = {
245      'status_line': ("[%(mins)02i:%(secs)02i|"
246                      "\033[34m%%%(progress) 4d\033[0m|"
247                      "\033[32m+%(passed) 4d\033[0m|"
248                      "\033[31m-%(failed) 4d\033[0m]: %(test)s"),
249      'stdout': "\033[1m%s\033[0m",
250      'stderr': "\033[31m%s\033[0m",
251    }
252    super(ColorProgressIndicator, self).__init__(templates)
253
254  def ClearLine(self, last_line_length):
255    print "\033[1K\r",
256
257
258class MonochromeProgressIndicator(CompactProgressIndicator):
259
260  def __init__(self):
261    templates = {
262      'status_line': ("[%(mins)02i:%(secs)02i|%%%(progress) 4d|"
263                      "+%(passed) 4d|-%(failed) 4d]: %(test)s"),
264      'stdout': '%s',
265      'stderr': '%s',
266    }
267    super(MonochromeProgressIndicator, self).__init__(templates)
268
269  def ClearLine(self, last_line_length):
270    print ("\r" + (" " * last_line_length) + "\r"),
271
272
273class JUnitTestProgressIndicator(ProgressIndicator):
274
275  def __init__(self, junitout, junittestsuite):
276    self.outputter = junit_output.JUnitTestOutput(junittestsuite)
277    if junitout:
278      self.outfile = open(junitout, "w")
279    else:
280      self.outfile = sys.stdout
281
282  def Done(self):
283    self.outputter.FinishAndWrite(self.outfile)
284    if self.outfile != sys.stdout:
285      self.outfile.close()
286
287  def HasRun(self, test, has_unexpected_output):
288    fail_text = ""
289    if has_unexpected_output:
290      stdout = test.output.stdout.strip()
291      if len(stdout):
292        fail_text += "stdout:\n%s\n" % stdout
293      stderr = test.output.stderr.strip()
294      if len(stderr):
295        fail_text += "stderr:\n%s\n" % stderr
296      fail_text += "Command: %s" % self._EscapeCommand(test)
297      if test.output.HasCrashed():
298        fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code
299      if test.output.HasTimedOut():
300        fail_text += "--- TIMEOUT ---"
301    self.outputter.HasRunTest(
302        [test.GetLabel()] + self.runner.context.mode_flags + test.flags,
303        test.duration,
304        fail_text)
305
306
307class JsonTestProgressIndicator(ProgressIndicator):
308
309  def __init__(self, json_test_results, arch, mode, random_seed):
310    self.json_test_results = json_test_results
311    self.arch = arch
312    self.mode = mode
313    self.random_seed = random_seed
314    self.results = []
315    self.tests = []
316
317  def Done(self):
318    complete_results = []
319    if os.path.exists(self.json_test_results):
320      with open(self.json_test_results, "r") as f:
321        # Buildbot might start out with an empty file.
322        complete_results = json.loads(f.read() or "[]")
323
324    duration_mean = None
325    if self.tests:
326      # Get duration mean.
327      duration_mean = (
328          sum(t.duration for t in self.tests) / float(len(self.tests)))
329
330    # Sort tests by duration.
331    timed_tests = [t for t in self.tests if t.duration is not None]
332    timed_tests.sort(lambda a, b: cmp(b.duration, a.duration))
333    slowest_tests = [
334      {
335        "name": test.GetLabel(),
336        "flags": test.flags,
337        "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""),
338        "duration": test.duration,
339        "marked_slow": statusfile.IsSlow(test.outcomes),
340      } for test in timed_tests[:20]
341    ]
342
343    complete_results.append({
344      "arch": self.arch,
345      "mode": self.mode,
346      "results": self.results,
347      "slowest_tests": slowest_tests,
348      "duration_mean": duration_mean,
349      "test_total": len(self.tests),
350    })
351
352    with open(self.json_test_results, "w") as f:
353      f.write(json.dumps(complete_results))
354
355  def HasRun(self, test, has_unexpected_output):
356    # Buffer all tests for sorting the durations in the end.
357    self.tests.append(test)
358    if not has_unexpected_output:
359      # Omit tests that run as expected. Passing tests of reruns after failures
360      # will have unexpected_output to be reported here has well.
361      return
362
363    self.results.append({
364      "name": test.GetLabel(),
365      "flags": test.flags,
366      "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""),
367      "run": test.run,
368      "stdout": test.output.stdout,
369      "stderr": test.output.stderr,
370      "exit_code": test.output.exit_code,
371      "result": test.suite.GetOutcome(test),
372      "expected": list(test.outcomes or ["PASS"]),
373      "duration": test.duration,
374
375      # TODO(machenbach): This stores only the global random seed from the
376      # context and not possible overrides when using random-seed stress.
377      "random_seed": self.random_seed,
378      "target_name": test.suite.shell(),
379      "variant": test.variant,
380    })
381
382
383PROGRESS_INDICATORS = {
384  'verbose': VerboseProgressIndicator,
385  'dots': DotsProgressIndicator,
386  'color': ColorProgressIndicator,
387  'mono': MonochromeProgressIndicator
388}
389