1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import json
30import os
31import sys
32import time
33
34from . import junit_output
35
36
37ABS_PATH_PREFIX = os.getcwd() + os.sep
38
39
40def EscapeCommand(command):
41  parts = []
42  for part in command:
43    if ' ' in part:
44      # Escape spaces.  We may need to escape more characters for this
45      # to work properly.
46      parts.append('"%s"' % part)
47    else:
48      parts.append(part)
49  return " ".join(parts)
50
51
52class ProgressIndicator(object):
53
54  def __init__(self):
55    self.runner = None
56
57  def Starting(self):
58    pass
59
60  def Done(self):
61    pass
62
63  def AboutToRun(self, test):
64    pass
65
66  def HasRun(self, test, has_unexpected_output):
67    pass
68
69  def PrintFailureHeader(self, test):
70    if test.suite.IsNegativeTest(test):
71      negative_marker = '[negative] '
72    else:
73      negative_marker = ''
74    print "=== %(label)s %(negative)s===" % {
75      'label': test.GetLabel(),
76      'negative': negative_marker
77    }
78
79
80class SimpleProgressIndicator(ProgressIndicator):
81  """Abstract base class for {Verbose,Dots}ProgressIndicator"""
82
83  def Starting(self):
84    print 'Running %i tests' % self.runner.total
85
86  def Done(self):
87    print
88    for failed in self.runner.failed:
89      self.PrintFailureHeader(failed)
90      if failed.output.stderr:
91        print "--- stderr ---"
92        print failed.output.stderr.strip()
93      if failed.output.stdout:
94        print "--- stdout ---"
95        print failed.output.stdout.strip()
96      print "Command: %s" % EscapeCommand(self.runner.GetCommand(failed))
97      if failed.output.HasCrashed():
98        print "exit code: %d" % failed.output.exit_code
99        print "--- CRASHED ---"
100      if failed.output.HasTimedOut():
101        print "--- TIMEOUT ---"
102    if len(self.runner.failed) == 0:
103      print "==="
104      print "=== All tests succeeded"
105      print "==="
106    else:
107      print
108      print "==="
109      print "=== %i tests failed" % len(self.runner.failed)
110      if self.runner.crashed > 0:
111        print "=== %i tests CRASHED" % self.runner.crashed
112      print "==="
113
114
115class VerboseProgressIndicator(SimpleProgressIndicator):
116
117  def AboutToRun(self, test):
118    print 'Starting %s...' % test.GetLabel()
119    sys.stdout.flush()
120
121  def HasRun(self, test, has_unexpected_output):
122    if has_unexpected_output:
123      if test.output.HasCrashed():
124        outcome = 'CRASH'
125      else:
126        outcome = 'FAIL'
127    else:
128      outcome = 'pass'
129    print 'Done running %s: %s' % (test.GetLabel(), outcome)
130
131
132class DotsProgressIndicator(SimpleProgressIndicator):
133
134  def HasRun(self, test, has_unexpected_output):
135    total = self.runner.succeeded + len(self.runner.failed)
136    if (total > 1) and (total % 50 == 1):
137      sys.stdout.write('\n')
138    if has_unexpected_output:
139      if test.output.HasCrashed():
140        sys.stdout.write('C')
141        sys.stdout.flush()
142      elif test.output.HasTimedOut():
143        sys.stdout.write('T')
144        sys.stdout.flush()
145      else:
146        sys.stdout.write('F')
147        sys.stdout.flush()
148    else:
149      sys.stdout.write('.')
150      sys.stdout.flush()
151
152
153class CompactProgressIndicator(ProgressIndicator):
154  """Abstract base class for {Color,Monochrome}ProgressIndicator"""
155
156  def __init__(self, templates):
157    super(CompactProgressIndicator, self).__init__()
158    self.templates = templates
159    self.last_status_length = 0
160    self.start_time = time.time()
161
162  def Done(self):
163    self.PrintProgress('Done')
164    print ""  # Line break.
165
166  def AboutToRun(self, test):
167    self.PrintProgress(test.GetLabel())
168
169  def HasRun(self, test, has_unexpected_output):
170    if has_unexpected_output:
171      self.ClearLine(self.last_status_length)
172      self.PrintFailureHeader(test)
173      stdout = test.output.stdout.strip()
174      if len(stdout):
175        print self.templates['stdout'] % stdout
176      stderr = test.output.stderr.strip()
177      if len(stderr):
178        print self.templates['stderr'] % stderr
179      print "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
180      if test.output.HasCrashed():
181        print "exit code: %d" % test.output.exit_code
182        print "--- CRASHED ---"
183      if test.output.HasTimedOut():
184        print "--- TIMEOUT ---"
185
186  def Truncate(self, string, length):
187    if length and (len(string) > (length - 3)):
188      return string[:(length - 3)] + "..."
189    else:
190      return string
191
192  def PrintProgress(self, name):
193    self.ClearLine(self.last_status_length)
194    elapsed = time.time() - self.start_time
195    status = self.templates['status_line'] % {
196      'passed': self.runner.succeeded,
197      'remaining': (((self.runner.total - self.runner.remaining) * 100) //
198                    self.runner.total),
199      'failed': len(self.runner.failed),
200      'test': name,
201      'mins': int(elapsed) / 60,
202      'secs': int(elapsed) % 60
203    }
204    status = self.Truncate(status, 78)
205    self.last_status_length = len(status)
206    print status,
207    sys.stdout.flush()
208
209
210class ColorProgressIndicator(CompactProgressIndicator):
211
212  def __init__(self):
213    templates = {
214      'status_line': ("[%(mins)02i:%(secs)02i|"
215                      "\033[34m%%%(remaining) 4d\033[0m|"
216                      "\033[32m+%(passed) 4d\033[0m|"
217                      "\033[31m-%(failed) 4d\033[0m]: %(test)s"),
218      'stdout': "\033[1m%s\033[0m",
219      'stderr': "\033[31m%s\033[0m",
220    }
221    super(ColorProgressIndicator, self).__init__(templates)
222
223  def ClearLine(self, last_line_length):
224    print "\033[1K\r",
225
226
227class MonochromeProgressIndicator(CompactProgressIndicator):
228
229  def __init__(self):
230    templates = {
231      'status_line': ("[%(mins)02i:%(secs)02i|%%%(remaining) 4d|"
232                      "+%(passed) 4d|-%(failed) 4d]: %(test)s"),
233      'stdout': '%s',
234      'stderr': '%s',
235    }
236    super(MonochromeProgressIndicator, self).__init__(templates)
237
238  def ClearLine(self, last_line_length):
239    print ("\r" + (" " * last_line_length) + "\r"),
240
241
242class JUnitTestProgressIndicator(ProgressIndicator):
243
244  def __init__(self, progress_indicator, junitout, junittestsuite):
245    self.progress_indicator = progress_indicator
246    self.outputter = junit_output.JUnitTestOutput(junittestsuite)
247    if junitout:
248      self.outfile = open(junitout, "w")
249    else:
250      self.outfile = sys.stdout
251
252  def Starting(self):
253    self.progress_indicator.runner = self.runner
254    self.progress_indicator.Starting()
255
256  def Done(self):
257    self.progress_indicator.Done()
258    self.outputter.FinishAndWrite(self.outfile)
259    if self.outfile != sys.stdout:
260      self.outfile.close()
261
262  def AboutToRun(self, test):
263    self.progress_indicator.AboutToRun(test)
264
265  def HasRun(self, test, has_unexpected_output):
266    self.progress_indicator.HasRun(test, has_unexpected_output)
267    fail_text = ""
268    if has_unexpected_output:
269      stdout = test.output.stdout.strip()
270      if len(stdout):
271        fail_text += "stdout:\n%s\n" % stdout
272      stderr = test.output.stderr.strip()
273      if len(stderr):
274        fail_text += "stderr:\n%s\n" % stderr
275      fail_text += "Command: %s" % EscapeCommand(self.runner.GetCommand(test))
276      if test.output.HasCrashed():
277        fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code
278      if test.output.HasTimedOut():
279        fail_text += "--- TIMEOUT ---"
280    self.outputter.HasRunTest(
281        [test.GetLabel()] + self.runner.context.mode_flags + test.flags,
282        test.duration,
283        fail_text)
284
285
286class JsonTestProgressIndicator(ProgressIndicator):
287
288  def __init__(self, progress_indicator, json_test_results, arch, mode):
289    self.progress_indicator = progress_indicator
290    self.json_test_results = json_test_results
291    self.arch = arch
292    self.mode = mode
293    self.results = []
294
295  def Starting(self):
296    self.progress_indicator.runner = self.runner
297    self.progress_indicator.Starting()
298
299  def Done(self):
300    self.progress_indicator.Done()
301    complete_results = []
302    if os.path.exists(self.json_test_results):
303      with open(self.json_test_results, "r") as f:
304        # Buildbot might start out with an empty file.
305        complete_results = json.loads(f.read() or "[]")
306
307    complete_results.append({
308      "arch": self.arch,
309      "mode": self.mode,
310      "results": self.results,
311    })
312
313    with open(self.json_test_results, "w") as f:
314      f.write(json.dumps(complete_results))
315
316  def AboutToRun(self, test):
317    self.progress_indicator.AboutToRun(test)
318
319  def HasRun(self, test, has_unexpected_output):
320    self.progress_indicator.HasRun(test, has_unexpected_output)
321    if not has_unexpected_output:
322      # Omit tests that run as expected. Passing tests of reruns after failures
323      # will have unexpected_output to be reported here has well.
324      return
325
326    self.results.append({
327      "name": test.GetLabel(),
328      "flags": test.flags,
329      "command": EscapeCommand(self.runner.GetCommand(test)).replace(
330          ABS_PATH_PREFIX, ""),
331      "run": test.run,
332      "stdout": test.output.stdout,
333      "stderr": test.output.stderr,
334      "exit_code": test.output.exit_code,
335      "result": test.suite.GetOutcome(test),
336    })
337
338
339PROGRESS_INDICATORS = {
340  'verbose': VerboseProgressIndicator,
341  'dots': DotsProgressIndicator,
342  'color': ColorProgressIndicator,
343  'mono': MonochromeProgressIndicator
344}
345