1#!/usr/bin/env python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Executes WebView CTS tests and verifies results against known failures.
19"""
20
21import re
22import signal
23import subprocess
24import sys
25
26# Eventually this list will be empty!
27# If you add or remove tests from this lists please update the CTS
28# spreadsheet!
29EXPECTED_FAILURES = set([
30  'android.webkit.cts.WebViewClientTest#testOnScaleChanged',
31  'android.webkit.cts.WebViewTest#testCapturePicture',
32  # BUG=crbug.com/162967
33  'android.webkit.cts.WebViewTest#testPageScroll',
34  'android.webkit.cts.WebViewTest#testRequestChildRectangleOnScreen',
35  'android.webkit.cts.WebViewTest#testScrollBarOverlay',
36  'android.webkit.cts.WebViewTest#testSetInitialScale',
37  'android.webkit.cts.WebViewTest#testSetScrollBarStyle',
38  'android.webkit.cts.WebViewTest#testSetWebViewClient',
39  'android.webkit.cts.WebViewTest#testZoom',
40  # b/8763845
41  'android.webkit.cts.WebViewTest#testFindAll',
42  # b/8763944
43  'android.webkit.cts.WebViewTest#testGetContentHeight',
44  # See b/8231433 for Geolocation test failures.
45  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptAlways',
46  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestReject',
47  'android.webkit.cts.GeolocationTest#testSimpleGeolocationRequestAcceptOnce',
48  # b/9159785
49  'android.webkit.cts.WebViewTest#testFindNext',
50  # b/5006389
51  'android.webkit.cts.WebViewTest#testFlingScroll',
52  # b/9103603
53  'android.webkit.cts.WebViewTest#testRequestImageRef',
54  # b/9121594
55  'android.webkit.cts.WebHistoryItemTest#testWebHistoryItem',
56  # Don't forget to update the spreadsheet! :)
57])
58
59def main():
60  proc = None
61
62  # Send INT signal to test runner and exit gracefully so not to lose all
63  # output information in a run.
64  def handler(signum, frame):
65    if proc:
66      proc.send_signal(signum)
67  signal.signal(signal.SIGINT, handler)
68
69  proc = subprocess.Popen(
70      ['cts-tradefed', 'run', 'singleCommand', 'cts', '-p', 'android.webkit', '--screenshot-on-failure'],
71      stdout=subprocess.PIPE, stderr=subprocess.PIPE)
72
73  (stdout, stderr) = proc.communicate();
74
75  passes = set(re.findall(r'.*: (.*) PASS', stdout))
76  failures = set(re.findall(r'.*: (.*) FAIL', stdout))
77  test_results = '%d passes; %d failures' % (len(passes), len(failures))
78
79  unexpected_passes = EXPECTED_FAILURES.difference(failures)
80  if len(unexpected_passes) > 0:
81    test_results += '\n' + 'UNEXPECTED PASSES (update expectations!):'
82    for test in unexpected_passes:
83      test_results += '\n' + '\t%s' % (test)
84
85  unexpected_failures = failures.difference(EXPECTED_FAILURES)
86  if len(unexpected_failures) > 0:
87    test_results += '\n' + 'UNEXPECTED FAILURES (please fix!):'
88    for test in unexpected_failures:
89      test_results += '\n' + '\t%s' % (test)
90
91  unexpected_failures_count = len(unexpected_failures)
92  unexpected_passes_count = len(unexpected_passes)
93
94  # on the buildbot this is most useful at the start
95  print test_results
96
97  print '\nstdout dump follows...'
98  print stdout
99  print '\n'
100
101  # on the cmd line this is most useful at the end
102  print test_results
103
104  # Allow buildbot script to distinguish failures and possibly out of date
105  # test expectations.
106  if len(passes) + len(failures) < 100:
107    print 'Ran less than 100 cts tests? Something must be wrong'
108    return 2
109  elif unexpected_failures_count > 0:
110    return 1
111  elif unexpected_passes_count >= 5:
112    print ('More than 5 new passes? Either you''re running webview classic, or '
113           'it really is time to fix failure expectations.')
114    return 2
115  elif unexpected_passes_count > 0:
116    return 3  # STEP_WARNINGS
117  else:
118    return 0
119
120
121if __name__ == '__main__':
122  sys.exit(main())
123