1# Copyright 2015 gRPC authors.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Generate XML and HTML test reports."""
15
16try:
17    from mako.runtime import Context
18    from mako.template import Template
19    from mako import exceptions
20except (ImportError):
21    pass  # Mako not installed but it is ok.
22import datetime
23import os
24import string
25import xml.etree.cElementTree as ET
26import six
27
28
29def _filter_msg(msg, output_format):
30    """Filters out nonprintable and illegal characters from the message."""
31    if output_format in ['XML', 'HTML']:
32        # keep whitespaces but remove formfeed and vertical tab characters
33        # that make XML report unparsable.
34        filtered_msg = ''.join(
35            filter(lambda x: x in string.printable and x != '\f' and x != '\v',
36                   msg.decode('UTF-8', 'ignore')))
37        if output_format == 'HTML':
38            filtered_msg = filtered_msg.replace('"', '"')
39        return filtered_msg
40    else:
41        return msg
42
43
44def new_junit_xml_tree():
45    return ET.ElementTree(ET.Element('testsuites'))
46
47
48def render_junit_xml_report(resultset,
49                            report_file,
50                            suite_package='grpc',
51                            suite_name='tests',
52                            replace_dots=True,
53                            multi_target=False):
54    """Generate JUnit-like XML report."""
55    if not multi_target:
56        tree = new_junit_xml_tree()
57        append_junit_xml_results(tree, resultset, suite_package, suite_name,
58                                 '1', replace_dots)
59        create_xml_report_file(tree, report_file)
60    else:
61        # To have each test result displayed as a separate target by the Resultstore/Sponge UI,
62        # we generate a separate XML report file for each test result
63        for shortname, results in six.iteritems(resultset):
64            one_result = {shortname: results}
65            tree = new_junit_xml_tree()
66            append_junit_xml_results(tree, one_result,
67                                     '%s_%s' % (suite_package, shortname),
68                                     '%s_%s' % (suite_name, shortname), '1',
69                                     replace_dots)
70            per_suite_report_file = os.path.join(os.path.dirname(report_file),
71                                                 shortname,
72                                                 os.path.basename(report_file))
73            create_xml_report_file(tree, per_suite_report_file)
74
75
76def create_xml_report_file(tree, report_file):
77    """Generate JUnit-like report file from xml tree ."""
78    # ensure the report directory exists
79    report_dir = os.path.dirname(os.path.abspath(report_file))
80    if not os.path.exists(report_dir):
81        os.makedirs(report_dir)
82    tree.write(report_file, encoding='UTF-8')
83
84
85def append_junit_xml_results(tree,
86                             resultset,
87                             suite_package,
88                             suite_name,
89                             id,
90                             replace_dots=True):
91    """Append a JUnit-like XML report tree with test results as a new suite."""
92    if replace_dots:
93        # ResultStore UI displays test suite names containing dots only as the component
94        # after the last dot, which results bad info being displayed in the UI.
95        # We replace dots by another character to avoid this problem.
96        suite_name = suite_name.replace('.', '_')
97    testsuite = ET.SubElement(tree.getroot(),
98                              'testsuite',
99                              id=id,
100                              package=suite_package,
101                              name=suite_name,
102                              timestamp=datetime.datetime.now().isoformat())
103    failure_count = 0
104    error_count = 0
105    for shortname, results in six.iteritems(resultset):
106        for result in results:
107            xml_test = ET.SubElement(testsuite, 'testcase', name=shortname)
108            if result.elapsed_time:
109                xml_test.set('time', str(result.elapsed_time))
110            filtered_msg = _filter_msg(result.message, 'XML')
111            if result.state == 'FAILED':
112                ET.SubElement(xml_test, 'failure',
113                              message='Failure').text = filtered_msg
114                failure_count += 1
115            elif result.state == 'TIMEOUT':
116                ET.SubElement(xml_test, 'error',
117                              message='Timeout').text = filtered_msg
118                error_count += 1
119            elif result.state == 'SKIPPED':
120                ET.SubElement(xml_test, 'skipped', message='Skipped')
121    testsuite.set('failures', str(failure_count))
122    testsuite.set('errors', str(error_count))
123
124
125def render_interop_html_report(client_langs, server_langs, test_cases,
126                               auth_test_cases, http2_cases, http2_server_cases,
127                               resultset, num_failures, cloud_to_prod,
128                               prod_servers, http2_interop):
129    """Generate HTML report for interop tests."""
130    template_file = 'tools/run_tests/interop/interop_html_report.template'
131    try:
132        mytemplate = Template(filename=template_file, format_exceptions=True)
133    except NameError:
134        print(
135            'Mako template is not installed. Skipping HTML report generation.')
136        return
137    except IOError as e:
138        print('Failed to find the template %s: %s' % (template_file, e))
139        return
140
141    sorted_test_cases = sorted(test_cases)
142    sorted_auth_test_cases = sorted(auth_test_cases)
143    sorted_http2_cases = sorted(http2_cases)
144    sorted_http2_server_cases = sorted(http2_server_cases)
145    sorted_client_langs = sorted(client_langs)
146    sorted_server_langs = sorted(server_langs)
147    sorted_prod_servers = sorted(prod_servers)
148
149    args = {
150        'client_langs': sorted_client_langs,
151        'server_langs': sorted_server_langs,
152        'test_cases': sorted_test_cases,
153        'auth_test_cases': sorted_auth_test_cases,
154        'http2_cases': sorted_http2_cases,
155        'http2_server_cases': sorted_http2_server_cases,
156        'resultset': resultset,
157        'num_failures': num_failures,
158        'cloud_to_prod': cloud_to_prod,
159        'prod_servers': sorted_prod_servers,
160        'http2_interop': http2_interop
161    }
162
163    html_report_out_dir = 'reports'
164    if not os.path.exists(html_report_out_dir):
165        os.mkdir(html_report_out_dir)
166    html_file_path = os.path.join(html_report_out_dir, 'index.html')
167    try:
168        with open(html_file_path, 'w') as output_file:
169            mytemplate.render_context(Context(output_file, **args))
170    except:
171        print(exceptions.text_error_template().render())
172        raise
173
174
175def render_perf_profiling_results(output_filepath, profile_names):
176    with open(output_filepath, 'w') as output_file:
177        output_file.write('<ul>\n')
178        for name in profile_names:
179            output_file.write('<li><a href=%s>%s</a></li>\n' % (name, name))
180        output_file.write('</ul>\n')
181