1#
2# Copyright (C) 2017 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import base64
18import getpass
19import logging
20import os
21import socket
22import time
23
24from vts.proto import VtsReportMessage_pb2 as ReportMsg
25from vts.runners.host import keys
26from vts.utils.python.web import dashboard_rest_client
27from vts.utils.python.web import feature_utils
28
29_PROFILING_POINTS = "profiling_points"
30
31
32class WebFeature(feature_utils.Feature):
33    """Feature object for web functionality.
34
35    Attributes:
36        enabled: boolean, True if web feature is enabled, False otherwise
37        report_msg: TestReportMessage, Proto summarizing the test run
38        current_test_report_msg: TestCaseReportMessage, Proto summarizing the current test case
39        rest_client: DashboardRestClient, client to which data will be posted
40    """
41
42    _TOGGLE_PARAM = keys.ConfigKeys.IKEY_ENABLE_WEB
43    _REQUIRED_PARAMS = [
44        keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND,
45        keys.ConfigKeys.IKEY_SERVICE_JSON_PATH,
46        keys.ConfigKeys.KEY_TESTBED_NAME, keys.ConfigKeys.IKEY_BUILD,
47        keys.ConfigKeys.IKEY_ANDROID_DEVICE, keys.ConfigKeys.IKEY_ABI_NAME,
48        keys.ConfigKeys.IKEY_ABI_BITNESS
49    ]
50    _OPTIONAL_PARAMS = [
51        keys.ConfigKeys.RUN_AS_VTS_SELFTEST,
52        keys.ConfigKeys.IKEY_ENABLE_PROFILING,
53    ]
54
55    def __init__(self, user_params):
56        """Initializes the web feature.
57
58        Parses the arguments and initializes the web functionality.
59
60        Args:
61            user_params: A dictionary from parameter name (String) to parameter value.
62        """
63        self.ParseParameters(
64            toggle_param_name=self._TOGGLE_PARAM,
65            required_param_names=self._REQUIRED_PARAMS,
66            optional_param_names=self._OPTIONAL_PARAMS,
67            user_params=user_params)
68        if not self.enabled:
69            return
70
71        # Initialize the dashboard client
72        post_cmd = getattr(self, keys.ConfigKeys.IKEY_DASHBOARD_POST_COMMAND)
73        service_json_path = str(
74            getattr(self, keys.ConfigKeys.IKEY_SERVICE_JSON_PATH))
75        try:
76            self.rest_client = dashboard_rest_client.DashboardRestClient(
77                post_cmd, service_json_path)
78            if not self.rest_client.Initialize():
79                self.enabled = False
80        except Exception as e:
81            logging.exception("Failed to create REST client.")
82            self.enabled = False
83
84        self.report_msg = ReportMsg.TestReportMessage()
85        self.report_msg.test = str(
86            getattr(self, keys.ConfigKeys.KEY_TESTBED_NAME))
87
88        if getattr(self, keys.ConfigKeys.IKEY_ENABLE_PROFILING, False):
89            self.report_msg.test += "Profiling"
90
91        self.report_msg.test_type = ReportMsg.VTS_HOST_DRIVEN_STRUCTURAL
92        self.report_msg.start_timestamp = feature_utils.GetTimestamp()
93        self.report_msg.host_info.hostname = socket.gethostname()
94
95        android_devices = getattr(self, keys.ConfigKeys.IKEY_ANDROID_DEVICE,
96                                  None)
97        if not android_devices or not isinstance(android_devices, list):
98            logging.warn("android device information not available")
99            return
100
101        for device_spec in android_devices:
102            dev_info = self.report_msg.device_info.add()
103            for elem in [
104                    keys.ConfigKeys.IKEY_PRODUCT_TYPE,
105                    keys.ConfigKeys.IKEY_PRODUCT_VARIANT,
106                    keys.ConfigKeys.IKEY_BUILD_FLAVOR,
107                    keys.ConfigKeys.IKEY_BUILD_ID, keys.ConfigKeys.IKEY_BRANCH,
108                    keys.ConfigKeys.IKEY_BUILD_ALIAS,
109                    keys.ConfigKeys.IKEY_API_LEVEL, keys.ConfigKeys.IKEY_SERIAL
110            ]:
111                if elem in device_spec:
112                    setattr(dev_info, elem, str(device_spec[elem]))
113            # TODO: get abi information differently for multi-device support.
114            setattr(dev_info, keys.ConfigKeys.IKEY_ABI_NAME,
115                    str(getattr(self, keys.ConfigKeys.IKEY_ABI_NAME)))
116            setattr(dev_info, keys.ConfigKeys.IKEY_ABI_BITNESS,
117                    str(getattr(self, keys.ConfigKeys.IKEY_ABI_BITNESS)))
118
119    def SetTestResult(self, result=None):
120        """Set the current test case result to the provided result.
121
122        If None is provided as a result, the current test report will be cleared, which results
123        in a silent skip.
124
125        Requires the feature to be enabled; no-op otherwise.
126
127        Args:
128            result: ReportMsg.TestCaseResult, the result of the current test or None.
129        """
130        if not self.enabled:
131            return
132
133        if not result:
134            self.report_msg.test_case.remove(self.current_test_report_msg)
135            self.current_test_report_msg = None
136        else:
137            self.current_test_report_msg.test_result = result
138
139    def AddTestReport(self, test_name):
140        """Creates a report for the specified test.
141
142        Requires the feature to be enabled; no-op otherwise.
143
144        Args:
145            test_name: String, the name of the test
146        """
147        if not self.enabled:
148            return
149        self.current_test_report_msg = self.report_msg.test_case.add()
150        self.current_test_report_msg.name = test_name
151        self.current_test_report_msg.start_timestamp = feature_utils.GetTimestamp(
152        )
153
154    def AddApiCoverageReport(self, api_coverage_data_vec, isGlobal=True):
155        """Adds an API coverage report to the VtsReportMessage.
156
157        Translate each element in the give coverage data vector into a
158        ApiCoverageReportMessage within the report message.
159
160        Args:
161            api_coverage_data_vec: list of VTSApiCoverageData which contains
162                                   the metadata (e.g. package_name, version)
163                                   and the total/covered api names.
164            isGlobal: boolean, True if the coverage data is for the entire test,
165                      False if only for the current test case.
166        """
167
168        if not self.enabled:
169            return
170
171        if isGlobal:
172            report = self.report_msg
173        else:
174            report = self.current_test_report_msg
175
176        for api_coverage_data in api_coverage_data_vec:
177            api_coverage = report.api_coverage.add()
178            api_coverage.hal_interface.hal_package_name = api_coverage_data.package_name
179            api_coverage.hal_interface.hal_version_major = int(
180                api_coverage_data.version_major)
181            api_coverage.hal_interface.hal_version_minor = int(
182                api_coverage_data.version_minor)
183            api_coverage.hal_interface.hal_interface_name = api_coverage_data.interface_name
184            api_coverage.hal_api.extend(api_coverage_data.total_apis)
185            api_coverage.covered_hal_api.extend(api_coverage_data.covered_apis)
186
187    def AddCoverageReport(self,
188                          coverage_vec,
189                          src_file_path,
190                          git_project_name,
191                          git_project_path,
192                          revision,
193                          covered_count,
194                          line_count,
195                          isGlobal=True):
196        """Adds a coverage report to the VtsReportMessage.
197
198        Processes the source information, git project information, and processed
199        coverage information and stores it into a CoverageReportMessage within the
200        report message.
201
202        Args:
203            coverage_vec: list, list of coverage counts (int) for each line
204            src_file_path: the path to the original source file
205            git_project_name: the name of the git project containing the source
206            git_project_path: the path from the root to the git project
207            revision: the commit hash identifying the source code that was used to
208                      build a device image
209            covered_count: int, number of lines covered
210            line_count: int, total number of lines
211            isGlobal: boolean, True if the coverage data is for the entire test, False if only for
212                      the current test case.
213        """
214        if not self.enabled:
215            return
216
217        if isGlobal:
218            report = self.report_msg
219        else:
220            report = self.current_test_report_msg
221
222        coverage = report.coverage.add()
223        coverage.total_line_count = line_count
224        coverage.covered_line_count = covered_count
225        coverage.line_coverage_vector.extend(coverage_vec)
226
227        src_file_path = os.path.relpath(src_file_path, git_project_path)
228        coverage.file_path = src_file_path
229        coverage.revision = revision
230        coverage.project_name = git_project_name
231
232    def AddProfilingDataTimestamp(
233            self,
234            name,
235            start_timestamp,
236            end_timestamp,
237            x_axis_label="Latency (nano secs)",
238            y_axis_label="Frequency",
239            regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING):
240        """Adds the timestamp profiling data to the web DB.
241
242        Requires the feature to be enabled; no-op otherwise.
243
244        Args:
245            name: string, profiling point name.
246            start_timestamp: long, nanoseconds start time.
247            end_timestamp: long, nanoseconds end time.
248            x-axis_label: string, the x-axis label title for a graph plot.
249            y-axis_label: string, the y-axis label title for a graph plot.
250            regression_mode: specifies the direction of change which indicates
251                             performance regression.
252        """
253        if not self.enabled:
254            return
255
256        if not hasattr(self, _PROFILING_POINTS):
257            setattr(self, _PROFILING_POINTS, set())
258
259        if name in getattr(self, _PROFILING_POINTS):
260            logging.error("profiling point %s is already active.", name)
261            return
262
263        getattr(self, _PROFILING_POINTS).add(name)
264        profiling_msg = self.report_msg.profiling.add()
265        profiling_msg.name = name
266        profiling_msg.type = ReportMsg.VTS_PROFILING_TYPE_TIMESTAMP
267        profiling_msg.regression_mode = regression_mode
268        profiling_msg.start_timestamp = start_timestamp
269        profiling_msg.end_timestamp = end_timestamp
270        profiling_msg.x_axis_label = x_axis_label
271        profiling_msg.y_axis_label = y_axis_label
272
273    def AddProfilingDataVector(
274            self,
275            name,
276            labels,
277            values,
278            data_type,
279            options=[],
280            x_axis_label="x-axis",
281            y_axis_label="y-axis",
282            regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING):
283        """Adds the vector profiling data in order to upload to the web DB.
284
285        Requires the feature to be enabled; no-op otherwise.
286
287        Args:
288            name: string, profiling point name.
289            labels: a list or set of labels.
290            values: a list or set of values where each value is an integer.
291            data_type: profiling data type.
292            options: a set of options.
293            x-axis_label: string, the x-axis label title for a graph plot.
294            y-axis_label: string, the y-axis label title for a graph plot.
295            regression_mode: specifies the direction of change which indicates
296                             performance regression.
297        """
298        if not self.enabled:
299            return
300
301        if not hasattr(self, _PROFILING_POINTS):
302            setattr(self, _PROFILING_POINTS, set())
303
304        if name in getattr(self, _PROFILING_POINTS):
305            logging.error("profiling point %s is already active.", name)
306            return
307
308        getattr(self, _PROFILING_POINTS).add(name)
309        profiling_msg = self.report_msg.profiling.add()
310        profiling_msg.name = name
311        profiling_msg.type = data_type
312        profiling_msg.regression_mode = regression_mode
313        if labels:
314            profiling_msg.label.extend(labels)
315        profiling_msg.value.extend(values)
316        profiling_msg.x_axis_label = x_axis_label
317        profiling_msg.y_axis_label = y_axis_label
318        profiling_msg.options.extend(options)
319
320    def AddProfilingDataLabeledVector(
321            self,
322            name,
323            labels,
324            values,
325            options=[],
326            x_axis_label="x-axis",
327            y_axis_label="y-axis",
328            regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING):
329        """Adds the labeled vector profiling data in order to upload to the web DB.
330
331        Requires the feature to be enabled; no-op otherwise.
332
333        Args:
334            name: string, profiling point name.
335            labels: a list or set of labels.
336            values: a list or set of values where each value is an integer.
337            options: a set of options.
338            x-axis_label: string, the x-axis label title for a graph plot.
339            y-axis_label: string, the y-axis label title for a graph plot.
340            regression_mode: specifies the direction of change which indicates
341                             performance regression.
342        """
343        self.AddProfilingDataVector(
344            name, labels, values, ReportMsg.VTS_PROFILING_TYPE_LABELED_VECTOR,
345            options, x_axis_label, y_axis_label, regression_mode)
346
347    def AddProfilingDataUnlabeledVector(
348            self,
349            name,
350            values,
351            options=[],
352            x_axis_label="x-axis",
353            y_axis_label="y-axis",
354            regression_mode=ReportMsg.VTS_REGRESSION_MODE_INCREASING):
355        """Adds the unlabeled vector profiling data in order to upload to the web DB.
356
357        Requires the feature to be enabled; no-op otherwise.
358
359        Args:
360            name: string, profiling point name.
361            values: a list or set of values where each value is an integer.
362            options: a set of options.
363            x-axis_label: string, the x-axis label title for a graph plot.
364            y-axis_label: string, the y-axis label title for a graph plot.
365            regression_mode: specifies the direction of change which indicates
366                             performance regression.
367        """
368        self.AddProfilingDataVector(
369            name, None, values, ReportMsg.VTS_PROFILING_TYPE_UNLABELED_VECTOR,
370            options, x_axis_label, y_axis_label, regression_mode)
371
372    def AddSystraceUrl(self, url):
373        """Creates a systrace report message with a systrace URL.
374
375        Adds a systrace report to the current test case report and supplies the
376        url to the systrace report.
377
378        Requires the feature to be enabled; no-op otherwise.
379
380        Args:
381            url: String, the url of the systrace report.
382        """
383        if not self.enabled:
384            return
385        systrace_msg = self.current_test_report_msg.systrace.add()
386        systrace_msg.url.append(url)
387
388    def AddLogUrls(self, urls):
389        """Creates a log message with log file URLs.
390
391        Adds a log message to the current test module report and supplies the
392        url to the log files.
393
394        Requires the feature to be enabled; no-op otherwise.
395
396        Args:
397            urls: list of string, the URLs of the logs.
398        """
399        if not self.enabled or urls is None:
400            return
401
402        for url in urls:
403            for log_msg in self.report_msg.log:
404                if log_msg.url == url:
405                    continue
406
407            log_msg = self.report_msg.log.add()
408            log_msg.url = url
409            log_msg.name = os.path.basename(url)
410
411    def GetTestModuleKeys(self):
412        """Returns the test module name and start timestamp.
413
414        Those two values can be used to find the corresponding entry
415        in a used nosql database without having to lock all the data
416        (which is infesiable) thus are essential for strong consistency.
417        """
418        return self.report_msg.test, self.report_msg.start_timestamp
419
420    def GenerateReportMessage(self, requested, executed):
421        """Uploads the result to the web service.
422
423        Requires the feature to be enabled; no-op otherwise.
424
425        Args:
426            requested: list, A list of test case records requested to run
427            executed: list, A list of test case records that were executed
428
429        Returns:
430            binary string, serialized report message.
431            None if web is not enabled.
432        """
433        if not self.enabled:
434            return None
435
436        for test in requested[len(executed):]:
437            msg = self.report_msg.test_case.add()
438            msg.name = test.test_name
439            msg.start_timestamp = feature_utils.GetTimestamp()
440            msg.end_timestamp = msg.start_timestamp
441            msg.test_result = ReportMsg.TEST_CASE_RESULT_FAIL
442
443        self.report_msg.end_timestamp = feature_utils.GetTimestamp()
444
445        build = getattr(self, keys.ConfigKeys.IKEY_BUILD)
446        if keys.ConfigKeys.IKEY_BUILD_ID in build:
447            build_id = str(build[keys.ConfigKeys.IKEY_BUILD_ID])
448            self.report_msg.build_info.id = build_id
449
450        logging.debug("_tearDownClass hook: start (username: %s)",
451                      getpass.getuser())
452
453        if len(self.report_msg.test_case) == 0:
454            logging.warn("_tearDownClass hook: skip uploading (no test case)")
455            return ''
456
457        post_msg = ReportMsg.DashboardPostMessage()
458        post_msg.test_report.extend([self.report_msg])
459
460        self.rest_client.AddAuthToken(post_msg)
461
462        message_b = base64.b64encode(post_msg.SerializeToString())
463
464        logging.debug('Result proto message generated. size: %s',
465                      len(message_b))
466
467        logging.debug("_tearDownClass hook: status upload time stamp %s",
468                      str(self.report_msg.start_timestamp))
469
470        return message_b