1#
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the 'License');
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an 'AS IS' BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import logging
18import os
19import shutil
20import socket
21import time
22
23from host_controller import common
24from host_controller.command_processor import base_command_processor
25from host_controller.utils.gcp import gcs_utils
26from host_controller.utils.parser import xml_utils
27
28from vts.utils.python.common import cmd_utils
29
30from vti.dashboard.proto import TestSuiteResultMessage_pb2 as SuiteResMsg
31from vti.test_serving.proto import TestScheduleConfigMessage_pb2 as SchedCfgMsg
32
33
34class CommandUpload(base_command_processor.BaseCommandProcessor):
35    """Command processor for upload command.
36
37    Attributes:
38        arg_parser: ConsoleArgumentParser object, argument parser.
39        console: cmd.Cmd console object.
40        command: string, command name which this processor will handle.
41        command_detail: string, detailed explanation for the command.
42    """
43
44    command = "upload"
45    command_detail = "Upload <src> file to <dest> Google Cloud Storage. In <src> and <dest>, variables enclosed in {} are replaced with the values stored in the console."
46
47    # @Override
48    def SetUp(self):
49        """Initializes the parser for upload command."""
50        self.arg_parser.add_argument(
51            "--src",
52            required=True,
53            default="latest-system.img",
54            help="Path to a source file to upload. Only single file can be "
55            "uploaded per once. Use 'latest- prefix to upload the latest "
56            "fetch images. e.g. --src=latest-system.img  If argument "
57            "value is not given, the recently fetched system.img will be "
58            "uploaded.")
59        self.arg_parser.add_argument(
60            "--dest",
61            required=True,
62            help="Google Cloud Storage URL to which the file is uploaded.")
63        self.arg_parser.add_argument(
64            "--report_path",
65            help="Google Cloud Storage URL, the dest path of a report file")
66        self.arg_parser.add_argument(
67            "--clear_dest",
68            action="store_true",
69            help="Delete dest recursively before the upload.")
70        self.arg_parser.add_argument(
71            "--clear_results",
72            default=False,
73            help="True to clear all the results after the upload.")
74        self.arg_parser.add_argument(
75            "--result_from_suite",
76            default="",
77            choices=("", "vts", "cts", "gts", "sts"),
78            help="To specify the type of a test suite report, since there can "
79            "be multiple numbers of result sets from different test "
80            "suites. If not specified, the HC will upload the report "
81            "from last run suite and plan.")
82        self.arg_parser.add_argument(
83            "--result_from_plan",
84            default="",
85            help="To specify the type of the plan name from which "
86            "the report is generated.")
87
88    # @Override
89    def Run(self, arg_line):
90        """Upload args.src file to args.dest Google Cloud Storage."""
91        args = self.arg_parser.ParseLine(arg_line)
92
93        gsutil_path = gcs_utils.GetGsutilPath()
94        if not gsutil_path:
95            logging.error("Please check gsutil is installed and on your PATH")
96            return False
97
98        if args.src.startswith("latest-"):
99            src_name = args.src[7:]
100            if src_name in self.console.device_image_info:
101                src_paths = self.console.device_image_info[src_name]
102            else:
103                logging.error(
104                    "Unable to find {} in device_image_info".format(src_name))
105                return False
106        else:
107            try:
108                src_paths = self.console.FormatString(args.src)
109            except KeyError as e:
110                logging.error("Unknown or uninitialized variable in src: %s",
111                              e)
112                return False
113
114        src_path_list_tmp = src_paths.split(" ")
115        src_path_list = []
116        if src_path_list_tmp:
117            for src_path in src_path_list_tmp:
118                file_path = src_path.strip()
119                if os.path.isfile(file_path):
120                    src_path_list.append(file_path)
121                else:
122                    logging.error("Cannot find a file: {}".format(file_path))
123        src_paths = " ".join(src_path_list)
124
125        try:
126            dest_path = self.console.FormatString(args.dest)
127        except KeyError as e:
128            logging.error("Unknown or uninitialized variable in dest: %s", e)
129            return False
130
131        if not dest_path.startswith("gs://"):
132            logging.error("{} is not correct GCS url.".format(dest_path))
133            return False
134        """ TODO(jongmok) : Before upload, login status, authorization,
135                            and dest check are required. """
136        if args.clear_dest:
137            if not gcs_utils.Remove(gsutil_path, dest_path, recursive=True):
138                logging.error("Fail to remove %s", dest_path)
139
140        if not gcs_utils.Copy(gsutil_path, src_paths, dest_path):
141            logging.error("Fail to copy %s to %s", src_paths, dest_path)
142
143        if args.report_path or args.clear_results:
144            tools_path = ""
145            if args.result_from_suite:
146                tools_path = os.path.dirname(
147                    self.console.test_suite_info[args.result_from_suite])
148            else:
149                try:
150                    tools_path = os.path.dirname(self.console.test_suite_info[
151                        self.console.FormatString("{suite_name}")])
152                except KeyError:
153                    if self.console.vti_endpoint_client.CheckBootUpStatus():
154                        logging.error(
155                            "No test results found from any fetched test suite."
156                            " Please fetch a test suite and run 'test' command,"
157                            " then try running 'upload' command again.")
158                        return False
159            results_base_path = os.path.join(tools_path,
160                                             common._RESULTS_BASE_PATH)
161
162            if args.report_path:
163                report_path = self.console.FormatString(args.report_path)
164                if not report_path.startswith("gs://"):
165                    logging.error(
166                        "{} is not correct GCS url.".format(report_path))
167                else:
168                    self.UploadReport(
169                        gsutil_path, report_path, dest_path, results_base_path,
170                        args.result_from_suite, args.result_from_plan)
171
172            if args.clear_results:
173                shutil.rmtree(results_base_path, ignore_errors=True)
174
175    def UploadReport(self, gsutil_path, report_path, log_path, results_path,
176                     suite_name, plan_name):
177        """Uploads report summary file to the given path.
178
179        Args:
180            gsutil_path: string, the path of a gsutil binary.
181            report_path: string, the dest GCS URL to which the summarized report
182                                 file will be uploaded.
183            log_path: string, GCS URL where the log files from the test run
184                              have been uploaded.
185            results_path: string, the base path for the results.
186        """
187        suite_res_msg = SuiteResMsg.TestSuiteResultMessage()
188        suite_res_msg.result_path = log_path
189        suite_res_msg.branch = self.console.FormatString("{branch}")
190        suite_res_msg.target = self.console.FormatString("{target}")
191        vti = self.console.vti_endpoint_client
192        suite_res_msg.boot_success = vti.CheckBootUpStatus()
193        suite_res_msg.test_type = vti.GetJobTestType()
194
195        device_fetch_info = self.console.detailed_fetch_info[
196            common._ARTIFACT_TYPE_DEVICE]
197        gsi_fetch_info = None
198        if common._ARTIFACT_TYPE_GSI in self.console.detailed_fetch_info:
199            gsi_fetch_info = self.console.detailed_fetch_info[
200                common._ARTIFACT_TYPE_GSI]
201
202        if vti.CheckBootUpStatus():
203            former_results = [
204                result for result in os.listdir(results_path)
205                if os.path.isdir(os.path.join(results_path, result))
206                and not os.path.islink(os.path.join(results_path, result))
207            ]
208
209            if not former_results:
210                logging.error("No test result found.")
211                return False
212
213            former_results.sort()
214            latest_result = former_results[-1]
215            latest_result_xml_path = os.path.join(results_path, latest_result,
216                                                  common._TEST_RESULT_XML)
217
218            result_attrs = xml_utils.GetAttributes(
219                latest_result_xml_path, common._RESULT_TAG, [
220                    common._SUITE_NAME_ATTR_KEY, common._SUITE_PLAN_ATTR_KEY,
221                    common._SUITE_VERSION_ATTR_KEY,
222                    common._SUITE_BUILD_NUM_ATTR_KEY,
223                    common._START_TIME_ATTR_KEY, common._END_TIME_ATTR_KEY,
224                    common._HOST_NAME_ATTR_KEY
225                ])
226            build_attrs = xml_utils.GetAttributes(
227                latest_result_xml_path, common._BUILD_TAG, [
228                    common._FINGERPRINT_ATTR_KEY,
229                    common._SYSTEM_FINGERPRINT_ATTR_KEY,
230                    common._VENDOR_FINGERPRINT_ATTR_KEY
231                ])
232            summary_attrs = xml_utils.GetAttributes(
233                latest_result_xml_path, common._SUMMARY_TAG, [
234                    common._PASSED_ATTR_KEY, common._FAILED_ATTR_KEY,
235                    common._MODULES_TOTAL_ATTR_KEY,
236                    common._MODULES_DONE_ATTR_KEY
237                ])
238
239            suite_res_msg.build_id = result_attrs[
240                common._SUITE_BUILD_NUM_ATTR_KEY]
241            suite_res_msg.suite_name = result_attrs[
242                common._SUITE_NAME_ATTR_KEY]
243            suite_res_msg.suite_plan = result_attrs[
244                common._SUITE_PLAN_ATTR_KEY]
245            suite_res_msg.suite_version = result_attrs[
246                common._SUITE_VERSION_ATTR_KEY]
247            suite_res_msg.suite_build_number = result_attrs[
248                common._SUITE_BUILD_NUM_ATTR_KEY]
249            suite_res_msg.start_time = long(
250                result_attrs[common._START_TIME_ATTR_KEY])
251            suite_res_msg.end_time = long(
252                result_attrs[common._END_TIME_ATTR_KEY])
253            suite_res_msg.host_name = result_attrs[common._HOST_NAME_ATTR_KEY]
254            if common._SYSTEM_FINGERPRINT_ATTR_KEY in build_attrs:
255                suite_res_msg.build_system_fingerprint = build_attrs[
256                    common._SYSTEM_FINGERPRINT_ATTR_KEY]
257            else:
258                suite_res_msg.build_system_fingerprint = build_attrs[
259                    common._FINGERPRINT_ATTR_KEY]
260            if common._VENDOR_FINGERPRINT_ATTR_KEY in build_attrs:
261                suite_res_msg.build_vendor_fingerprint = build_attrs[
262                    common._VENDOR_FINGERPRINT_ATTR_KEY]
263            else:
264                suite_res_msg.build_vendor_fingerprint = build_attrs[
265                    common._FINGERPRINT_ATTR_KEY]
266            suite_res_msg.passed_test_case_count = int(
267                summary_attrs[common._PASSED_ATTR_KEY])
268            suite_res_msg.failed_test_case_count = int(
269                summary_attrs[common._FAILED_ATTR_KEY])
270            suite_res_msg.modules_done = int(
271                summary_attrs[common._MODULES_DONE_ATTR_KEY])
272            suite_res_msg.modules_total = int(
273                summary_attrs[common._MODULES_TOTAL_ATTR_KEY])
274        else:
275            suite_res_msg.build_id = self.console.fetch_info["build_id"]
276            suite_res_msg.suite_name = suite_name
277            suite_res_msg.suite_plan = plan_name
278            suite_res_msg.suite_version = ""
279            suite_res_msg.suite_build_number = suite_res_msg.build_id
280            suite_res_msg.start_time = long(time.time() * 1000)
281            suite_res_msg.end_time = suite_res_msg.start_time
282            suite_res_msg.host_name = socket.gethostname()
283            suite_res_msg.build_vendor_fingerprint = "%s/%s/%s" % (
284                device_fetch_info["branch"], device_fetch_info["target"],
285                device_fetch_info["build_id"])
286            if gsi_fetch_info:
287                suite_res_msg.build_system_fingerprint = "%s/%s/%s" % (
288                    gsi_fetch_info["branch"], gsi_fetch_info["target"],
289                    gsi_fetch_info["build_id"])
290            else:
291                suite_res_msg.build_system_fingerprint = suite_res_msg.build_vendor_fingerprint
292            suite_res_msg.passed_test_case_count = 0
293            suite_res_msg.failed_test_case_count = 0
294            suite_res_msg.modules_done = 0
295            suite_res_msg.modules_total = 0
296
297        suite_res_msg.infra_log_path = self.console.FormatString(
298            "{hc_log_upload_path}")
299        repack_path_list = []
300        repack_path_list.append(self.console.FormatString("{repack_path}"))
301        suite_res_msg.repacked_image_path.extend(repack_path_list)
302
303        suite_res_msg.schedule_config.build_target.extend(
304            [SchedCfgMsg.BuildScheduleConfigMessage()])
305        build_target_msg = suite_res_msg.schedule_config.build_target[0]
306        build_target_msg.test_schedule.extend(
307            [SchedCfgMsg.TestScheduleConfigMessage()])
308        test_schedule_msg = build_target_msg.test_schedule[0]
309
310        suite_res_msg.vendor_build_id = device_fetch_info["build_id"]
311        suite_res_msg.schedule_config.manifest_branch = str(
312            device_fetch_info["branch"])
313        build_target_msg.name = str(device_fetch_info["target"])
314        if device_fetch_info["account_id"]:
315            suite_res_msg.schedule_config.pab_account_id = str(
316                device_fetch_info["account_id"])
317        if device_fetch_info["fetch_signed_build"]:
318            build_target_msg.require_signed_device_build = device_fetch_info[
319                "fetch_signed_build"]
320        if gsi_fetch_info:
321            test_schedule_msg.gsi_branch = str(gsi_fetch_info["branch"])
322            test_schedule_msg.gsi_build_target = str(gsi_fetch_info["target"])
323            suite_res_msg.gsi_build_id = str(gsi_fetch_info["build_id"])
324            if gsi_fetch_info["account_id"]:
325                test_schedule_msg.gsi_pab_account_id = str(
326                    gsi_fetch_info["account_id"])
327            test_schedule_msg.gsi_vendor_version = str(
328                self.console.FormatString("{gsispl.vendor_version}"))
329        test_schedule_msg.test_pab_account_id = str(
330            self.console.FormatString("{account_id}"))
331        build_target_msg.has_bootloader_img = "bootloader.img" in self.console.device_image_info
332        build_target_msg.has_radio_img = "radio.img" in self.console.device_image_info
333
334        report_file_path = os.path.join(
335            self.console.tmp_logdir,
336            self.console.FormatString("{timestamp_time}.bin"))
337        with open(report_file_path, "w") as fd:
338            fd.write(suite_res_msg.SerializeToString())
339            fd.close()
340
341        copy_command = "{} cp {} {}".format(
342            gsutil_path, report_file_path,
343            os.path.join(report_path, os.path.basename(report_file_path)))
344        _, stderr, err_code = cmd_utils.ExecuteOneShellCommand(copy_command)
345        if err_code:
346            logging.error(stderr)
347