1#
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the 'License');
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an 'AS IS' BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17import itertools
18import logging
19import os
20import zipfile
21
22from host_controller import common
23from host_controller.command_processor import base_command_processor
24from host_controller.utils.gcp import gcs_utils
25from host_controller.utils.parser import xml_utils
26
27from vts.utils.python.common import cmd_utils
28
29# The command list for cleaning up each devices listed for the retry command.
30_DEVICE_CLEANUP_COMMAND_LIST = [
31    "adb -s {serial} reboot bootloader",
32    "fastboot -s {serial} erase metadata -- -w",
33    "fastboot -s {serial} reboot",
34    "adb -s {serial} wait-for-device",
35    "dut --operation=wifi_on --serial={serial} --ap=" +
36    common._DEFAULT_WIFI_AP,
37]
38
39
40class CommandRetry(base_command_processor.BaseCommandProcessor):
41    """Command processor for retry command."""
42
43    command = "retry"
44    command_detail = "Retry last run test plan for certain times."
45
46    def IsResultZipFile(self, zip_ref):
47        """Determines whether the given zip_ref is the right result archive.
48
49        Need to check the number of contents of the zip file since
50        the "log-result_<>.zip" file only contains "test_result.xml",
51        but cannot parsed by vts-tf properly when trying to do the retry.
52
53        Args:
54            zip_ref: ZipFile, reference to the downloaded results_<>.zip file
55
56        Returns:
57            True if the downloaded zip file is usable from vts-fs,
58            False otherwise.
59        """
60        if len(zip_ref.namelist()) > 1:
61            for name in zip_ref.namelist():
62                if common._TEST_RESULT_XML in name:
63                    return True
64        return False
65
66    def GetResultFromGCS(self, gcs_result_path, local_results_dir):
67        """Downloads a vts-tf result zip archive from GCS.
68
69        And unzip the file to "android-vts/results/" path so
70        the vts-tf will parse the result correctly.
71
72        Args:
73            gcs_result_path: string, path to GCS file.
74            local_results_dir: string, abs path to the result directory of
75                               currently running vts-tf.
76        Returns:
77            A string which is the name of unzipped result directory.
78            None if the download has failed or the downloaded zip file
79            is not a correct result archive.
80        """
81        gsutil_path = gcs_utils.GetGsutilPath()
82        if not gsutil_path:
83            logging.error("Please check gsutil is installed and on your PATH")
84            return None
85
86        if (not gcs_result_path.startswith("gs://")
87                or not gcs_utils.IsGcsFile(gsutil_path, gcs_result_path)):
88            logging.error("%s is not correct GCS url.", gcs_result_path)
89            return None
90        if not gcs_result_path.endswith(".zip"):
91            logging.error("%s is not a correct result archive file.",
92                          gcs_result_path)
93            return None
94
95        if not os.path.exists(local_results_dir):
96            os.mkdir(local_results_dir)
97        if not gcs_utils.Copy(gsutil_path, gcs_result_path, local_results_dir):
98            logging.error("Fail to copy from %s.", gcs_result_path)
99            return None
100        result_zip = os.path.join(local_results_dir,
101                                  gcs_result_path.split("/")[-1])
102        with zipfile.ZipFile(result_zip, mode="r") as zip_ref:
103            if self.IsResultZipFile(zip_ref):
104                unzipped_result_dir = zip_ref.namelist()[0].rstrip("/")
105                zip_ref.extractall(local_results_dir)
106                return unzipped_result_dir
107            else:
108                logging.error("Not a correct vts-tf result archive file.")
109                return None
110
111    # @Override
112    def SetUp(self):
113        """Initializes the parser for retry command."""
114        self.arg_parser.add_argument(
115            "--suite",
116            default="vts",
117            choices=("vts", "cts", "gts", "sts"),
118            help="To specify the type of a test suite to be run.")
119        self.arg_parser.add_argument(
120            "--count",
121            type=int,
122            default=common.DEFAULT_RETRY_COUNT,
123            help="Retry count. Default retry count is %s." %
124            common.DEFAULT_RETRY_COUNT)
125        self.arg_parser.add_argument(
126            "--force-count",
127            type=int,
128            default=3,
129            help="Forced retry count. Retry certain test plan for the given "
130            "times whether all testcases has passed or not.")
131        self.arg_parser.add_argument(
132            "--result-from-gcs",
133            help="Google Cloud Storage URL from which the result is downloaded. "
134            "Will retry based on the fetched result data")
135        self.arg_parser.add_argument(
136            "--serial",
137            action="append",
138            default=[],
139            help="Serial number for device. Can pass this flag multiple times."
140        )
141        self.arg_parser.add_argument(
142            "--shards", type=int, help="Test plan's shard count.")
143        self.arg_parser.add_argument(
144            "--shard-count",
145            type=int,
146            help=
147            "Test plan's shard count. Same as the \"--shards\" flag but the "
148            "value will be passed to the tradefed with \"--shard-count\" flag."
149        )
150        self.arg_parser.add_argument(
151            "--cleanup_devices",
152            default=False,
153            type=bool,
154            help="True to erase metadata and userdata (equivalent to "
155            "factory reset) between retries.")
156        self.arg_parser.add_argument(
157            "--retry_plan", help="The name of a retry plan to use.")
158
159    # @Override
160    def Run(self, arg_line):
161        """Retry last run plan for certain times."""
162        args = self.arg_parser.ParseLine(arg_line)
163        retry_count = args.count
164        force_retry_count = args.force_count
165
166        if args.suite not in self.console.test_suite_info:
167            logging.error("test_suite_info doesn't have '%s': %s", args.suite,
168                          self.console.test_suite_info)
169            return False
170
171        tools_path = os.path.dirname(self.console.test_suite_info[args.suite])
172        results_path = os.path.join(tools_path, common._RESULTS_BASE_PATH)
173
174        unzipped_result_dir = ""
175        unzipped_result_session_id = -1
176        if args.result_from_gcs:
177            unzipped_result_dir = self.GetResultFromGCS(
178                args.result_from_gcs, results_path)
179            if not unzipped_result_dir:
180                return False
181
182        former_results = [
183            result for result in os.listdir(results_path)
184            if os.path.isdir(os.path.join(results_path, result))
185            and not os.path.islink(os.path.join(results_path, result))
186        ]
187        former_result_count = len(former_results)
188        if former_result_count < 1:
189            logging.error(
190                "No test plan has been run yet, former results count is %d",
191                former_result_count)
192            return False
193
194        if unzipped_result_dir:
195            former_results.sort()
196            unzipped_result_session_id = former_results.index(
197                unzipped_result_dir)
198
199        for result_index in range(retry_count):
200            if unzipped_result_session_id >= 0:
201                session_id = unzipped_result_session_id
202                unzipped_result_session_id = -1
203                latest_result_xml_path = os.path.join(
204                    results_path, unzipped_result_dir, common._TEST_RESULT_XML)
205            else:
206                session_id = former_result_count - 1 + result_index
207                latest_result_xml_path = os.path.join(results_path, "latest",
208                                                      common._TEST_RESULT_XML)
209                if not os.path.exists(latest_result_xml_path):
210                    latest_result_xml_path = os.path.join(
211                        results_path, former_results[-1],
212                        common._TEST_RESULT_XML)
213
214            result_attrs = xml_utils.GetAttributes(
215                latest_result_xml_path, common._RESULT_TAG,
216                [common._SUITE_PLAN_ATTR_KEY])
217
218            summary_attrs = xml_utils.GetAttributes(
219                latest_result_xml_path, common._SUMMARY_TAG, [
220                    common._FAILED_ATTR_KEY, common._MODULES_TOTAL_ATTR_KEY,
221                    common._MODULES_DONE_ATTR_KEY
222                ])
223
224            result_fail_count = int(summary_attrs[common._FAILED_ATTR_KEY])
225            result_skip_count = int(
226                summary_attrs[common._MODULES_TOTAL_ATTR_KEY]) - int(
227                    summary_attrs[common._MODULES_DONE_ATTR_KEY])
228
229            if (result_index >= force_retry_count and result_skip_count == 0
230                    and result_fail_count == 0):
231                logging.info("All modules have run and passed. "
232                             "Skipping remaining %d retry runs.",
233                             (retry_count - result_index))
234                break
235
236            shard_flag_literal = ""
237            if args.shards:
238                shard_flag_literal = "--shards"
239                shard_num = args.shards
240            if args.shard_count:
241                shard_flag_literal = "--shard-count"
242                shard_num = args.shard_count
243
244            if args.retry_plan:
245                retry_plan = args.retry_plan
246            else:
247                retry_plan = result_attrs[common._SUITE_PLAN_ATTR_KEY]
248            if shard_flag_literal:
249                retry_test_command = (
250                    "test --suite=%s --keep-result -- %s --retry %d %s %d" %
251                    (args.suite, retry_plan, session_id, shard_flag_literal,
252                     shard_num))
253            else:
254                retry_test_command = (
255                    "test --suite=%s --keep-result -- %s --retry %d" %
256                    (args.suite, retry_plan, session_id))
257            if args.serial:
258                for serial in args.serial:
259                    retry_test_command += " --serial %s" % serial
260
261            if args.cleanup_devices:
262                for (command, serial) in itertools.product(
263                        _DEVICE_CLEANUP_COMMAND_LIST, args.serial):
264                    if not self.console.onecmd(command.format(serial=serial)):
265                        logging.error(
266                            "Factory reset failed on the devices %s. "
267                            "Skipping retry run(s)", serial)
268                        self.console.device_status[
269                            serial] = common._DEVICE_STATUS_DICT["use"]
270                        return
271
272            self.console.onecmd(retry_test_command)
273
274            for result in os.listdir(results_path):
275                new_result = os.path.join(results_path, result)
276                if (os.path.isdir(new_result)
277                        and not os.path.islink(new_result)
278                        and result not in former_results):
279                    former_results.append(result)
280                    break
281
282            summary_after_retry = xml_utils.GetAttributes(
283                os.path.join(results_path, former_results[-1],
284                             common._TEST_RESULT_XML), common._SUMMARY_TAG,
285                [
286                    common._FAILED_ATTR_KEY, common._MODULES_TOTAL_ATTR_KEY,
287                    common._MODULES_DONE_ATTR_KEY
288                ])
289            fail_count_after_retry = int(
290                summary_after_retry[common._FAILED_ATTR_KEY])
291            skip_count_after_retry = int(
292                summary_after_retry[common._MODULES_TOTAL_ATTR_KEY]) - int(
293                    summary_after_retry[common._MODULES_DONE_ATTR_KEY])
294            if (result_index >= force_retry_count
295                    and fail_count_after_retry == result_fail_count
296                    and skip_count_after_retry == result_skip_count):
297                logging.warning(
298                    "Same result as the former test run from the retry run. "
299                    "Skipping remaining %d retry runs.",
300                    (retry_count - result_index))
301                break
302