1#
2# Copyright (C) 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16
17from future import standard_library
18standard_library.install_aliases()
19
20import copy
21import importlib
22import inspect
23import logging
24import os
25import pkgutil
26import signal
27import sys
28import threading
29
30from vts.runners.host import base_test
31from vts.runners.host import config_parser
32from vts.runners.host import keys
33from vts.runners.host import logger
34from vts.runners.host import records
35from vts.runners.host import signals
36from vts.runners.host import utils
37from vts.utils.python.common import timeout_utils
38from vts.utils.python.instrumentation import test_framework_instrumentation as tfi
39
40
41def main():
42    """Execute the test class in a test module.
43
44    This is to be used in a test script's main so the script can be executed
45    directly. It will discover all the classes that inherit from BaseTestClass
46    and excute them. all the test results will be aggregated into one.
47
48    A VTS host-driven test case has three args:
49       1st arg: the path of a test case config file.
50       2nd arg: the serial ID of a target device (device config).
51       3rd arg: the path of a test case data dir.
52
53    Returns:
54        The TestResult object that holds the results of the test run.
55    """
56    event = tfi.Begin('Test runner main method')
57    test_classes = []
58    main_module_members = sys.modules["__main__"]
59    for _, module_member in main_module_members.__dict__.items():
60        if inspect.isclass(module_member):
61            if issubclass(module_member, base_test.BaseTestClass):
62                test_classes.append(module_member)
63    # TODO(angli): Need to handle the case where more than one test class is in
64    # a test script. The challenge is to handle multiple configs and how to do
65    # default config in this case.
66    if len(test_classes) != 1:
67        logging.error("Expected 1 test class per file, found %s (%s).",
68                      len(test_classes), test_classes)
69        sys.exit(1)
70    test_result = runTestClass(test_classes[0])
71    event.End()
72    tfi.CompileResults()
73    return test_result
74
75
76def runTestClass(test_class):
77    """Execute one test class.
78
79    This will create a TestRunner, execute one test run with one test class.
80
81    Args:
82        test_class: The test class to instantiate and execute.
83
84    Returns:
85        The TestResult object that holds the results of the test run.
86    """
87    test_cls_name = test_class.__name__
88    if len(sys.argv) < 2:
89        logging.warning("Missing a configuration file. Using the default.")
90        test_configs = [config_parser.GetDefaultConfig(test_cls_name)]
91    else:
92        try:
93            config_path = sys.argv[1]
94            baseline_config = config_parser.GetDefaultConfig(test_cls_name)
95            baseline_config[keys.ConfigKeys.KEY_TESTBED] = [
96                baseline_config[keys.ConfigKeys.KEY_TESTBED]
97            ]
98            test_configs = config_parser.load_test_config_file(
99                config_path, baseline_config=baseline_config)
100        except IndexError:
101            logging.error("No valid config file found.")
102            sys.exit(1)
103        except Exception as e:
104            logging.error("Unexpected exception")
105            logging.exception(e)
106
107    test_identifiers = [(test_cls_name, None)]
108
109    for config in test_configs:
110        def watchStdin():
111            while True:
112                line = sys.stdin.readline()
113                if not line:
114                    break
115            utils.stop_current_process(base_test.TIMEOUT_SECS_TEARDOWN_CLASS)
116
117        watcher_thread = threading.Thread(target=watchStdin, name="watchStdin")
118        watcher_thread.daemon = True
119        watcher_thread.start()
120
121        tr = TestRunner(config, test_identifiers)
122        tr.parseTestConfig(config)
123        try:
124            tr.runTestClass(test_class, None)
125        except (signals.TestAbortAll, KeyboardInterrupt) as e:
126            logging.error("Abort all test")
127        except Exception as e:
128            logging.error("Unexpected exception")
129            logging.exception(e)
130        finally:
131            tr.stop()
132            return tr.results
133
134
135class TestRunner(object):
136    """The class that instantiates test classes, executes test cases, and
137    report results.
138
139    Attributes:
140        test_run_info: A dictionary containing the information needed by
141                       test classes for this test run, including params,
142                       controllers, and other objects. All of these will
143                       be passed to test classes.
144        test_configs: A dictionary that is the original test configuration
145                      passed in by user.
146        id: A string that is the unique identifier of this test run.
147        log_path: A string representing the path of the dir under which
148                  all logs from this test run should be written.
149        controller_registry: A dictionary that holds the controller
150                             objects used in a test run.
151        controller_destructors: A dictionary that holds the controller
152                                distructors. Keys are controllers' names.
153        run_list: A list of tuples specifying what tests to run.
154        results: The test result object used to record the results of
155                 this test run.
156        running: A boolean signifies whether this test run is ongoing or
157                 not.
158        test_cls_instances: list of test class instances that were executed
159                            or scheduled to be executed.
160        log_severity: string, log severity level for the test logger.
161                      Currently, this parameter only affects how logs are displayed
162                      to the console, and is not recommended to be used.
163    """
164
165    def __init__(self, test_configs, run_list):
166        self.test_run_info = {}
167        self.test_run_info[keys.ConfigKeys.IKEY_DATA_FILE_PATH] = getattr(
168            test_configs, keys.ConfigKeys.IKEY_DATA_FILE_PATH, "./")
169        self.test_configs = test_configs
170        self.testbed_configs = self.test_configs[keys.ConfigKeys.KEY_TESTBED]
171        self.testbed_name = self.testbed_configs[
172            keys.ConfigKeys.KEY_TESTBED_NAME]
173        start_time = logger.getLogFileTimestamp()
174        self.id = "{}@{}".format(self.testbed_name, start_time)
175        # log_path should be set before parsing configs.
176        l_path = os.path.join(self.test_configs[keys.ConfigKeys.KEY_LOG_PATH],
177                              self.testbed_name, start_time)
178        self.log_path = os.path.abspath(l_path)
179        self.log_severity = self.test_configs.get(
180            keys.ConfigKeys.KEY_LOG_SEVERITY, "INFO").upper()
181        logger.setupTestLogger(
182            self.log_path,
183            self.testbed_name,
184            filename="test_run_details.txt",
185            log_severity=self.log_severity)
186        self.controller_registry = {}
187        self.controller_destructors = {}
188        self.run_list = run_list
189        self.results = records.TestResult()
190        self.running = False
191        self.test_cls_instances = []
192
193    def __enter__(self):
194        return self
195
196    def __exit__(self, *args):
197        self.stop()
198
199    def importTestModules(self, test_paths):
200        """Imports test classes from test scripts.
201
202        1. Locate all .py files under test paths.
203        2. Import the .py files as modules.
204        3. Find the module members that are test classes.
205        4. Categorize the test classes by name.
206
207        Args:
208            test_paths: A list of directory paths where the test files reside.
209
210        Returns:
211            A dictionary where keys are test class name strings, values are
212            actual test classes that can be instantiated.
213        """
214
215        def is_testfile_name(name, ext):
216            if ext == ".py":
217                if name.endswith("Test") or name.endswith("_test"):
218                    return True
219            return False
220
221        file_list = utils.find_files(test_paths, is_testfile_name)
222        test_classes = {}
223        for path, name, _ in file_list:
224            sys.path.append(path)
225            try:
226                module = importlib.import_module(name)
227            except:
228                for test_cls_name, _ in self.run_list:
229                    alt_name = name.replace('_', '').lower()
230                    alt_cls_name = test_cls_name.lower()
231                    # Only block if a test class on the run list causes an
232                    # import error. We need to check against both naming
233                    # conventions: AaaBbb and aaa_bbb.
234                    if name == test_cls_name or alt_name == alt_cls_name:
235                        msg = ("Encountered error importing test class %s, "
236                               "abort.") % test_cls_name
237                        # This exception is logged here to help with debugging
238                        # under py2, because "raise X from Y" syntax is only
239                        # supported under py3.
240                        logging.exception(msg)
241                        raise USERError(msg)
242                continue
243            for member_name in dir(module):
244                if not member_name.startswith("__"):
245                    if member_name.endswith("Test"):
246                        test_class = getattr(module, member_name)
247                        if inspect.isclass(test_class):
248                            test_classes[member_name] = test_class
249        return test_classes
250
251    def verifyControllerModule(self, module):
252        """Verifies a module object follows the required interface for
253        controllers.
254
255        Args:
256            module: An object that is a controller module. This is usually
257                    imported with import statements or loaded by importlib.
258
259        Raises:
260            ControllerError is raised if the module does not match the vts.runners.host
261            controller interface, or one of the required members is null.
262        """
263        required_attributes = ("create", "destroy",
264                               "VTS_CONTROLLER_CONFIG_NAME")
265        for attr in required_attributes:
266            if not hasattr(module, attr):
267                raise signals.ControllerError(
268                    ("Module %s missing required "
269                     "controller module attribute %s.") % (module.__name__,
270                                                           attr))
271            if not getattr(module, attr):
272                raise signals.ControllerError(
273                    ("Controller interface %s in %s "
274                     "cannot be null.") % (attr, module.__name__))
275
276    def registerController(self, module, start_services=True):
277        """Registers a controller module for a test run.
278
279        This declares a controller dependency of this test class. If the target
280        module exists and matches the controller interface, the controller
281        module will be instantiated with corresponding configs in the test
282        config file. The module should be imported first.
283
284        Params:
285            module: A module that follows the controller module interface.
286            start_services: boolean, controls whether services (e.g VTS agent)
287                            are started on the target.
288
289        Returns:
290            A list of controller objects instantiated from controller_module.
291
292        Raises:
293            ControllerError is raised if no corresponding config can be found,
294            or if the controller module has already been registered.
295        """
296        event = tfi.Begin('test_runner registerController',
297                          tfi.categories.FRAMEWORK_SETUP)
298        logging.debug("cwd: %s", os.getcwd())
299        logging.info("adb devices: %s", module.list_adb_devices())
300        self.verifyControllerModule(module)
301        module_ref_name = module.__name__.split('.')[-1]
302        if module_ref_name in self.controller_registry:
303            event.End()
304            raise signals.ControllerError(
305                ("Controller module %s has already "
306                 "been registered. It can not be "
307                 "registered again.") % module_ref_name)
308        # Create controller objects.
309        create = module.create
310        module_config_name = module.VTS_CONTROLLER_CONFIG_NAME
311        if module_config_name not in self.testbed_configs:
312            msg = "No corresponding config found for %s" % module_config_name
313            event.Remove(msg)
314            raise signals.ControllerError(msg)
315        try:
316            # Make a deep copy of the config to pass to the controller module,
317            # in case the controller module modifies the config internally.
318            original_config = self.testbed_configs[module_config_name]
319            controller_config = copy.deepcopy(original_config)
320            # Add log_severity config to device controller config.
321            if isinstance(controller_config, list):
322                for config in controller_config:
323                    if isinstance(config, dict):
324                        config["log_severity"] = self.log_severity
325            logging.debug("controller_config: %s", controller_config)
326            if "use_vts_agent" not in self.testbed_configs:
327                objects = create(controller_config, start_services)
328            else:
329                objects = create(controller_config,
330                                 self.testbed_configs["use_vts_agent"])
331        except:
332            msg = "Failed to initialize objects for controller %s, abort!" % module_config_name
333            event.Remove(msg)
334            logging.error(msg)
335            raise
336        if not isinstance(objects, list):
337            msg = "Controller module %s did not return a list of objects, abort." % module_ref_name
338            event.Remove(msg)
339            raise signals.ControllerError(msg)
340
341        self.controller_registry[module_ref_name] = objects
342        logging.debug("Found %d objects for controller %s",
343                      len(objects), module_config_name)
344        destroy_func = module.destroy
345        self.controller_destructors[module_ref_name] = destroy_func
346        event.End()
347        return objects
348
349    @timeout_utils.timeout(base_test.TIMEOUT_SECS_TEARDOWN_CLASS,
350                           message='unregisterControllers method timed out.',
351                           no_exception=True)
352    def unregisterControllers(self):
353        """Destroy controller objects and clear internal registry.
354
355        This will be called at the end of each TestRunner.run call.
356        """
357        event = tfi.Begin('test_runner unregisterControllers',
358                          tfi.categories.FRAMEWORK_TEARDOWN)
359        for name, destroy in self.controller_destructors.items():
360            try:
361                logging.debug("Destroying %s.", name)
362                dut = self.controller_destructors[name]
363                destroy(self.controller_registry[name])
364            except:
365                logging.exception("Exception occurred destroying %s.", name)
366        self.controller_registry = {}
367        self.controller_destructors = {}
368        event.End()
369
370    def parseTestConfig(self, test_configs):
371        """Parses the test configuration and unpacks objects and parameters
372        into a dictionary to be passed to test classes.
373
374        Args:
375            test_configs: A json object representing the test configurations.
376        """
377        self.test_run_info[
378            keys.ConfigKeys.IKEY_TESTBED_NAME] = self.testbed_name
379        # Unpack other params.
380        self.test_run_info["registerController"] = self.registerController
381        self.test_run_info[keys.ConfigKeys.IKEY_LOG_PATH] = self.log_path
382        user_param_pairs = []
383        for item in test_configs.items():
384            if item[0] not in keys.ConfigKeys.RESERVED_KEYS:
385                user_param_pairs.append(item)
386        self.test_run_info[keys.ConfigKeys.IKEY_USER_PARAM] = copy.deepcopy(
387            dict(user_param_pairs))
388
389    def runTestClass(self, test_cls, test_cases=None):
390        """Instantiates and executes a test class.
391
392        If test_cases is None, the test cases listed by self.tests will be
393        executed instead. If self.tests is empty as well, no test case in this
394        test class will be executed.
395
396        Args:
397            test_cls: The test class to be instantiated and executed.
398            test_cases: List of test case names to execute within the class.
399
400        Returns:
401            A tuple, with the number of cases passed at index 0, and the total
402            number of test cases at index 1.
403        """
404        self.running = True
405        with test_cls(self.test_run_info) as test_cls_instance:
406            try:
407                if test_cls_instance not in self.test_cls_instances:
408                    self.test_cls_instances.append(test_cls_instance)
409                cls_result = test_cls_instance.run(test_cases)
410            finally:
411                self.unregisterControllers()
412
413    def run(self):
414        """Executes test cases.
415
416        This will instantiate controller and test classes, and execute test
417        classes. This can be called multiple times to repeatly execute the
418        requested test cases.
419
420        A call to TestRunner.stop should eventually happen to conclude the life
421        cycle of a TestRunner.
422
423        Args:
424            test_classes: A dictionary where the key is test class name, and
425                          the values are actual test classes.
426        """
427        if not self.running:
428            self.running = True
429        # Initialize controller objects and pack appropriate objects/params
430        # to be passed to test class.
431        self.parseTestConfig(self.test_configs)
432        t_configs = self.test_configs[keys.ConfigKeys.KEY_TEST_PATHS]
433        test_classes = self.importTestModules(t_configs)
434        logging.debug("Executing run list %s.", self.run_list)
435        try:
436            for test_cls_name, test_case_names in self.run_list:
437                if not self.running:
438                    break
439                if test_case_names:
440                    logging.debug("Executing test cases %s in test class %s.",
441                                  test_case_names, test_cls_name)
442                else:
443                    logging.debug("Executing test class %s", test_cls_name)
444                try:
445                    test_cls = test_classes[test_cls_name]
446                except KeyError:
447                    raise USERError(
448                        ("Unable to locate class %s in any of the test "
449                         "paths specified.") % test_cls_name)
450                try:
451                    self.runTestClass(test_cls, test_case_names)
452                except signals.TestAbortAll as e:
453                    logging.warning(
454                        ("Abort all subsequent test classes. Reason: "
455                         "%s"), e)
456                    raise
457        except Exception as e:
458            logging.error("Unexpected exception")
459            logging.exception(e)
460        finally:
461            self.unregisterControllers()
462
463    def stop(self):
464        """Releases resources from test run. Should always be called after
465        TestRunner.run finishes.
466
467        This function concludes a test run and writes out a test report.
468        """
469        if self.running:
470
471            for test_cls_instance in self.test_cls_instances:
472                self.results += test_cls_instance.results
473
474            msg = "\nSummary for test run %s: %s\n" % (self.id,
475                                                       self.results.summary())
476            self._writeResultsJsonString()
477            logging.info(msg.strip())
478            logger.killTestLogger(logging.getLogger())
479            self.running = False
480
481    def _writeResultsJsonString(self):
482        """Writes out a json file with the test result info for easy parsing.
483        """
484        path = os.path.join(self.log_path, "test_run_summary.json")
485        with open(path, 'w') as f:
486            f.write(self.results.jsonString())
487