1#!/usr/bin/env python3.4
2#
3#   Copyright 2016 - The Android Open Source Project
4#
5#   Licensed under the Apache License, Version 2.0 (the "License");
6#   you may not use this file except in compliance with the License.
7#   You may obtain a copy of the License at
8#
9#       http://www.apache.org/licenses/LICENSE-2.0
10#
11#   Unless required by applicable law or agreed to in writing, software
12#   distributed under the License is distributed on an "AS IS" BASIS,
13#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14#   See the License for the specific language governing permissions and
15#   limitations under the License.
16
17from future import standard_library
18
19standard_library.install_aliases()
20
21import copy
22import importlib
23import inspect
24import fnmatch
25import logging
26import os
27import pkgutil
28import sys
29
30from acts import base_test
31from acts import config_parser
32from acts import keys
33from acts import logger
34from acts import records
35from acts import signals
36from acts import utils
37
38
39def _find_test_class():
40    """Finds the test class in a test script.
41
42    Walk through module members and find the subclass of BaseTestClass. Only
43    one subclass is allowed in a test script.
44
45    Returns:
46        The test class in the test module.
47    """
48    test_classes = []
49    main_module_members = sys.modules["__main__"]
50    for _, module_member in main_module_members.__dict__.items():
51        if inspect.isclass(module_member):
52            if issubclass(module_member, base_test.BaseTestClass):
53                test_classes.append(module_member)
54    if len(test_classes) != 1:
55        logging.error("Expected 1 test class per file, found %s.",
56                      [t.__name__ for t in test_classes])
57        sys.exit(1)
58    return test_classes[0]
59
60
61def execute_one_test_class(test_class, test_config, test_identifier):
62    """Executes one specific test class.
63
64    You could call this function in your own cli test entry point if you choose
65    not to use act.py.
66
67    Args:
68        test_class: A subclass of acts.base_test.BaseTestClass that has the test
69                    logic to be executed.
70        test_config: A dict representing one set of configs for a test run.
71        test_identifier: A list of tuples specifying which test cases to run in
72                         the test class.
73
74    Returns:
75        True if all tests passed without any error, False otherwise.
76
77    Raises:
78        If signals.TestAbortAll is raised by a test run, pipe it through.
79    """
80    tr = TestRunner(test_config, test_identifier)
81    try:
82        tr.run(test_class)
83        return tr.results.is_all_pass
84    except signals.TestAbortAll:
85        raise
86    except:
87        logging.exception("Exception when executing %s.", tr.testbed_name)
88    finally:
89        tr.stop()
90
91
92class TestRunner(object):
93    """The class that instantiates test classes, executes test cases, and
94    report results.
95
96    Attributes:
97        self.test_run_info: A dictionary containing the information needed by
98                            test classes for this test run, including params,
99                            controllers, and other objects. All of these will
100                            be passed to test classes.
101        self.test_configs: A dictionary that is the original test configuration
102                           passed in by user.
103        self.id: A string that is the unique identifier of this test run.
104        self.log_path: A string representing the path of the dir under which
105                       all logs from this test run should be written.
106        self.log: The logger object used throughout this test run.
107        self.controller_registry: A dictionary that holds the controller
108                                  objects used in a test run.
109        self.test_classes: A dictionary where we can look up the test classes
110                           by name to instantiate. Supports unix shell style
111                           wildcards.
112        self.run_list: A list of tuples specifying what tests to run.
113        self.results: The test result object used to record the results of
114                      this test run.
115        self.running: A boolean signifies whether this test run is ongoing or
116                      not.
117    """
118
119    def __init__(self, test_configs, run_list):
120        self.test_run_info = {}
121        self.test_configs = test_configs
122        self.testbed_configs = self.test_configs[keys.Config.key_testbed.value]
123        self.testbed_name = self.testbed_configs[
124            keys.Config.key_testbed_name.value]
125        start_time = logger.get_log_file_timestamp()
126        self.id = "{}@{}".format(self.testbed_name, start_time)
127        # log_path should be set before parsing configs.
128        l_path = os.path.join(
129            self.test_configs[keys.Config.key_log_path.value],
130            self.testbed_name, start_time)
131        self.log_path = os.path.abspath(l_path)
132        logger.setup_test_logger(self.log_path, self.testbed_name)
133        self.log = logging.getLogger()
134        self.controller_registry = {}
135        if self.test_configs.get(keys.Config.key_random.value):
136            test_case_iterations = self.test_configs.get(
137                keys.Config.key_test_case_iterations.value, 10)
138            self.log.info(
139                "Campaign randomizer is enabled with test_case_iterations %s",
140                test_case_iterations)
141            self.run_list = config_parser.test_randomizer(
142                run_list, test_case_iterations=test_case_iterations)
143            self.write_test_campaign()
144        else:
145            self.run_list = run_list
146        self.results = records.TestResult()
147        self.running = False
148
149    def import_test_modules(self, test_paths):
150        """Imports test classes from test scripts.
151
152        1. Locate all .py files under test paths.
153        2. Import the .py files as modules.
154        3. Find the module members that are test classes.
155        4. Categorize the test classes by name.
156
157        Args:
158            test_paths: A list of directory paths where the test files reside.
159
160        Returns:
161            A dictionary where keys are test class name strings, values are
162            actual test classes that can be instantiated.
163        """
164
165        def is_testfile_name(name, ext):
166            if ext == ".py":
167                if name.endswith("Test") or name.endswith("_test"):
168                    return True
169            return False
170
171        file_list = utils.find_files(test_paths, is_testfile_name)
172        test_classes = {}
173        for path, name, _ in file_list:
174            sys.path.append(path)
175            try:
176                module = importlib.import_module(name)
177            except:
178                for test_cls_name, _ in self.run_list:
179                    alt_name = name.replace('_', '').lower()
180                    alt_cls_name = test_cls_name.lower()
181                    # Only block if a test class on the run list causes an
182                    # import error. We need to check against both naming
183                    # conventions: AaaBbb and aaa_bbb.
184                    if name == test_cls_name or alt_name == alt_cls_name:
185                        msg = ("Encountered error importing test class %s, "
186                               "abort.") % test_cls_name
187                        # This exception is logged here to help with debugging
188                        # under py2, because "raise X from Y" syntax is only
189                        # supported under py3.
190                        self.log.exception(msg)
191                        raise ValueError(msg)
192                continue
193            for member_name in dir(module):
194                if not member_name.startswith("__"):
195                    if member_name.endswith("Test"):
196                        test_class = getattr(module, member_name)
197                        if inspect.isclass(test_class):
198                            test_classes[member_name] = test_class
199        return test_classes
200
201    def _import_builtin_controllers(self):
202        """Import built-in controller modules.
203
204        Go through the testbed configs, find any built-in controller configs
205        and import the corresponding controller module from acts.controllers
206        package.
207
208        Returns:
209            A list of controller modules.
210        """
211        builtin_controllers = []
212        for ctrl_name in keys.Config.builtin_controller_names.value:
213            if ctrl_name in self.testbed_configs:
214                module_name = keys.get_module_name(ctrl_name)
215                module = importlib.import_module(
216                    "acts.controllers.%s" % module_name)
217                builtin_controllers.append(module)
218        return builtin_controllers
219
220    @staticmethod
221    def verify_controller_module(module):
222        """Verifies a module object follows the required interface for
223        controllers.
224
225        Args:
226            module: An object that is a controller module. This is usually
227                    imported with import statements or loaded by importlib.
228
229        Raises:
230            ControllerError is raised if the module does not match the ACTS
231            controller interface, or one of the required members is null.
232        """
233        required_attributes = ("create", "destroy",
234                               "ACTS_CONTROLLER_CONFIG_NAME")
235        for attr in required_attributes:
236            if not hasattr(module, attr):
237                raise signals.ControllerError(
238                    ("Module %s missing required "
239                     "controller module attribute %s.") % (module.__name__,
240                                                           attr))
241            if not getattr(module, attr):
242                raise signals.ControllerError(
243                    "Controller interface %s in %s cannot be null." %
244                    (attr, module.__name__))
245
246    @staticmethod
247    def get_module_reference_name(a_module):
248        """Returns the module's reference name.
249
250        This is largely for backwards compatibility with log parsing. If the
251        module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that
252        value, or the module's submodule name.
253
254        Args:
255            a_module: Any module. Ideally, a controller module.
256        Returns:
257            A string corresponding to the module's name.
258        """
259        if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'):
260            return a_module.ACTS_CONTROLLER_REFERENCE_NAME
261        else:
262            return a_module.__name__.split('.')[-1]
263
264    def register_controller(self,
265                            controller_module,
266                            required=True,
267                            builtin=False):
268        """Registers an ACTS controller module for a test run.
269
270        An ACTS controller module is a Python lib that can be used to control
271        a device, service, or equipment. To be ACTS compatible, a controller
272        module needs to have the following members:
273
274            def create(configs):
275                [Required] Creates controller objects from configurations.
276                Args:
277                    configs: A list of serialized data like string/dict. Each
278                             element of the list is a configuration for a
279                             controller object.
280                Returns:
281                    A list of objects.
282
283            def destroy(objects):
284                [Required] Destroys controller objects created by the create
285                function. Each controller object shall be properly cleaned up
286                and all the resources held should be released, e.g. memory
287                allocation, sockets, file handlers etc.
288                Args:
289                    A list of controller objects created by the create function.
290
291            def get_info(objects):
292                [Optional] Gets info from the controller objects used in a test
293                run. The info will be included in test_result_summary.json under
294                the key "ControllerInfo". Such information could include unique
295                ID, version, or anything that could be useful for describing the
296                test bed and debugging.
297                Args:
298                    objects: A list of controller objects created by the create
299                             function.
300                Returns:
301                    A list of json serializable objects, each represents the
302                    info of a controller object. The order of the info object
303                    should follow that of the input objects.
304            def get_post_job_info(controller_list):
305                [Optional] Returns information about the controller after the
306                test has run. This info is sent to test_run_summary.json's
307                "Extras" key.
308                Args:
309                    The list of controller objects created by the module
310                Returns:
311                    A (name, data) tuple.
312        Registering a controller module declares a test class's dependency the
313        controller. If the module config exists and the module matches the
314        controller interface, controller objects will be instantiated with
315        corresponding configs. The module should be imported first.
316
317        Args:
318            controller_module: A module that follows the controller module
319                interface.
320            required: A bool. If True, failing to register the specified
321                controller module raises exceptions. If False, returns None upon
322                failures.
323            builtin: Specifies that the module is a builtin controller module in
324                ACTS. If true, adds itself to test_run_info.
325        Returns:
326            A list of controller objects instantiated from controller_module, or
327            None.
328
329        Raises:
330            When required is True, ControllerError is raised if no corresponding
331            config can be found.
332            Regardless of the value of "required", ControllerError is raised if
333            the controller module has already been registered or any other error
334            occurred in the registration process.
335        """
336        TestRunner.verify_controller_module(controller_module)
337        module_ref_name = self.get_module_reference_name(controller_module)
338
339        if controller_module in self.controller_registry:
340            raise signals.ControllerError(
341                "Controller module %s has already been registered. It can not "
342                "be registered again." % module_ref_name)
343        # Create controller objects.
344        module_config_name = controller_module.ACTS_CONTROLLER_CONFIG_NAME
345        if module_config_name not in self.testbed_configs:
346            if required:
347                raise signals.ControllerError(
348                    "No corresponding config found for %s" %
349                    module_config_name)
350            else:
351                self.log.warning(
352                    "No corresponding config found for optional controller %s",
353                    module_config_name)
354            return None
355        try:
356            # Make a deep copy of the config to pass to the controller module,
357            # in case the controller module modifies the config internally.
358            original_config = self.testbed_configs[module_config_name]
359            controller_config = copy.deepcopy(original_config)
360            controllers = controller_module.create(controller_config)
361        except:
362            self.log.exception(
363                "Failed to initialize objects for controller %s, abort!",
364                module_config_name)
365            raise
366        if not isinstance(controllers, list):
367            raise signals.ControllerError(
368                "Controller module %s did not return a list of objects, abort."
369                % module_ref_name)
370        self.controller_registry[controller_module] = controllers
371        # Collect controller information and write to test result.
372        # Implementation of "get_info" is optional for a controller module.
373        if hasattr(controller_module, "get_info"):
374            controller_info = controller_module.get_info(controllers)
375            self.log.info("Controller %s: %s", module_config_name,
376                          controller_info)
377            self.results.add_controller_info(module_config_name,
378                                             controller_info)
379        else:
380            self.log.warning("No controller info obtained for %s",
381                             module_config_name)
382
383        # TODO(angli): After all tests move to register_controller, stop
384        # tracking controller objs in test_run_info.
385        if builtin:
386            self.test_run_info[module_ref_name] = controllers
387        self.log.debug("Found %d objects for controller %s", len(controllers),
388                       module_config_name)
389        return controllers
390
391    def unregister_controllers(self):
392        """Destroy controller objects and clear internal registry.
393
394        This will be called at the end of each TestRunner.run call.
395        """
396        for controller_module, controllers in self.controller_registry.items():
397            name = self.get_module_reference_name(controller_module)
398            if hasattr(controller_module, 'get_post_job_info'):
399                self.log.debug('Getting post job info for %s', name)
400                name, value = controller_module.get_post_job_info(controllers)
401                self.results.set_extra_data(name, value)
402            try:
403                self.log.debug('Destroying %s.', name)
404                controller_module.destroy(controllers)
405            except:
406                self.log.exception("Exception occurred destroying %s.", name)
407        self.controller_registry = {}
408
409    def parse_config(self, test_configs):
410        """Parses the test configuration and unpacks objects and parameters
411        into a dictionary to be passed to test classes.
412
413        Args:
414            test_configs: A json object representing the test configurations.
415        """
416        self.test_run_info[
417            keys.Config.ikey_testbed_name.value] = self.testbed_name
418        # Unpack other params.
419        self.test_run_info["register_controller"] = self.register_controller
420        self.test_run_info[keys.Config.ikey_logpath.value] = self.log_path
421        self.test_run_info[keys.Config.ikey_logger.value] = self.log
422        cli_args = test_configs.get(keys.Config.ikey_cli_args.value)
423        self.test_run_info[keys.Config.ikey_cli_args.value] = cli_args
424        user_param_pairs = []
425        for item in test_configs.items():
426            if item[0] not in keys.Config.reserved_keys.value:
427                user_param_pairs.append(item)
428        self.test_run_info[keys.Config.ikey_user_param.value] = copy.deepcopy(
429            dict(user_param_pairs))
430
431    def set_test_util_logs(self, module=None):
432        """Sets the log object to each test util module.
433
434        This recursively include all modules under acts.test_utils and sets the
435        main test logger to each module.
436
437        Args:
438            module: A module under acts.test_utils.
439        """
440        # Initial condition of recursion.
441        if not module:
442            module = importlib.import_module("acts.test_utils")
443        # Somehow pkgutil.walk_packages is not working for me.
444        # Using iter_modules for now.
445        pkg_iter = pkgutil.iter_modules(module.__path__, module.__name__ + '.')
446        for _, module_name, ispkg in pkg_iter:
447            m = importlib.import_module(module_name)
448            if ispkg:
449                self.set_test_util_logs(module=m)
450            else:
451                self.log.debug("Setting logger to test util module %s",
452                               module_name)
453                setattr(m, "log", self.log)
454
455    def run_test_class(self, test_cls_name, test_cases=None):
456        """Instantiates and executes a test class.
457
458        If test_cases is None, the test cases listed by self.tests will be
459        executed instead. If self.tests is empty as well, no test case in this
460        test class will be executed.
461
462        Args:
463            test_cls_name: Name of the test class to execute.
464            test_cases: List of test case names to execute within the class.
465
466        Raises:
467            ValueError is raised if the requested test class could not be found
468            in the test_paths directories.
469        """
470        matches = fnmatch.filter(self.test_classes.keys(), test_cls_name)
471        if not matches:
472            self.log.info(
473                "Cannot find test class %s or classes matching pattern, "
474                "skipping for now." % test_cls_name)
475            record = records.TestResultRecord("*all*", test_cls_name)
476            record.test_skip(signals.TestSkip("Test class does not exist."))
477            self.results.add_record(record)
478            return
479        if matches != [test_cls_name]:
480            self.log.info("Found classes matching pattern %s: %s",
481                          test_cls_name, matches)
482
483        for test_cls_name_match in matches:
484            test_cls = self.test_classes[test_cls_name_match]
485            if self.test_configs.get(keys.Config.key_random.value) or (
486                    "Preflight" in test_cls_name_match) or (
487                        "Postflight" in test_cls_name_match):
488                test_case_iterations = 1
489            else:
490                test_case_iterations = self.test_configs.get(
491                    keys.Config.key_test_case_iterations.value, 1)
492
493            with test_cls(self.test_run_info) as test_cls_instance:
494                try:
495                    cls_result = test_cls_instance.run(test_cases,
496                                                       test_case_iterations)
497                    self.results += cls_result
498                    self._write_results_json_str()
499                except signals.TestAbortAll as e:
500                    self.results += e.results
501                    raise e
502
503    def run(self, test_class=None):
504        """Executes test cases.
505
506        This will instantiate controller and test classes, and execute test
507        classes. This can be called multiple times to repeatedly execute the
508        requested test cases.
509
510        A call to TestRunner.stop should eventually happen to conclude the life
511        cycle of a TestRunner.
512
513        Args:
514            test_class: The python module of a test class. If provided, run this
515                        class; otherwise, import modules in under test_paths
516                        based on run_list.
517        """
518        if not self.running:
519            self.running = True
520        # Initialize controller objects and pack appropriate objects/params
521        # to be passed to test class.
522        self.parse_config(self.test_configs)
523        if test_class:
524            self.test_classes = {test_class.__name__: test_class}
525        else:
526            t_paths = self.test_configs[keys.Config.key_test_paths.value]
527            self.test_classes = self.import_test_modules(t_paths)
528        self.log.debug("Executing run list %s.", self.run_list)
529        for test_cls_name, test_case_names in self.run_list:
530            if not self.running:
531                break
532
533            if test_case_names:
534                self.log.debug("Executing test cases %s in test class %s.",
535                               test_case_names, test_cls_name)
536            else:
537                self.log.debug("Executing test class %s", test_cls_name)
538            try:
539                # Import and register the built-in controller modules specified
540                # in testbed config.
541                for module in self._import_builtin_controllers():
542                    self.register_controller(module, builtin=True)
543                self.run_test_class(test_cls_name, test_case_names)
544            except signals.TestAbortAll as e:
545                self.log.warning(
546                    "Abort all subsequent test classes. Reason: %s", e)
547                raise
548            finally:
549                self.unregister_controllers()
550
551    def stop(self):
552        """Releases resources from test run. Should always be called after
553        TestRunner.run finishes.
554
555        This function concludes a test run and writes out a test report.
556        """
557        if self.running:
558            msg = "\nSummary for test run %s: %s\n" % (
559                self.id, self.results.summary_str())
560            self._write_results_json_str()
561            self.log.info(msg.strip())
562            logger.kill_test_logger(self.log)
563            self.running = False
564
565    def _write_results_json_str(self):
566        """Writes out a json file with the test result info for easy parsing.
567
568        TODO(angli): This should be replaced by standard log record mechanism.
569        """
570        path = os.path.join(self.log_path, "test_run_summary.json")
571        with open(path, 'w') as f:
572            f.write(self.results.json_str())
573
574    def write_test_campaign(self):
575        """Log test campaign file."""
576        path = os.path.join(self.log_path, "test_campaign.log")
577        with open(path, 'w') as f:
578            for test_class, test_cases in self.run_list:
579                f.write("%s:\n%s" % (test_class, ",\n".join(test_cases)))
580                f.write("\n\n")
581