1 #!/usr/bin/env python3 2 # 3 # Copyright 2016 - The Android Open Source Project 4 # 5 # Licensed under the Apache License, Version 2.0 (the "License"); 6 # you may not use this file except in compliance with the License. 7 # You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 17 import fnmatch 18 import functools 19 import importlib 20 import logging 21 import os 22 import traceback 23 from concurrent.futures import ThreadPoolExecutor 24 25 from acts import asserts 26 from acts import error 27 from acts import keys 28 from acts import logger 29 from acts import records 30 from acts import signals 31 from acts import tracelogger 32 from acts import utils 33 from acts.event import event_bus 34 from acts.event import subscription_bundle 35 from acts.event.decorators import subscribe_static 36 from acts.event.event import TestCaseBeginEvent 37 from acts.event.event import TestCaseEndEvent 38 from acts.event.event import TestClassBeginEvent 39 from acts.event.event import TestClassEndEvent 40 from acts.event.subscription_bundle import SubscriptionBundle 41 42 from mobly.base_test import BaseTestClass as MoblyBaseTest 43 from mobly.records import ExceptionRecord 44 45 # Macro strings for test result reporting 46 TEST_CASE_TOKEN = "[Test Case]" 47 RESULT_LINE_TEMPLATE = TEST_CASE_TOKEN + " %s %s" 48 49 50 @subscribe_static(TestCaseBeginEvent) 51 def _logcat_log_test_begin(event): 52 """Ensures that logcat is running. Write a logcat line indicating test case 53 begin.""" 54 test_instance = event.test_class 55 try: 56 for ad in getattr(test_instance, 'android_devices', []): 57 if not ad.is_adb_logcat_on: 58 ad.start_adb_logcat() 59 # Write test start token to adb log if android device is attached. 60 if not ad.skip_sl4a and ad.droid: 61 ad.droid.logV("%s BEGIN %s" % 62 (TEST_CASE_TOKEN, event.test_case_name)) 63 64 except error.ActsError as e: 65 test_instance.results.error.append( 66 ExceptionRecord(e, 'Logcat for test begin: %s' % 67 event.test_case_name)) 68 test_instance.log.error('BaseTest setup_test error: %s' % e.details) 69 except Exception as e: 70 test_instance.log.warning( 71 'Unable to send BEGIN log command to all devices.') 72 test_instance.log.warning('Error: %s' % e) 73 74 75 @subscribe_static(TestCaseEndEvent) 76 def _logcat_log_test_end(event): 77 """Write a logcat line indicating test case end.""" 78 test_instance = event.test_class 79 try: 80 # Write test end token to adb log if android device is attached. 81 for ad in getattr(test_instance, 'android_devices', []): 82 if not ad.skip_sl4a and ad.droid: 83 ad.droid.logV("%s END %s" % 84 (TEST_CASE_TOKEN, event.test_case_name)) 85 86 except error.ActsError as e: 87 test_instance.results.error.append( 88 ExceptionRecord(e, 89 'Logcat for test end: %s' % event.test_case_name)) 90 test_instance.log.error('BaseTest teardown_test error: %s' % e.details) 91 except Exception as e: 92 test_instance.log.warning( 93 'Unable to send END log command to all devices.') 94 test_instance.log.warning('Error: %s' % e) 95 96 97 @subscribe_static(TestCaseBeginEvent) 98 def _syslog_log_test_begin(event): 99 """This adds a BEGIN log message with the test name to the syslog of any 100 Fuchsia device""" 101 test_instance = event.test_class 102 try: 103 for fd in getattr(test_instance, 'fuchsia_devices', []): 104 if not fd.skip_sl4f: 105 fd.logging_lib.logI("%s BEGIN %s" % 106 (TEST_CASE_TOKEN, event.test_case_name)) 107 108 except Exception as e: 109 test_instance.log.warning( 110 'Unable to send BEGIN log command to all devices.') 111 test_instance.log.warning('Error: %s' % e) 112 113 114 @subscribe_static(TestCaseEndEvent) 115 def _syslog_log_test_end(event): 116 """This adds a END log message with the test name to the syslog of any 117 Fuchsia device""" 118 test_instance = event.test_class 119 try: 120 for fd in getattr(test_instance, 'fuchsia_devices', []): 121 if not fd.skip_sl4f: 122 fd.logging_lib.logI("%s END %s" % 123 (TEST_CASE_TOKEN, event.test_case_name)) 124 125 except Exception as e: 126 test_instance.log.warning( 127 'Unable to send END log command to all devices.') 128 test_instance.log.warning('Error: %s' % e) 129 130 131 event_bus.register_subscription(_logcat_log_test_begin.subscription) 132 event_bus.register_subscription(_logcat_log_test_end.subscription) 133 event_bus.register_subscription(_syslog_log_test_begin.subscription) 134 event_bus.register_subscription(_syslog_log_test_end.subscription) 135 136 137 class Error(Exception): 138 """Raised for exceptions that occured in BaseTestClass.""" 139 140 141 class BaseTestClass(MoblyBaseTest): 142 """Base class for all test classes to inherit from. Inherits some 143 functionality from Mobly's base test class. 144 145 This class gets all the controller objects from test_runner and executes 146 the test cases requested within itself. 147 148 Most attributes of this class are set at runtime based on the configuration 149 provided. 150 151 Attributes: 152 tests: A list of strings, each representing a test case name. 153 TAG: A string used to refer to a test class. Default is the test class 154 name. 155 log: A logger object used for logging. 156 results: A records.TestResult object for aggregating test results from 157 the execution of test cases. 158 controller_configs: A dict of controller configs provided by the user 159 via the testbed config. 160 consecutive_failures: Tracks the number of consecutive test case 161 failures within this class. 162 consecutive_failure_limit: Number of consecutive test failures to allow 163 before blocking remaining tests in the same 164 test class. 165 size_limit_reached: True if the size of the log directory has reached 166 its limit. 167 current_test_name: A string that's the name of the test case currently 168 being executed. If no test is executing, this should 169 be None. 170 """ 171 172 TAG = None 173 174 def __init__(self, configs): 175 """Initializes a BaseTestClass given a TestRunConfig, which provides 176 all of the config information for this test class. 177 178 Args: 179 configs: A config_parser.TestRunConfig object. 180 """ 181 super().__init__(configs) 182 183 self.__handle_file_user_params() 184 185 self.class_subscriptions = SubscriptionBundle() 186 self.class_subscriptions.register() 187 self.all_subscriptions = [self.class_subscriptions] 188 189 self.current_test_name = None 190 self.log = tracelogger.TraceLogger(logging.getLogger()) 191 # TODO: remove after converging log path definitions with mobly 192 self.log_path = configs.log_path 193 194 self.consecutive_failures = 0 195 self.consecutive_failure_limit = self.user_params.get( 196 'consecutive_failure_limit', -1) 197 self.size_limit_reached = False 198 self.retryable_exceptions = signals.TestFailure 199 200 def _import_builtin_controllers(self): 201 """Import built-in controller modules. 202 203 Go through the testbed configs, find any built-in controller configs 204 and import the corresponding controller module from acts.controllers 205 package. 206 207 Returns: 208 A list of controller modules. 209 """ 210 builtin_controllers = [] 211 for ctrl_name in keys.Config.builtin_controller_names.value: 212 if ctrl_name in self.controller_configs: 213 module_name = keys.get_module_name(ctrl_name) 214 module = importlib.import_module("acts.controllers.%s" % 215 module_name) 216 builtin_controllers.append(module) 217 return builtin_controllers 218 219 def __handle_file_user_params(self): 220 """For backwards compatibility, moves all contents of the "files" dict 221 into the root level of user_params. 222 223 This allows existing tests to run with the new Mobly-style format 224 without needing to make changes. 225 """ 226 for key, value in self.user_params.items(): 227 if key.endswith('files') and isinstance(value, dict): 228 new_user_params = dict(value) 229 new_user_params.update(self.user_params) 230 self.user_params = new_user_params 231 break 232 233 @staticmethod 234 def get_module_reference_name(a_module): 235 """Returns the module's reference name. 236 237 This is largely for backwards compatibility with log parsing. If the 238 module defines ACTS_CONTROLLER_REFERENCE_NAME, it will return that 239 value, or the module's submodule name. 240 241 Args: 242 a_module: Any module. Ideally, a controller module. 243 Returns: 244 A string corresponding to the module's name. 245 """ 246 if hasattr(a_module, 'ACTS_CONTROLLER_REFERENCE_NAME'): 247 return a_module.ACTS_CONTROLLER_REFERENCE_NAME 248 else: 249 return a_module.__name__.split('.')[-1] 250 251 def register_controller(self, 252 controller_module, 253 required=True, 254 builtin=False): 255 """Registers an ACTS controller module for a test class. Invokes Mobly's 256 implementation of register_controller. 257 258 An ACTS controller module is a Python lib that can be used to control 259 a device, service, or equipment. To be ACTS compatible, a controller 260 module needs to have the following members: 261 262 def create(configs): 263 [Required] Creates controller objects from configurations. 264 Args: 265 configs: A list of serialized data like string/dict. Each 266 element of the list is a configuration for a 267 controller object. 268 Returns: 269 A list of objects. 270 271 def destroy(objects): 272 [Required] Destroys controller objects created by the create 273 function. Each controller object shall be properly cleaned up 274 and all the resources held should be released, e.g. memory 275 allocation, sockets, file handlers etc. 276 Args: 277 A list of controller objects created by the create function. 278 279 def get_info(objects): 280 [Optional] Gets info from the controller objects used in a test 281 run. The info will be included in test_result_summary.json under 282 the key "ControllerInfo". Such information could include unique 283 ID, version, or anything that could be useful for describing the 284 test bed and debugging. 285 Args: 286 objects: A list of controller objects created by the create 287 function. 288 Returns: 289 A list of json serializable objects, each represents the 290 info of a controller object. The order of the info object 291 should follow that of the input objects. 292 Registering a controller module declares a test class's dependency the 293 controller. If the module config exists and the module matches the 294 controller interface, controller objects will be instantiated with 295 corresponding configs. The module should be imported first. 296 297 Args: 298 controller_module: A module that follows the controller module 299 interface. 300 required: A bool. If True, failing to register the specified 301 controller module raises exceptions. If False, returns None upon 302 failures. 303 builtin: Specifies that the module is a builtin controller module in 304 ACTS. If true, adds itself to test attributes. 305 Returns: 306 A list of controller objects instantiated from controller_module, or 307 None. 308 309 Raises: 310 When required is True, ControllerError is raised if no corresponding 311 config can be found. 312 Regardless of the value of "required", ControllerError is raised if 313 the controller module has already been registered or any other error 314 occurred in the registration process. 315 """ 316 module_ref_name = self.get_module_reference_name(controller_module) 317 module_config_name = controller_module.MOBLY_CONTROLLER_CONFIG_NAME 318 319 # Get controller objects from Mobly's register_controller 320 controllers = self._controller_manager.register_controller( 321 controller_module, required=required) 322 if not controllers: 323 return None 324 325 # Log controller information 326 # Implementation of "get_info" is optional for a controller module. 327 if hasattr(controller_module, "get_info"): 328 controller_info = controller_module.get_info(controllers) 329 self.log.info("Controller %s: %s", module_config_name, 330 controller_info) 331 332 if builtin: 333 setattr(self, module_ref_name, controllers) 334 return controllers 335 336 def _setup_class(self): 337 """Proxy function to guarantee the base implementation of setup_class 338 is called. 339 """ 340 event_bus.post(TestClassBeginEvent(self)) 341 # Import and register the built-in controller modules specified 342 # in testbed config. 343 for module in self._import_builtin_controllers(): 344 self.register_controller(module, builtin=True) 345 return self.setup_class() 346 347 def _teardown_class(self): 348 """Proxy function to guarantee the base implementation of teardown_class 349 is called. 350 """ 351 super()._teardown_class() 352 event_bus.post(TestClassEndEvent(self, self.results)) 353 354 def _setup_test(self, test_name): 355 """Proxy function to guarantee the base implementation of setup_test is 356 called. 357 """ 358 self.current_test_name = test_name 359 360 # Skip the test if the consecutive test case failure limit is reached. 361 if self.consecutive_failures == self.consecutive_failure_limit: 362 raise signals.TestError('Consecutive test failure') 363 364 return self.setup_test() 365 366 def setup_test(self): 367 """Setup function that will be called every time before executing each 368 test case in the test class. 369 370 To signal setup failure, return False or raise an exception. If 371 exceptions were raised, the stack trace would appear in log, but the 372 exceptions would not propagate to upper levels. 373 374 Implementation is optional. 375 """ 376 return True 377 378 def _teardown_test(self, test_name): 379 """Proxy function to guarantee the base implementation of teardown_test 380 is called. 381 """ 382 self.log.debug('Tearing down test %s' % test_name) 383 self.teardown_test() 384 385 def _on_fail(self, record): 386 """Proxy function to guarantee the base implementation of on_fail is 387 called. 388 389 Args: 390 record: The records.TestResultRecord object for the failed test 391 case. 392 """ 393 self.consecutive_failures += 1 394 if record.details: 395 self.log.error(record.details) 396 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 397 self.on_fail(record.test_name, record.begin_time) 398 399 def on_fail(self, test_name, begin_time): 400 """A function that is executed upon a test case failure. 401 402 User implementation is optional. 403 404 Args: 405 test_name: Name of the test that triggered this function. 406 begin_time: Logline format timestamp taken when the test started. 407 """ 408 def _on_pass(self, record): 409 """Proxy function to guarantee the base implementation of on_pass is 410 called. 411 412 Args: 413 record: The records.TestResultRecord object for the passed test 414 case. 415 """ 416 self.consecutive_failures = 0 417 msg = record.details 418 if msg: 419 self.log.info(msg) 420 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 421 self.on_pass(record.test_name, record.begin_time) 422 423 def on_pass(self, test_name, begin_time): 424 """A function that is executed upon a test case passing. 425 426 Implementation is optional. 427 428 Args: 429 test_name: Name of the test that triggered this function. 430 begin_time: Logline format timestamp taken when the test started. 431 """ 432 def _on_skip(self, record): 433 """Proxy function to guarantee the base implementation of on_skip is 434 called. 435 436 Args: 437 record: The records.TestResultRecord object for the skipped test 438 case. 439 """ 440 self.log.info(RESULT_LINE_TEMPLATE, record.test_name, record.result) 441 self.log.info("Reason to skip: %s", record.details) 442 self.on_skip(record.test_name, record.begin_time) 443 444 def on_skip(self, test_name, begin_time): 445 """A function that is executed upon a test case being skipped. 446 447 Implementation is optional. 448 449 Args: 450 test_name: Name of the test that triggered this function. 451 begin_time: Logline format timestamp taken when the test started. 452 """ 453 def _on_exception(self, record): 454 """Proxy function to guarantee the base implementation of on_exception 455 is called. 456 457 Args: 458 record: The records.TestResultRecord object for the failed test 459 case. 460 """ 461 self.log.exception(record.details) 462 self.on_exception(record.test_name, record.begin_time) 463 464 def on_exception(self, test_name, begin_time): 465 """A function that is executed upon an unhandled exception from a test 466 case. 467 468 Implementation is optional. 469 470 Args: 471 test_name: Name of the test that triggered this function. 472 begin_time: Logline format timestamp taken when the test started. 473 """ 474 def on_retry(self): 475 """Function to run before retrying a test through get_func_with_retry. 476 477 This function runs when a test is automatically retried. The function 478 can be used to modify internal test parameters, for example, to retry 479 a test with slightly different input variables. 480 """ 481 def _exec_procedure_func(self, func, tr_record): 482 """Executes a procedure function like on_pass, on_fail etc. 483 484 This function will alternate the 'Result' of the test's record if 485 exceptions happened when executing the procedure function. 486 487 This will let signals.TestAbortAll through so abort_all works in all 488 procedure functions. 489 490 Args: 491 func: The procedure function to be executed. 492 tr_record: The TestResultRecord object associated with the test 493 case executed. 494 """ 495 try: 496 func(tr_record) 497 except signals.TestAbortAll: 498 raise 499 except Exception as e: 500 self.log.exception("Exception happened when executing %s for %s.", 501 func.__name__, self.current_test_name) 502 tr_record.add_error(func.__name__, e) 503 504 def exec_one_testcase(self, test_name, test_func): 505 """Executes one test case and update test results. 506 507 Executes one test case, create a records.TestResultRecord object with 508 the execution information, and add the record to the test class's test 509 results. 510 511 Args: 512 test_name: Name of the test. 513 test_func: The test function. 514 """ 515 class_name = self.__class__.__name__ 516 tr_record = records.TestResultRecord(test_name, class_name) 517 tr_record.test_begin() 518 self.begin_time = int(tr_record.begin_time) 519 self.log_begin_time = tr_record.log_begin_time 520 self.test_name = tr_record.test_name 521 event_bus.post(TestCaseBeginEvent(self, self.test_name)) 522 self.log.info("%s %s", TEST_CASE_TOKEN, test_name) 523 524 # Enable test retry if specified in the ACTS config 525 retry_tests = self.user_params.get('retry_tests', []) 526 full_test_name = '%s.%s' % (class_name, self.test_name) 527 if any(name in retry_tests for name in [class_name, full_test_name]): 528 test_func = self.get_func_with_retry(test_func) 529 530 verdict = None 531 test_signal = None 532 try: 533 try: 534 ret = self._setup_test(self.test_name) 535 asserts.assert_true(ret is not False, 536 "Setup for %s failed." % test_name) 537 verdict = test_func() 538 finally: 539 try: 540 self._teardown_test(self.test_name) 541 except signals.TestAbortAll: 542 raise 543 except Exception as e: 544 self.log.error(traceback.format_exc()) 545 tr_record.add_error("teardown_test", e) 546 except (signals.TestFailure, AssertionError) as e: 547 test_signal = e 548 if self.user_params.get( 549 keys.Config.key_test_failure_tracebacks.value, False): 550 self.log.exception(e) 551 tr_record.test_fail(e) 552 except signals.TestSkip as e: 553 # Test skipped. 554 test_signal = e 555 tr_record.test_skip(e) 556 except (signals.TestAbortClass, signals.TestAbortAll) as e: 557 # Abort signals, pass along. 558 test_signal = e 559 tr_record.test_fail(e) 560 raise e 561 except signals.TestPass as e: 562 # Explicit test pass. 563 test_signal = e 564 tr_record.test_pass(e) 565 except Exception as e: 566 test_signal = e 567 self.log.error(traceback.format_exc()) 568 # Exception happened during test. 569 tr_record.test_error(e) 570 else: 571 if verdict or (verdict is None): 572 # Test passed. 573 tr_record.test_pass() 574 return 575 tr_record.test_fail() 576 finally: 577 tr_record.update_record() 578 try: 579 # Execute post-test procedures 580 result = tr_record.result 581 if result == records.TestResultEnums.TEST_RESULT_PASS: 582 self._exec_procedure_func(self._on_pass, tr_record) 583 elif result == records.TestResultEnums.TEST_RESULT_FAIL: 584 self._exec_procedure_func(self._on_fail, tr_record) 585 elif result == records.TestResultEnums.TEST_RESULT_SKIP: 586 self._exec_procedure_func(self._on_skip, tr_record) 587 elif result == records.TestResultEnums.TEST_RESULT_ERROR: 588 self._exec_procedure_func(self._on_exception, tr_record) 589 self._exec_procedure_func(self._on_fail, tr_record) 590 finally: 591 self.results.add_record(tr_record) 592 self.summary_writer.dump(tr_record.to_dict(), 593 records.TestSummaryEntryType.RECORD) 594 self.current_test_name = None 595 event_bus.post( 596 TestCaseEndEvent(self, self.test_name, test_signal)) 597 598 def get_func_with_retry(self, func, attempts=2): 599 """Returns a wrapped test method that re-runs after failure. Return test 600 result upon success. If attempt limit reached, collect all failure 601 messages and raise a TestFailure signal. 602 603 Params: 604 func: The test method 605 attempts: Number of attempts to run test 606 607 Returns: result of the test method 608 """ 609 exceptions = self.retryable_exceptions 610 611 def wrapper(*args, **kwargs): 612 error_msgs = [] 613 extras = {} 614 retry = False 615 for i in range(attempts): 616 try: 617 if retry: 618 self.teardown_test() 619 self.setup_test() 620 self.on_retry() 621 return func(*args, **kwargs) 622 except exceptions as e: 623 retry = True 624 msg = 'Failure on attempt %d: %s' % (i + 1, e.details) 625 self.log.warning(msg) 626 error_msgs.append(msg) 627 if e.extras: 628 extras['Attempt %d' % (i + 1)] = e.extras 629 raise signals.TestFailure('\n'.join(error_msgs), extras) 630 631 return wrapper 632 633 def run_generated_testcases(self, 634 test_func, 635 settings, 636 args=None, 637 kwargs=None, 638 tag="", 639 name_func=None, 640 format_args=False): 641 """Deprecated. Please use setup_generated_tests and generate_tests. 642 643 Generated test cases are not written down as functions, but as a list 644 of parameter sets. This way we reduce code repetition and improve 645 test case scalability. 646 647 Args: 648 test_func: The common logic shared by all these generated test 649 cases. This function should take at least one argument, 650 which is a parameter set. 651 settings: A list of strings representing parameter sets. These are 652 usually json strings that get loaded in the test_func. 653 args: Iterable of additional position args to be passed to 654 test_func. 655 kwargs: Dict of additional keyword args to be passed to test_func 656 tag: Name of this group of generated test cases. Ignored if 657 name_func is provided and operates properly. 658 name_func: A function that takes a test setting and generates a 659 proper test name. The test name should be shorter than 660 utils.MAX_FILENAME_LEN. Names over the limit will be 661 truncated. 662 format_args: If True, args will be appended as the first argument 663 in the args list passed to test_func. 664 665 Returns: 666 A list of settings that did not pass. 667 """ 668 args = args or () 669 kwargs = kwargs or {} 670 failed_settings = [] 671 672 for setting in settings: 673 test_name = "{} {}".format(tag, setting) 674 675 if name_func: 676 try: 677 test_name = name_func(setting, *args, **kwargs) 678 except: 679 self.log.exception(("Failed to get test name from " 680 "test_func. Fall back to default %s"), 681 test_name) 682 683 self.results.requested.append(test_name) 684 685 if len(test_name) > utils.MAX_FILENAME_LEN: 686 test_name = test_name[:utils.MAX_FILENAME_LEN] 687 688 previous_success_cnt = len(self.results.passed) 689 690 if format_args: 691 self.exec_one_testcase( 692 test_name, 693 functools.partial(test_func, *(args + (setting, )), 694 **kwargs)) 695 else: 696 self.exec_one_testcase( 697 test_name, 698 functools.partial(test_func, *((setting, ) + args), 699 **kwargs)) 700 701 if len(self.results.passed) - previous_success_cnt != 1: 702 failed_settings.append(setting) 703 704 return failed_settings 705 706 def _exec_func(self, func, *args): 707 """Executes a function with exception safeguard. 708 709 This will let signals.TestAbortAll through so abort_all works in all 710 procedure functions. 711 712 Args: 713 func: Function to be executed. 714 args: Arguments to be passed to the function. 715 716 Returns: 717 Whatever the function returns, or False if unhandled exception 718 occured. 719 """ 720 try: 721 return func(*args) 722 except signals.TestAbortAll: 723 raise 724 except: 725 self.log.exception("Exception happened when executing %s in %s.", 726 func.__name__, self.TAG) 727 return False 728 729 def _block_all_test_cases(self, tests, reason='Failed class setup'): 730 """ 731 Block all passed in test cases. 732 Args: 733 tests: The tests to block. 734 reason: Message describing the reason that the tests are blocked. 735 Default is 'Failed class setup' 736 """ 737 for test_name, test_func in tests: 738 signal = signals.TestError(reason) 739 record = records.TestResultRecord(test_name, self.TAG) 740 record.test_begin() 741 if hasattr(test_func, 'gather'): 742 signal.extras = test_func.gather() 743 record.test_error(signal) 744 self.results.add_record(record) 745 self.summary_writer.dump(record.to_dict(), 746 records.TestSummaryEntryType.RECORD) 747 self._on_skip(record) 748 749 def run(self, test_names=None): 750 """Runs test cases within a test class by the order they appear in the 751 execution list. 752 753 One of these test cases lists will be executed, shown here in priority 754 order: 755 1. The test_names list, which is passed from cmd line. 756 2. The self.tests list defined in test class. Invalid names are 757 ignored. 758 3. All function that matches test case naming convention in the test 759 class. 760 761 Args: 762 test_names: A list of string that are test case names/patterns 763 requested in cmd line. 764 765 Returns: 766 The test results object of this class. 767 """ 768 # Executes pre-setup procedures, like generating test methods. 769 if not self._setup_generated_tests(): 770 return self.results 771 772 self.register_test_class_event_subscriptions() 773 self.log.info("==========> %s <==========", self.TAG) 774 # Devise the actual test cases to run in the test class. 775 if self.tests: 776 # Specified by run list in class. 777 valid_tests = list(self.tests) 778 else: 779 # No test case specified by user, gather the run list automatically. 780 valid_tests = self.get_existing_test_names() 781 if test_names: 782 # Match test cases with any of the user-specified patterns 783 matches = [] 784 for test_name in test_names: 785 for valid_test in valid_tests: 786 if (fnmatch.fnmatch(valid_test, test_name) 787 and valid_test not in matches): 788 matches.append(valid_test) 789 else: 790 matches = valid_tests 791 self.results.requested = matches 792 self.summary_writer.dump(self.results.requested_test_names_dict(), 793 records.TestSummaryEntryType.TEST_NAME_LIST) 794 tests = self._get_test_methods(matches) 795 796 # Setup for the class. 797 setup_fail = False 798 try: 799 if self._setup_class() is False: 800 self.log.error("Failed to setup %s.", self.TAG) 801 self._block_all_test_cases(tests) 802 setup_fail = True 803 except signals.TestAbortClass: 804 self.log.exception('Test class %s aborted' % self.TAG) 805 setup_fail = True 806 except Exception as e: 807 self.log.exception("Failed to setup %s.", self.TAG) 808 self._block_all_test_cases(tests) 809 setup_fail = True 810 if setup_fail: 811 self._exec_func(self._teardown_class) 812 self.log.info("Summary for test class %s: %s", self.TAG, 813 self.results.summary_str()) 814 return self.results 815 816 # Run tests in order. 817 test_case_iterations = self.user_params.get( 818 keys.Config.key_test_case_iterations.value, 1) 819 if any([substr in self.__class__.__name__ for substr in 820 ['Preflight', 'Postflight']]): 821 test_case_iterations = 1 822 try: 823 for test_name, test_func in tests: 824 for _ in range(test_case_iterations): 825 self.exec_one_testcase(test_name, test_func) 826 return self.results 827 except signals.TestAbortClass: 828 self.log.exception('Test class %s aborted' % self.TAG) 829 return self.results 830 except signals.TestAbortAll as e: 831 # Piggy-back test results on this exception object so we don't lose 832 # results from this test class. 833 setattr(e, "results", self.results) 834 raise e 835 finally: 836 self._exec_func(self._teardown_class) 837 self.log.info("Summary for test class %s: %s", self.TAG, 838 self.results.summary_str()) 839 840 def _ad_take_bugreport(self, ad, test_name, begin_time): 841 for i in range(3): 842 try: 843 ad.take_bug_report(test_name, begin_time) 844 return True 845 except Exception as e: 846 ad.log.error("bugreport attempt %s error: %s", i + 1, e) 847 848 def _ad_take_extra_logs(self, ad, test_name, begin_time): 849 result = True 850 if getattr(ad, "qxdm_log", False): 851 # Gather qxdm log modified 3 minutes earlier than test start time 852 if begin_time: 853 qxdm_begin_time = begin_time - 1000 * 60 * 3 854 else: 855 qxdm_begin_time = None 856 try: 857 ad.get_qxdm_logs(test_name, qxdm_begin_time) 858 except Exception as e: 859 ad.log.error("Failed to get QXDM log for %s with error %s", 860 test_name, e) 861 result = False 862 863 try: 864 ad.check_crash_report(test_name, begin_time, log_crash_report=True) 865 except Exception as e: 866 ad.log.error("Failed to check crash report for %s with error %s", 867 test_name, e) 868 result = False 869 return result 870 871 def _skip_bug_report(self, test_name): 872 """A function to check whether we should skip creating a bug report. 873 874 Args: 875 test_name: The test case name 876 877 Returns: True if bug report is to be skipped. 878 """ 879 if "no_bug_report_on_fail" in self.user_params: 880 return True 881 882 # If the current test class or test case is found in the set of 883 # problematic tests, we skip bugreport and other failure artifact 884 # creation. 885 class_name = self.__class__.__name__ 886 quiet_tests = self.user_params.get('quiet_tests', []) 887 if class_name in quiet_tests: 888 self.log.info( 889 "Skipping bug report, as directed for this test class.") 890 return True 891 full_test_name = '%s.%s' % (class_name, test_name) 892 if full_test_name in quiet_tests: 893 self.log.info( 894 "Skipping bug report, as directed for this test case.") 895 return True 896 897 # Once we hit a certain log path size, it's not going to get smaller. 898 # We cache the result so we don't have to keep doing directory walks. 899 if self.size_limit_reached: 900 return True 901 try: 902 max_log_size = int( 903 self.user_params.get("soft_output_size_limit") or "invalid") 904 log_path = getattr(logging, "log_path", None) 905 if log_path: 906 curr_log_size = utils.get_directory_size(log_path) 907 if curr_log_size > max_log_size: 908 self.log.info( 909 "Skipping bug report, as we've reached the size limit." 910 ) 911 self.size_limit_reached = True 912 return True 913 except ValueError: 914 pass 915 return False 916 917 def _take_bug_report(self, test_name, begin_time): 918 if self._skip_bug_report(test_name): 919 return 920 921 executor = ThreadPoolExecutor(max_workers=10) 922 for ad in getattr(self, 'android_devices', []): 923 executor.submit(self._ad_take_bugreport, ad, test_name, begin_time) 924 executor.submit(self._ad_take_extra_logs, ad, test_name, 925 begin_time) 926 executor.shutdown() 927 928 def _reboot_device(self, ad): 929 ad.log.info("Rebooting device.") 930 ad = ad.reboot() 931 932 def _cleanup_logger_sessions(self): 933 for (mylogger, session) in self.logger_sessions: 934 self.log.info("Resetting a diagnostic session %s, %s", mylogger, 935 session) 936 mylogger.reset() 937 self.logger_sessions = [] 938 939 def _pull_diag_logs(self, test_name, begin_time): 940 for (mylogger, session) in self.logger_sessions: 941 self.log.info("Pulling diagnostic session %s", mylogger) 942 mylogger.stop(session) 943 diag_path = os.path.join( 944 self.log_path, logger.epoch_to_log_line_timestamp(begin_time)) 945 os.makedirs(diag_path, exist_ok=True) 946 mylogger.pull(session, diag_path) 947 948 def register_test_class_event_subscriptions(self): 949 self.class_subscriptions = subscription_bundle.create_from_instance( 950 self) 951 self.class_subscriptions.register() 952 953 def unregister_test_class_event_subscriptions(self): 954 for package in self.all_subscriptions: 955 package.unregister() 956