1# 2# Copyright (C) 2016 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import logging 18import os 19import re 20import signal 21import sys 22import threading 23import time 24 25from vts.proto import VtsReportMessage_pb2 as ReportMsg 26from vts.runners.host import asserts 27from vts.runners.host import const 28from vts.runners.host import errors 29from vts.runners.host import keys 30from vts.runners.host import logger 31from vts.runners.host import records 32from vts.runners.host import signals 33from vts.runners.host import utils 34from vts.utils.python.controllers import adb 35from vts.utils.python.controllers import android_device 36from vts.utils.python.controllers.adb import AdbError 37from vts.utils.python.common import cmd_utils 38from vts.utils.python.common import filter_utils 39from vts.utils.python.common import list_utils 40from vts.utils.python.common import timeout_utils 41from vts.utils.python.coverage import coverage_utils 42from vts.utils.python.coverage import sancov_utils 43from vts.utils.python.instrumentation import test_framework_instrumentation as tfi 44from vts.utils.python.io import file_util 45from vts.utils.python.precondition import precondition_utils 46from vts.utils.python.profiling import profiling_utils 47from vts.utils.python.reporting import log_uploading_utils 48from vts.utils.python.systrace import systrace_utils 49from vts.utils.python.web import feature_utils 50from vts.utils.python.web import web_utils 51 52 53# Macro strings for test result reporting 54TEST_CASE_TEMPLATE = "[Test Case] %s %s" 55RESULT_LINE_TEMPLATE = TEST_CASE_TEMPLATE + " %s" 56STR_TEST = "test" 57STR_GENERATE = "generate" 58TIMEOUT_SECS_LOG_UPLOADING = 60 59TIMEOUT_SECS_TEARDOWN_CLASS = 120 60_REPORT_MESSAGE_FILE_NAME = "report_proto.msg" 61_BUG_REPORT_FILE_PREFIX = "bugreport_" 62_BUG_REPORT_FILE_EXTENSION = ".zip" 63_DEFAULT_TEST_TIMEOUT_SECS = 60 * 3 64_LOGCAT_FILE_PREFIX = "logcat_" 65_LOGCAT_FILE_EXTENSION = ".txt" 66_ANDROID_DEVICES = '_android_devices' 67_REASON_TO_SKIP_ALL_TESTS = '_reason_to_skip_all_tests' 68_SETUP_RETRY_NUMBER = 5 69# the name of a system property which tells whether to stop properly configured 70# native servers where properly configured means a server's init.rc is 71# configured to stop when that property's value is 1. 72SYSPROP_VTS_NATIVE_SERVER = "vts.native_server.on" 73 74LOGCAT_BUFFERS = [ 75 'radio', 76 'events', 77 'main', 78 'system', 79 'crash' 80] 81 82 83class BaseTestClass(object): 84 """Base class for all test classes to inherit from. 85 86 This class gets all the controller objects from test_runner and executes 87 the test cases requested within itself. 88 89 Most attributes of this class are set at runtime based on the configuration 90 provided. 91 92 Attributes: 93 android_devices: A list of AndroidDevice object, representing android 94 devices. 95 test_module_name: A string representing the test module name. 96 tests: A list of strings, each representing a test case name. 97 log: A logger object used for logging. 98 results: A records.TestResult object for aggregating test results from 99 the execution of test cases. 100 _current_record: A records.TestResultRecord object for the test case 101 currently being executed. If no test is running, this 102 should be None. 103 _interrupted: Whether the test execution has been interrupted. 104 _interrupt_lock: The threading.Lock object that protects _interrupted. 105 _timer: The threading.Timer object that interrupts main thread when 106 timeout. 107 timeout: A float, the timeout, in seconds, configured for this object. 108 include_filer: A list of string, each representing a test case name to 109 include. 110 exclude_filer: A list of string, each representing a test case name to 111 exclude. Has no effect if include_filer is not empty. 112 abi_name: String, name of abi in use 113 abi_bitness: String, bitness of abi in use 114 web: WebFeature, object storing web feature util for test run 115 coverage: CoverageFeature, object storing coverage feature util for test run 116 sancov: SancovFeature, object storing sancov feature util for test run 117 start_time_sec: int, time value in seconds when the module execution starts. 118 start_vts_agents: whether to start vts agents when registering new 119 android devices. 120 profiling: ProfilingFeature, object storing profiling feature util for test run 121 _bug_report_on_failure: bool, whether to catch bug report at the end 122 of failed test cases. Default is False 123 _logcat_on_failure: bool, whether to dump logcat at the end 124 of failed test cases. Default is True 125 _is_final_run: bool, whether the current test run is the final run during retry. 126 test_filter: Filter object to filter test names. 127 _test_filter_retry: Filter object for retry filtering. 128 max_retry_count: int, max number of retries. 129 """ 130 _current_record = None 131 start_vts_agents = True 132 133 def __init__(self, configs): 134 self.start_time_sec = time.time() 135 self.tests = [] 136 # Set all the controller objects and params. 137 for name, value in configs.items(): 138 setattr(self, name, value) 139 self.results = records.TestResult() 140 self.log = logger.LoggerProxy() 141 142 # Timeout 143 self._interrupted = False 144 self._interrupt_lock = threading.Lock() 145 self._timer = None 146 147 timeout_milli = self.getUserParam(keys.ConfigKeys.KEY_TEST_TIMEOUT, 0) 148 self.timeout = timeout_milli / 1000 if timeout_milli > 0 else _DEFAULT_TEST_TIMEOUT_SECS 149 150 # Setup test filters 151 # TODO(yuexima): remove include_filter and exclude_filter from class attributes 152 # after confirming all modules no longer have reference to them 153 self.include_filter = self.getUserParam( 154 [ 155 keys.ConfigKeys.KEY_TEST_SUITE, 156 keys.ConfigKeys.KEY_INCLUDE_FILTER 157 ], 158 default_value=[]) 159 self.exclude_filter = self.getUserParam( 160 [ 161 keys.ConfigKeys.KEY_TEST_SUITE, 162 keys.ConfigKeys.KEY_EXCLUDE_FILTER 163 ], 164 default_value=[]) 165 166 self.test_module_name = self.getUserParam( 167 keys.ConfigKeys.KEY_TESTBED_NAME, 168 warn_if_not_found=True, 169 default_value=self.__class__.__name__) 170 171 self.updateTestFilter() 172 173 logging.debug('Test filter: %s' % self.test_filter) 174 175 # TODO: get abi information differently for multi-device support. 176 # Set other optional parameters 177 self.abi_name = self.getUserParam( 178 keys.ConfigKeys.IKEY_ABI_NAME, default_value=None) 179 self.abi_bitness = self.getUserParam( 180 keys.ConfigKeys.IKEY_ABI_BITNESS, default_value=None) 181 self.skip_on_32bit_abi = self.getUserParam( 182 keys.ConfigKeys.IKEY_SKIP_ON_32BIT_ABI, default_value=False) 183 self.skip_on_64bit_abi = self.getUserParam( 184 keys.ConfigKeys.IKEY_SKIP_ON_64BIT_ABI, default_value=False) 185 self.run_32bit_on_64bit_abi = self.getUserParam( 186 keys.ConfigKeys.IKEY_RUN_32BIT_ON_64BIT_ABI, default_value=False) 187 self.max_retry_count = self.getUserParam( 188 keys.ConfigKeys.IKEY_MAX_RETRY_COUNT, default_value=0) 189 190 self.web = web_utils.WebFeature(self.user_params) 191 self.coverage = coverage_utils.CoverageFeature( 192 self.user_params, web=self.web) 193 self.sancov = sancov_utils.SancovFeature( 194 self.user_params, web=self.web) 195 self.profiling = profiling_utils.ProfilingFeature( 196 self.user_params, web=self.web) 197 self.systrace = systrace_utils.SystraceFeature( 198 self.user_params, web=self.web) 199 self.log_uploading = log_uploading_utils.LogUploadingFeature( 200 self.user_params, web=self.web) 201 self.collect_tests_only = self.getUserParam( 202 keys.ConfigKeys.IKEY_COLLECT_TESTS_ONLY, default_value=False) 203 self.run_as_vts_self_test = self.getUserParam( 204 keys.ConfigKeys.RUN_AS_VTS_SELFTEST, default_value=False) 205 self.run_as_compliance_test = self.getUserParam( 206 keys.ConfigKeys.RUN_AS_COMPLIANCE_TEST, default_value=False) 207 self._bug_report_on_failure = self.getUserParam( 208 keys.ConfigKeys.IKEY_BUG_REPORT_ON_FAILURE, default_value=False) 209 self._logcat_on_failure = self.getUserParam( 210 keys.ConfigKeys.IKEY_LOGCAT_ON_FAILURE, default_value=True) 211 self._test_filter_retry = None 212 213 @property 214 def android_devices(self): 215 """Returns a list of AndroidDevice objects""" 216 if not hasattr(self, _ANDROID_DEVICES): 217 event = tfi.Begin('base_test registering android_device. ' 218 'Start agents: %s' % self.start_vts_agents, 219 tfi.categories.FRAMEWORK_SETUP) 220 setattr(self, _ANDROID_DEVICES, 221 self.registerController(android_device, 222 start_services=self.start_vts_agents)) 223 event.End() 224 225 for device in getattr(self, _ANDROID_DEVICES): 226 device.shell_default_nohup = self.getUserParam( 227 keys.ConfigKeys.SHELL_DEFAULT_NOHUP, default_value=True) 228 return getattr(self, _ANDROID_DEVICES) 229 230 @android_devices.setter 231 def android_devices(self, devices): 232 """Set the list of AndroidDevice objects""" 233 setattr(self, _ANDROID_DEVICES, devices) 234 235 def __enter__(self): 236 return self 237 238 def __exit__(self, *args): 239 self._exec_func(self.cleanUp) 240 241 def updateTestFilter(self): 242 """Updates test filter using include and exclude filters.""" 243 self.include_filter = list_utils.ExpandItemDelimiters( 244 list_utils.ItemsToStr(self.include_filter), ',') 245 self.exclude_filter = list_utils.ExpandItemDelimiters( 246 list_utils.ItemsToStr(self.exclude_filter), ',') 247 248 exclude_over_include = self.getUserParam( 249 keys.ConfigKeys.KEY_EXCLUDE_OVER_INCLUDE, default_value=None) 250 251 self.test_filter = filter_utils.Filter( 252 self.include_filter, 253 self.exclude_filter, 254 enable_regex=True, 255 exclude_over_include=exclude_over_include, 256 enable_negative_pattern=True, 257 enable_module_name_prefix_matching=True, 258 module_name=self.test_module_name, 259 expand_bitness=True) 260 261 def unpack_userparams(self, req_param_names=[], opt_param_names=[], **kwargs): 262 """Wrapper for test cases using ACTS runner API.""" 263 return self.getUserParams(req_param_names, opt_param_names, **kwargs) 264 265 def getUserParams(self, req_param_names=[], opt_param_names=[], **kwargs): 266 """Unpacks user defined parameters in test config into individual 267 variables. 268 269 Instead of accessing the user param with self.user_params["xxx"], the 270 variable can be directly accessed with self.xxx. 271 272 A missing required param will raise an exception. If an optional param 273 is missing, an INFO line will be logged. 274 275 Args: 276 req_param_names: A list of names of the required user params. 277 opt_param_names: A list of names of the optional user params. 278 **kwargs: Arguments that provide default values. 279 e.g. getUserParams(required_list, opt_list, arg_a="hello") 280 self.arg_a will be "hello" unless it is specified again in 281 required_list or opt_list. 282 283 Raises: 284 BaseTestError is raised if a required user params is missing from 285 test config. 286 """ 287 for k, v in kwargs.items(): 288 setattr(self, k, v) 289 for name in req_param_names: 290 if name not in self.user_params: 291 raise errors.BaseTestError(("Missing required user param '%s' " 292 "in test configuration.") % name) 293 setattr(self, name, self.user_params[name]) 294 for name in opt_param_names: 295 if name not in self.user_params: 296 logging.debug(("Missing optional user param '%s' in " 297 "configuration, continue."), name) 298 else: 299 setattr(self, name, self.user_params[name]) 300 301 def getUserParam(self, 302 param_name, 303 error_if_not_found=False, 304 warn_if_not_found=False, 305 default_value=None, 306 to_str=False): 307 """Get the value of a single user parameter. 308 309 This method returns the value of specified user parameter. 310 311 Note: unlike getUserParams(), this method will not automatically set 312 attribute using the parameter name and value. 313 314 Args: 315 param_name: string or list of string, denoting user parameter names. If provided 316 a single string, self.user_params["<param_name>"] will be accessed. 317 If provided multiple strings, 318 self.user_params["<param_name1>"]["<param_name2>"]["<param_name3>"]... 319 will be accessed. 320 error_if_not_found: bool, whether to raise error if parameter not 321 exists. Default: False 322 warn_if_not_found: bool, log a warning message if parameter value 323 not found. Default: False 324 default_value: object, default value to return if not found. 325 If error_if_not_found is true, this parameter has no 326 effect. Default: None 327 to_str: boolean, whether to convert the result object to string if 328 not None. 329 Note, strings passing in from java json config are often 330 unicode. 331 332 Returns: 333 object, value of the specified parameter name chain if exists; 334 <default_value> otherwise. 335 """ 336 337 def ToStr(return_value): 338 """Check to_str option and convert to string if not None""" 339 if to_str and return_value is not None: 340 return str(return_value) 341 return return_value 342 343 if not param_name: 344 if error_if_not_found: 345 raise errors.BaseTestError("empty param_name provided") 346 logging.error("empty param_name") 347 return ToStr(default_value) 348 349 if not isinstance(param_name, list): 350 param_name = [param_name] 351 352 curr_obj = self.user_params 353 for param in param_name: 354 if param not in curr_obj: 355 msg = ("Missing user param '%s' in test configuration.\n" 356 "User params: %s") % (param_name, self.user_params) 357 if error_if_not_found: 358 raise errors.BaseTestError(msg) 359 elif warn_if_not_found: 360 logging.warn(msg) 361 return ToStr(default_value) 362 curr_obj = curr_obj[param] 363 364 return ToStr(curr_obj) 365 366 def _getUserConfig(self, 367 config_type, 368 key, 369 default_value=None, 370 error_if_not_found=False, 371 warn_if_not_found=False, 372 to_str=False): 373 """Get the value of a user config given the key. 374 375 This method returns the value of specified user config type. 376 377 Args: 378 config_type: string, type of user config 379 key: string, key of the value string in string config map. 380 default_value: object, default value to return if not found. 381 If error_if_not_found is true, this parameter has no 382 effect. Default: None 383 error_if_not_found: bool, whether to raise error if parameter not 384 exists. Default: False 385 warn_if_not_found: bool, log a warning message if parameter value 386 not found. Default: False 387 to_str: boolean, whether to apply str() method to result value 388 if result is not None. 389 Note, strings passing in from java json config are ofen 390 unicode. 391 392 Returns: 393 Value in config matching the given key and type if exists; 394 <default_value> otherwise. 395 """ 396 dic = self.getUserParam(config_type, 397 error_if_not_found=False, 398 warn_if_not_found=False, 399 default_value=None, 400 to_str=False) 401 402 if dic is None or key not in dic: 403 msg = ("Config key %s not found in user config type %s.\n" 404 "User params: %s") % (key, config_type, self.user_params) 405 if error_if_not_found: 406 raise errors.BaseTestError(msg) 407 elif warn_if_not_found: 408 logging.warn(msg) 409 410 return default_value 411 412 return dic[key] if not to_str else str(dic[key]) 413 414 def getUserConfigStr(self, key, **kwargs): 415 """Get the value of a user config string given the key. 416 417 See _getUserConfig method for more details. 418 """ 419 kwargs["to_str"] = True 420 return self._getUserConfig(keys.ConfigKeys.IKEY_USER_CONFIG_STR, 421 key, 422 **kwargs) 423 424 def getUserConfigInt(self, key, **kwargs): 425 """Get the value of a user config int given the key. 426 427 See _getUserConfig method for more details. 428 """ 429 return self._getUserConfig(keys.ConfigKeys.IKEY_USER_CONFIG_INT, 430 key, 431 **kwargs) 432 433 def getUserConfigBool(self, key, **kwargs): 434 """Get the value of a user config bool given the key. 435 436 See _getUserConfig method for more details. 437 """ 438 return self._getUserConfig(keys.ConfigKeys.IKEY_USER_CONFIG_BOOL, 439 key, 440 **kwargs) 441 442 def _setUpClass(self): 443 """Proxy function to guarantee the base implementation of setUpClass 444 is called. 445 """ 446 event = tfi.Begin('_setUpClass method for test class', 447 tfi.categories.TEST_CLASS_SETUP) 448 timeout = self.timeout - time.time() + self.start_time_sec 449 if timeout < 0: 450 timeout = 1 451 self.resetTimeout(timeout) 452 if not precondition_utils.MeetFirstApiLevelPrecondition(self): 453 self.skipAllTests("The device's first API level doesn't meet the " 454 "precondition.") 455 for device in self.android_devices: 456 if not precondition_utils.CheckFeaturePrecondition(self, device): 457 self.skipAllTests("Precondition feature check fail.") 458 459 if (self.getUserParam( 460 keys.ConfigKeys.IKEY_DISABLE_FRAMEWORK, default_value=False) or 461 # @Deprecated Legacy configuration option name. 462 self.getUserParam( 463 keys.ConfigKeys.IKEY_BINARY_TEST_DISABLE_FRAMEWORK, 464 default_value=False)): 465 stop_native_server = ( 466 self.getUserParam( 467 keys.ConfigKeys.IKEY_STOP_NATIVE_SERVERS, 468 default_value=False) or 469 # @Deprecated Legacy configuration option name. 470 self.getUserParam( 471 keys.ConfigKeys.IKEY_BINARY_TEST_STOP_NATIVE_SERVERS, 472 default_value=False)) 473 # Disable the framework if requested. 474 for device in self.android_devices: 475 device.stop(stop_native_server) 476 else: 477 # Enable the framework if requested. 478 for device in self.android_devices: 479 device.start() 480 481 # Wait for the native service process to stop. 482 native_server_process_names = self.getUserParam( 483 keys.ConfigKeys.IKEY_NATIVE_SERVER_PROCESS_NAME, 484 default_value=[]) 485 for device in self.android_devices: 486 device.waitForProcessStop(native_server_process_names) 487 488 489 event_sub = tfi.Begin('setUpClass method from test script', 490 tfi.categories.TEST_CLASS_SETUP, 491 enable_logging=False) 492 result = self.setUpClass() 493 event_sub.End() 494 event.End() 495 return result 496 497 def setUpClass(self): 498 """Setup function that will be called before executing any test case in 499 the test class. 500 501 To signal setup failure, return False or raise an exception. If 502 exceptions were raised, the stack trace would appear in log, but the 503 exceptions would not propagate to upper levels. 504 505 Implementation is optional. 506 """ 507 pass 508 509 def _tearDownClass(self): 510 """Proxy function to guarantee the base implementation of tearDownClass 511 is called. 512 """ 513 event = tfi.Begin('_tearDownClass method for test class', 514 tfi.categories.TEST_CLASS_TEARDOWN) 515 self.cancelTimeout() 516 517 event_sub = tfi.Begin('tearDownClass method from test script', 518 tfi.categories.TEST_CLASS_TEARDOWN, 519 enable_logging=False) 520 521 @timeout_utils.timeout(TIMEOUT_SECS_TEARDOWN_CLASS, 522 message='tearDownClass method timed out.', 523 no_exception=True) 524 def _executeTearDownClass(baseTest): 525 baseTest._exec_func(baseTest.tearDownClass) 526 _executeTearDownClass(self) 527 528 event_sub.End() 529 530 if self.web.enabled: 531 if self.results.class_errors: 532 # Create a result to make the module shown as failure. 533 self.web.AddTestReport("setup_class") 534 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_FAIL) 535 536 # Attach log destination urls to proto message so the urls will be 537 # recorded and uploaded to dashboard. The actual log uploading is postponed 538 # after generating report message to prevent failure due to timeout of log uploading. 539 self.log_uploading.UploadLogs(dryrun=True) 540 541 message_b = self.web.GenerateReportMessage(self.results.requested, 542 self.results.executed) 543 else: 544 message_b = '' 545 546 report_proto_path = os.path.join(logging.log_path, 547 _REPORT_MESSAGE_FILE_NAME) 548 549 if message_b: 550 logging.debug('Result proto message path: %s', report_proto_path) 551 552 with open(report_proto_path, "wb") as f: 553 f.write(message_b) 554 555 if self.log_uploading.enabled: 556 @timeout_utils.timeout(TIMEOUT_SECS_LOG_UPLOADING, 557 message='_tearDownClass method in base_test timed out.', 558 no_exception=True) 559 def _executeLogUpload(_log_uploading): 560 _log_uploading.UploadLogs() 561 562 event_upload = tfi.Begin('Log upload', 563 tfi.categories.RESULT_PROCESSING) 564 _executeLogUpload(self.log_uploading) 565 event_upload.End() 566 567 event.End() 568 569 def tearDownClass(self): 570 """Teardown function that will be called after all the selected test 571 cases in the test class have been executed. 572 573 Implementation is optional. 574 """ 575 pass 576 577 def interrupt(self): 578 """Interrupts test execution and terminates process.""" 579 with self._interrupt_lock: 580 if self._interrupted: 581 logging.warning("Cannot interrupt more than once.") 582 return 583 self._interrupted = True 584 logging.info("Test timed out, interrupt") 585 utils.stop_current_process(TIMEOUT_SECS_TEARDOWN_CLASS) 586 587 def cancelTimeout(self): 588 """Cancels main thread timer.""" 589 if hasattr(signal, "SIGALRM"): 590 signal.alarm(0) 591 else: 592 with self._interrupt_lock: 593 if self._interrupted: 594 logging.warning("Test execution has been interrupted. " 595 "Cannot cancel or reset timeout.") 596 return 597 598 if self._timer: 599 self._timer.cancel() 600 601 def resetTimeout(self, timeout): 602 """Restarts the timer that will interrupt the main thread. 603 604 This class starts the timer before setUpClass. As the timeout depends 605 on number of generated tests, the subclass can restart the timer. 606 607 Args: 608 timeout: A float, wait time in seconds before interrupt. 609 """ 610 logging.debug("Start timer with timeout=%ssec.", timeout) 611 if hasattr(signal, "SIGALRM"): 612 signal.signal(signal.SIGALRM, utils._timeout_handler) 613 signal.alarm(int(timeout)) 614 else: 615 self.cancelTimeout() 616 617 self._timer = threading.Timer(timeout, self.interrupt) 618 self._timer.daemon = True 619 self._timer.start() 620 621 def _testEntry(self, test_record): 622 """Internal function to be called upon entry of a test case. 623 624 Args: 625 test_record: The TestResultRecord object for the test case going to 626 be executed. 627 """ 628 self._current_record = test_record 629 if self.web.enabled: 630 self.web.AddTestReport(test_record.test_name) 631 632 def _setUp(self, test_name): 633 """Proxy function to guarantee the base implementation of setUp is 634 called. 635 """ 636 event = tfi.Begin('_setUp method for test case', 637 tfi.categories.TEST_CASE_SETUP,) 638 if not self.Heal(passive=True): 639 msg = 'Framework self diagnose didn\'t pass for %s. Marking test as fail.' % test_name 640 logging.error(msg) 641 event.Remove(msg) 642 asserts.fail(msg) 643 644 if self.systrace.enabled: 645 self.systrace.StartSystrace() 646 647 event_sub = tfi.Begin('_setUp method from test script', 648 tfi.categories.TEST_CASE_SETUP, 649 enable_logging=False) 650 result = self.setUp() 651 event_sub.End() 652 event.End() 653 654 def setUp(self): 655 """Setup function that will be called every time before executing each 656 test case in the test class. 657 658 To signal setup failure, return False or raise an exception. If 659 exceptions were raised, the stack trace would appear in log, but the 660 exceptions would not propagate to upper levels. 661 662 Implementation is optional. 663 """ 664 665 def _testExit(self): 666 """Internal function to be called upon exit of a test.""" 667 self._current_record = None 668 669 def _tearDown(self, test_name): 670 """Proxy function to guarantee the base implementation of tearDown 671 is called. 672 """ 673 event = tfi.Begin('_tearDown method from base_test', 674 tfi.categories.TEST_CASE_TEARDOWN) 675 if self.systrace.enabled: 676 self._exec_func(self.systrace.ProcessAndUploadSystrace, test_name) 677 event_sub = tfi.Begin('_tearDown method from test script', 678 tfi.categories.TEST_CASE_TEARDOWN, 679 enable_logging=False) 680 self._exec_func(self.tearDown) 681 event_sub.End() 682 event.End() 683 684 def tearDown(self): 685 """Teardown function that will be called every time a test case has 686 been executed. 687 688 Implementation is optional. 689 """ 690 691 def _onFail(self): 692 """Proxy function to guarantee the base implementation of onFail is 693 called. 694 """ 695 record = self._current_record 696 logging.error(record.details) 697 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 698 logging.error(RESULT_LINE_TEMPLATE, self.results.progressStr, 699 record.test_name, record.result) 700 if self.web.enabled: 701 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_FAIL) 702 703 event = tfi.Begin('_onFail method from BaseTest', 704 tfi.categories.FAILED_TEST_CASE_PROCESSING, 705 enable_logging=False) 706 self.onFail(record.test_name, begin_time) 707 if self._bug_report_on_failure: 708 self.DumpBugReport(record.test_name) 709 if self._logcat_on_failure: 710 self.DumpLogcat(record.test_name) 711 event.End() 712 713 def onFail(self, test_name, begin_time): 714 """A function that is executed upon a test case failure. 715 716 User implementation is optional. 717 718 Args: 719 test_name: Name of the test that triggered this function. 720 begin_time: Logline format timestamp taken when the test started. 721 """ 722 723 def _onPass(self): 724 """Proxy function to guarantee the base implementation of onPass is 725 called. 726 """ 727 record = self._current_record 728 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 729 msg = record.details 730 if msg: 731 logging.debug(msg) 732 logging.info(RESULT_LINE_TEMPLATE, self.results.progressStr, 733 record.test_name, record.result) 734 if self.web.enabled: 735 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_PASS) 736 self.onPass(record.test_name, begin_time) 737 738 def onPass(self, test_name, begin_time): 739 """A function that is executed upon a test case passing. 740 741 Implementation is optional. 742 743 Args: 744 test_name: Name of the test that triggered this function. 745 begin_time: Logline format timestamp taken when the test started. 746 """ 747 748 def _onSkip(self): 749 """Proxy function to guarantee the base implementation of onSkip is 750 called. 751 """ 752 record = self._current_record 753 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 754 logging.info(RESULT_LINE_TEMPLATE, self.results.progressStr, 755 record.test_name, record.result) 756 logging.debug("Reason to skip: %s", record.details) 757 if self.web.enabled: 758 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_SKIP) 759 self.onSkip(record.test_name, begin_time) 760 761 def onSkip(self, test_name, begin_time): 762 """A function that is executed upon a test case being skipped. 763 764 Implementation is optional. 765 766 Args: 767 test_name: Name of the test that triggered this function. 768 begin_time: Logline format timestamp taken when the test started. 769 """ 770 771 def _onSilent(self): 772 """Proxy function to guarantee the base implementation of onSilent is 773 called. 774 """ 775 record = self._current_record 776 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 777 if self.web.enabled: 778 self.web.SetTestResult(None) 779 self.onSilent(record.test_name, begin_time) 780 781 def onSilent(self, test_name, begin_time): 782 """A function that is executed upon a test case being marked as silent. 783 784 Implementation is optional. 785 786 Args: 787 test_name: Name of the test that triggered this function. 788 begin_time: Logline format timestamp taken when the test started. 789 """ 790 791 def _onException(self): 792 """Proxy function to guarantee the base implementation of onException 793 is called. 794 """ 795 record = self._current_record 796 logging.exception(record.details) 797 begin_time = logger.epochToLogLineTimestamp(record.begin_time) 798 if self.web.enabled: 799 self.web.SetTestResult(ReportMsg.TEST_CASE_RESULT_EXCEPTION) 800 801 event = tfi.Begin('_onFail method from BaseTest', 802 tfi.categories.FAILED_TEST_CASE_PROCESSING, 803 enable_logging=False) 804 self.onException(record.test_name, begin_time) 805 if self._bug_report_on_failure: 806 self.DumpBugReport(ecord.test_name) 807 if self._logcat_on_failure: 808 self.DumpLogcat(record.test_name) 809 event.End() 810 811 def onException(self, test_name, begin_time): 812 """A function that is executed upon an unhandled exception from a test 813 case. 814 815 Implementation is optional. 816 817 Args: 818 test_name: Name of the test that triggered this function. 819 begin_time: Logline format timestamp taken when the test started. 820 """ 821 822 def _exec_procedure_func(self, func): 823 """Executes a procedure function like onPass, onFail etc. 824 825 This function will alternate the 'Result' of the test's record if 826 exceptions happened when executing the procedure function. 827 828 This will let signals.TestAbortAll through so abortAll works in all 829 procedure functions. 830 831 Args: 832 func: The procedure function to be executed. 833 """ 834 record = self._current_record 835 836 if (func not in (self._onPass, self._onSilent, self._onSkip) 837 and not self._is_final_run): 838 logging.debug('Skipping test failure procedure function during retry') 839 logging.info(RESULT_LINE_TEMPLATE, self.results.progressStr, 840 record.test_name, 'RETRY') 841 return 842 843 if record is None: 844 logging.error("Cannot execute %s. No record for current test.", 845 func.__name__) 846 return 847 try: 848 func() 849 except signals.TestAbortAll as e: 850 raise signals.TestAbortAll, e, sys.exc_info()[2] 851 except Exception as e: 852 logging.exception("Exception happened when executing %s for %s.", 853 func.__name__, record.test_name) 854 record.addError(func.__name__, e) 855 856 def addTableToResult(self, name, rows): 857 """Adds a table to current test record. 858 859 A subclass can call this method to add a table to _current_record when 860 running test cases. 861 862 Args: 863 name: String, the table name. 864 rows: A 2-dimensional list which contains the data. 865 """ 866 self._current_record.addTable(name, rows) 867 868 def filterOneTest(self, test_name, test_filter=None): 869 """Check test filters for a test name. 870 871 The first layer of filter is user defined test filters: 872 if a include filter is not empty, only tests in include filter will 873 be executed regardless whether they are also in exclude filter. Else 874 if include filter is empty, only tests not in exclude filter will be 875 executed. 876 877 The second layer of filter is checking whether skipAllTests method is 878 called. If the flag is set, this method raises signals.TestSkip. 879 880 The third layer of filter is checking abi bitness: 881 if a test has a suffix indicating the intended architecture bitness, 882 and the current abi bitness information is available, non matching tests 883 will be skipped. By our convention, this function will look for bitness in suffix 884 formated as "32bit", "32Bit", "32BIT", or 64 bit equivalents. 885 886 This method assumes const.SUFFIX_32BIT and const.SUFFIX_64BIT are in lower cases. 887 888 Args: 889 test_name: string, name of a test 890 test_filter: Filter object, test filter 891 892 Raises: 893 signals.TestSilent if a test should not be executed 894 signals.TestSkip if a test should be logged but not be executed 895 """ 896 if self._test_filter_retry and not self._test_filter_retry.Filter(test_name): 897 # TODO: TestSilent will remove test case from record, 898 # TestSkip will record test skip with a reason. 899 # skip during retry is neither of these, as the test should be skipped, 900 # not being recorded and not being deleted. 901 # Create a new signal type if callers outside this class wants to distinguish 902 # between these skip types. 903 raise signals.TestSkip('Skipping completed tests in retry run attempt.') 904 905 self._filterOneTestThroughTestFilter(test_name, test_filter) 906 self._filterOneTestThroughAbiBitness(test_name) 907 908 def _filterOneTestThroughTestFilter(self, test_name, test_filter=None): 909 """Check test filter for the given test name. 910 911 Args: 912 test_name: string, name of a test 913 914 Raises: 915 signals.TestSilent if a test should not be executed 916 signals.TestSkip if a test should be logged but not be executed 917 """ 918 if not test_filter: 919 test_filter = self.test_filter 920 921 if not test_filter.Filter(test_name): 922 raise signals.TestSilent("Test case '%s' did not pass filters.") 923 924 if self.isSkipAllTests(): 925 raise signals.TestSkip(self.getSkipAllTestsReason()) 926 927 def _filterOneTestThroughAbiBitness(self, test_name): 928 """Check test filter for the given test name. 929 930 Args: 931 test_name: string, name of a test 932 933 Raises: 934 signals.TestSilent if a test should not be executed 935 """ 936 asserts.skipIf( 937 self.abi_bitness and 938 ((self.skip_on_32bit_abi is True) and self.abi_bitness == "32") or 939 ((self.skip_on_64bit_abi is True) and self.abi_bitness == "64") or 940 (test_name.lower().endswith(const.SUFFIX_32BIT) and 941 self.abi_bitness != "32") or 942 (test_name.lower().endswith(const.SUFFIX_64BIT) and 943 self.abi_bitness != "64" and not self.run_32bit_on_64bit_abi), 944 "Test case '{}' excluded as ABI bitness is {}.".format( 945 test_name, self.abi_bitness)) 946 947 def execOneTest(self, test_name, test_func, args, **kwargs): 948 """Executes one test case and update test results. 949 950 Executes one test case, create a records.TestResultRecord object with 951 the execution information, and add the record to the test class's test 952 results. 953 954 Args: 955 test_name: Name of the test. 956 test_func: The test function. 957 args: A tuple of params. 958 kwargs: Extra kwargs. 959 """ 960 if self._test_filter_retry and not self._test_filter_retry.Filter(test_name): 961 return 962 963 is_silenced = False 964 tr_record = records.TestResultRecord(test_name, self.test_module_name) 965 tr_record.testBegin() 966 logging.info(TEST_CASE_TEMPLATE, self.results.progressStr, test_name) 967 verdict = None 968 finished = False 969 try: 970 ret = self._testEntry(tr_record) 971 asserts.assertTrue(ret is not False, 972 "Setup test entry for %s failed." % test_name) 973 self.filterOneTest(test_name) 974 if self.collect_tests_only: 975 asserts.explicitPass("Collect tests only.") 976 977 try: 978 ret = self._setUp(test_name) 979 asserts.assertTrue(ret is not False, 980 "Setup for %s failed." % test_name) 981 982 event_test = tfi.Begin("test function", 983 tfi.categories.TEST_CASE_EXECUTION) 984 if args or kwargs: 985 verdict = test_func(*args, **kwargs) 986 else: 987 verdict = test_func() 988 event_test.End() 989 finished = True 990 finally: 991 self._tearDown(test_name) 992 except (signals.TestFailure, AssertionError) as e: 993 tr_record.testFail(e) 994 self._exec_procedure_func(self._onFail) 995 finished = True 996 except signals.TestSkip as e: 997 # Test skipped. 998 tr_record.testSkip(e) 999 self._exec_procedure_func(self._onSkip) 1000 finished = True 1001 except signals.TestAbortClass as e: 1002 # Abort signals, pass along. 1003 tr_record.testFail(e) 1004 self._is_final_run = True 1005 finished = True 1006 raise signals.TestAbortClass, e, sys.exc_info()[2] 1007 except signals.TestAbortAll as e: 1008 # Abort signals, pass along. 1009 tr_record.testFail(e) 1010 self._is_final_run = True 1011 finished = True 1012 raise signals.TestAbortAll, e, sys.exc_info()[2] 1013 except utils.TimeoutError as e: 1014 logging.exception(e) 1015 # Mark current test case as failure and abort remaining tests. 1016 tr_record.testFail(e) 1017 self._is_final_run = True 1018 finished = True 1019 raise signals.TestAbortAll, e, sys.exc_info()[2] 1020 except KeyboardInterrupt as e: 1021 logging.error("Received KeyboardInterrupt signal") 1022 # Abort signals, pass along. 1023 tr_record.testFail(e) 1024 self._is_final_run = True 1025 finished = True 1026 raise 1027 except AdbError as e: 1028 logging.error(e) 1029 if not self.Heal(): 1030 # Non-recoverable adb failure. Mark test failure and abort 1031 tr_record.testFail(e) 1032 self._is_final_run = True 1033 finished = True 1034 raise signals.TestAbortAll, e, sys.exc_info()[2] 1035 # error specific to the test case, mark test failure and continue with remaining test 1036 tr_record.testFail(e) 1037 self._exec_procedure_func(self._onFail) 1038 finished = True 1039 except signals.TestPass as e: 1040 # Explicit test pass. 1041 tr_record.testPass(e) 1042 self._exec_procedure_func(self._onPass) 1043 finished = True 1044 except signals.TestSilent as e: 1045 # Suppress test reporting. 1046 is_silenced = True 1047 self._exec_procedure_func(self._onSilent) 1048 self.results.removeRecord(tr_record) 1049 finished = True 1050 except Exception as e: 1051 # Exception happened during test. 1052 logging.exception(e) 1053 tr_record.testError(e) 1054 self._exec_procedure_func(self._onException) 1055 self._exec_procedure_func(self._onFail) 1056 finished = True 1057 else: 1058 # Keep supporting return False for now. 1059 # TODO(angli): Deprecate return False support. 1060 if verdict or (verdict is None): 1061 # Test passed. 1062 tr_record.testPass() 1063 self._exec_procedure_func(self._onPass) 1064 return 1065 # Test failed because it didn't return True. 1066 # This should be removed eventually. 1067 tr_record.testFail() 1068 self._exec_procedure_func(self._onFail) 1069 finished = True 1070 finally: 1071 if not finished: 1072 for device in self.android_devices: 1073 # if shell has not been set up yet 1074 if device.shell is not None: 1075 device.shell.DisableShell() 1076 1077 logging.error('Test timed out.') 1078 tr_record.testError() 1079 self._exec_procedure_func(self._onException) 1080 self._exec_procedure_func(self._onFail) 1081 1082 if not is_silenced: 1083 self.results.addRecord(tr_record) 1084 self._testExit() 1085 1086 def runGeneratedTests(self, 1087 test_func, 1088 settings, 1089 args=None, 1090 kwargs=None, 1091 tag="", 1092 name_func=None): 1093 """Runs generated test cases. 1094 1095 Generated test cases are not written down as functions, but as a list 1096 of parameter sets. This way we reduce code repetition and improve 1097 test case scalability. 1098 1099 Args: 1100 test_func: The common logic shared by all these generated test 1101 cases. This function should take at least one argument, 1102 which is a parameter set. 1103 settings: A list of strings representing parameter sets. These are 1104 usually json strings that get loaded in the test_func. 1105 args: Iterable of additional position args to be passed to 1106 test_func. 1107 kwargs: Dict of additional keyword args to be passed to test_func 1108 tag: Name of this group of generated test cases. Ignored if 1109 name_func is provided and operates properly. 1110 name_func: A function that takes a test setting and generates a 1111 proper test name. The test name should be shorter than 1112 utils.MAX_FILENAME_LEN. Names over the limit will be 1113 truncated. 1114 1115 Returns: 1116 A list of settings that did not pass. 1117 """ 1118 args = args or () 1119 kwargs = kwargs or {} 1120 failed_settings = [] 1121 1122 def GenerateTestName(setting): 1123 test_name = "{} {}".format(tag, setting) 1124 if name_func: 1125 try: 1126 test_name = name_func(setting, *args, **kwargs) 1127 except: 1128 logging.exception(("Failed to get test name from " 1129 "test_func. Fall back to default %s"), 1130 test_name) 1131 1132 if len(test_name) > utils.MAX_FILENAME_LEN: 1133 test_name = test_name[:utils.MAX_FILENAME_LEN] 1134 1135 return test_name 1136 1137 for setting in settings: 1138 test_name = GenerateTestName(setting) 1139 1140 tr_record = records.TestResultRecord(test_name, self.test_module_name) 1141 self.results.requested.append(tr_record) 1142 1143 for setting in settings: 1144 test_name = GenerateTestName(setting) 1145 previous_success_cnt = len(self.results.passed) 1146 1147 event_exec = tfi.Begin('BaseTest execOneTest method for generated tests', 1148 enable_logging=False) 1149 self.execOneTest(test_name, test_func, (setting, ) + args, **kwargs) 1150 event_exec.End() 1151 if len(self.results.passed) - previous_success_cnt != 1: 1152 failed_settings.append(setting) 1153 1154 return failed_settings 1155 1156 def _exec_func(self, func, *args): 1157 """Executes a function with exception safeguard. 1158 1159 This will let signals.TestAbortAll through so abortAll works in all 1160 procedure functions. 1161 1162 Args: 1163 func: Function to be executed. 1164 args: Arguments to be passed to the function. 1165 1166 Returns: 1167 Whatever the function returns, or False if non-caught exception 1168 occurred. 1169 """ 1170 try: 1171 return func(*args) 1172 except signals.TestAbortAll as e: 1173 raise signals.TestAbortAll, e, sys.exc_info()[2] 1174 except: 1175 logging.exception("Exception happened when executing %s in %s.", 1176 func.__name__, self.test_module_name) 1177 return False 1178 1179 def _get_all_test_names(self): 1180 """Finds all the function names that match the test case naming 1181 convention in this class. 1182 1183 Returns: 1184 A list of strings, each is a test case name. 1185 """ 1186 test_names = [] 1187 for name in dir(self): 1188 if name.startswith(STR_TEST) or name.startswith(STR_GENERATE): 1189 attr_func = getattr(self, name) 1190 if hasattr(attr_func, "__call__"): 1191 test_names.append(name) 1192 return test_names 1193 1194 def _get_test_funcs(self, test_names): 1195 """Obtain the actual functions of test cases based on test names. 1196 1197 Args: 1198 test_names: A list of strings, each string is a test case name. 1199 1200 Returns: 1201 A list of tuples of (string, function). String is the test case 1202 name, function is the actual test case function. 1203 1204 Raises: 1205 errors.USERError is raised if the test name does not follow 1206 naming convention "test_*". This can only be caused by user input 1207 here. 1208 """ 1209 test_funcs = [] 1210 for test_name in test_names: 1211 if not hasattr(self, test_name): 1212 logging.warning("%s does not have test case %s.", 1213 self.test_module_name, test_name) 1214 elif (test_name.startswith(STR_TEST) or 1215 test_name.startswith(STR_GENERATE)): 1216 test_funcs.append((test_name, getattr(self, test_name))) 1217 else: 1218 msg = ("Test case name %s does not follow naming convention " 1219 "test*, abort.") % test_name 1220 raise errors.USERError(msg) 1221 1222 return test_funcs 1223 1224 def getTests(self, test_names=None): 1225 """Get the test cases within a test class. 1226 1227 Args: 1228 test_names: A list of string that are test case names requested in 1229 cmd line. 1230 1231 Returns: 1232 A list of tuples of (string, function). String is the test case 1233 name, function is the actual test case function. 1234 """ 1235 if not test_names: 1236 if self.tests: 1237 # Specified by run list in class. 1238 test_names = list(self.tests) 1239 else: 1240 # No test case specified by user, execute all in the test class 1241 test_names = self._get_all_test_names() 1242 1243 tests = self._get_test_funcs(test_names) 1244 return tests 1245 1246 def _DiagnoseHost(self): 1247 """Runs diagnosis commands on host and logs the results.""" 1248 commands = ['ps aux | grep adb', 1249 'adb --version', 1250 'adb devices'] 1251 for cmd in commands: 1252 results = cmd_utils.ExecuteShellCommand(cmd) 1253 logging.debug('host diagnosis command %s', cmd) 1254 logging.debug(' output: %s', results[cmd_utils.STDOUT][0]) 1255 1256 def _DiagnoseDevice(self, device): 1257 """Runs diagnosis commands on device and logs the results.""" 1258 commands = ['ps aux | grep vts', 1259 'cat /proc/meminfo'] 1260 for cmd in commands: 1261 results = device.adb.shell(cmd, no_except=True, timeout=adb.DEFAULT_ADB_SHORT_TIMEOUT) 1262 logging.debug('device diagnosis command %s', cmd) 1263 logging.debug(' output: %s', results[const.STDOUT]) 1264 1265 def Heal(self, passive=False, timeout=900): 1266 """Performs a self healing. 1267 1268 Includes self diagnosis that looks for any framework or device state errors. 1269 Includes self recovery that attempts to correct discovered errors. 1270 1271 Args: 1272 passive: bool, whether to perform passive only self-check. A passive check means 1273 only to check known status stored in memory. No command will be issued 1274 to host or device - which makes the check fast. 1275 timeout: int, timeout in seconds. 1276 1277 Returns: 1278 bool, True if everything is ok. Fales if some error is not recoverable. 1279 """ 1280 start = time.time() 1281 1282 if not passive: 1283 available_devices = android_device.list_adb_devices() 1284 self._DiagnoseHost() 1285 1286 for device in self.android_devices: 1287 if device.serial not in available_devices: 1288 device.log.warn( 1289 "device become unavailable after tests. wait for device come back") 1290 _timeout = timeout - time.time() + start 1291 if _timeout < 0 or not device.waitForBootCompletion(timeout=_timeout): 1292 device.log.error('failed to restore device %s') 1293 return False 1294 device.rootAdb() 1295 device.stopServices() 1296 device.startServices() 1297 self._DiagnoseHost() 1298 else: 1299 self._DiagnoseDevice(device) 1300 1301 return all(map(lambda device: device.Heal(), self.android_devices)) 1302 1303 def runTestsWithRetry(self, tests): 1304 """Run tests with retry and collect test results. 1305 1306 Args: 1307 tests: A list of tests to be run. 1308 """ 1309 for count in range(self.max_retry_count + 1): 1310 if count: 1311 if not self.Heal(): 1312 logging.error('Self heal failed. ' 1313 'Some error is not recoverable within time constraint.') 1314 return 1315 1316 include_filter = map(lambda item: item.test_name, 1317 self.results.getNonPassingRecords(skipped=False)) 1318 self._test_filter_retry = filter_utils.Filter(include_filter=include_filter) 1319 logging.info('Automatically retrying %s test cases. Run attempt %s of %s', 1320 len(include_filter), 1321 count + 1, 1322 self.max_retry_count + 1) 1323 msg = 'Retrying the following test cases: %s' % include_filter 1324 logging.debug(msg) 1325 1326 path_retry_log = os.path.join(logging.log_path, 'retry_log.txt') 1327 with open(path_retry_log, 'a+') as f: 1328 f.write(msg + '\n') 1329 1330 self._is_final_run = count == self.max_retry_count 1331 1332 try: 1333 self._runTests(tests) 1334 except Exception as e: 1335 if self._is_final_run: 1336 raise e 1337 1338 if self._is_final_run: 1339 break 1340 1341 def _runTests(self, tests): 1342 """Run tests and collect test results. 1343 1344 Args: 1345 tests: A list of tests to be run. 1346 """ 1347 # Setup for the class with retry. 1348 for i in xrange(_SETUP_RETRY_NUMBER): 1349 setup_done = False 1350 caught_exception = None 1351 try: 1352 if self._setUpClass() is False: 1353 raise signals.TestFailure( 1354 "Failed to setup %s." % self.test_module_name) 1355 else: 1356 setup_done = True 1357 except Exception as e: 1358 caught_exception = e 1359 logging.exception("Failed to setup %s.", self.test_module_name) 1360 finally: 1361 if setup_done: 1362 break 1363 elif not caught_exception or i + 1 == _SETUP_RETRY_NUMBER: 1364 self.results.failClass(self.test_module_name, 1365 caught_exception) 1366 self._exec_func(self._tearDownClass) 1367 self._is_final_run = True 1368 return 1369 else: 1370 # restart services before retry setup. 1371 for device in self.android_devices: 1372 logging.info("restarting service on device %s", device.serial) 1373 device.stopServices() 1374 device.startServices() 1375 1376 class_error = None 1377 # Run tests in order. 1378 try: 1379 # Check if module is running in self test mode. 1380 if self.run_as_vts_self_test: 1381 logging.debug('setUpClass function was executed successfully.') 1382 self.results.passClass(self.test_module_name) 1383 return 1384 1385 for test_name, test_func in tests: 1386 if test_name.startswith(STR_GENERATE): 1387 logging.debug( 1388 "Executing generated test trigger function '%s'", 1389 test_name) 1390 test_func() 1391 logging.debug("Finished '%s'", test_name) 1392 else: 1393 event_exec = tfi.Begin('BaseTest execOneTest method for individual tests', 1394 enable_logging=False) 1395 self.execOneTest(test_name, test_func, None) 1396 event_exec.End() 1397 if self.isSkipAllTests() and not self.results.executed: 1398 self.results.skipClass( 1399 self.test_module_name, 1400 "All test cases skipped; unable to find any test case.") 1401 except signals.TestAbortClass as e: 1402 logging.error("Received TestAbortClass signal") 1403 class_error = e 1404 self._is_final_run = True 1405 except signals.TestAbortAll as e: 1406 logging.info("Received TestAbortAll signal") 1407 class_error = e 1408 self._is_final_run = True 1409 # Piggy-back test results on this exception object so we don't lose 1410 # results from this test class. 1411 setattr(e, "results", self.results) 1412 raise signals.TestAbortAll, e, sys.exc_info()[2] 1413 except KeyboardInterrupt as e: 1414 class_error = e 1415 self._is_final_run = True 1416 # Piggy-back test results on this exception object so we don't lose 1417 # results from this test class. 1418 setattr(e, "results", self.results) 1419 raise 1420 except Exception as e: 1421 # Exception happened during test. 1422 logging.exception(e) 1423 class_error = e 1424 raise e 1425 finally: 1426 if not self.results.getNonPassingRecords(skipped=False): 1427 self._is_final_run = True 1428 1429 if class_error and self._is_final_run: 1430 self.results.failClass(self.test_module_name, class_error) 1431 1432 self._exec_func(self._tearDownClass) 1433 1434 if self._is_final_run: 1435 if self.web.enabled: 1436 name, timestamp = self.web.GetTestModuleKeys() 1437 self.results.setTestModuleKeys(name, timestamp) 1438 1439 logging.info("Summary for test class %s: %s", 1440 self.test_module_name, self.results.summary()) 1441 1442 def run(self, test_names=None): 1443 """Runs test cases within a test class by the order they appear in the 1444 execution list. 1445 1446 One of these test cases lists will be executed, shown here in priority 1447 order: 1448 1. The test_names list, which is passed from cmd line. Invalid names 1449 are guarded by cmd line arg parsing. 1450 2. The self.tests list defined in test class. Invalid names are 1451 ignored. 1452 3. All function that matches test case naming convention in the test 1453 class. 1454 1455 Args: 1456 test_names: A list of string that are test case names requested in 1457 cmd line. 1458 1459 Returns: 1460 The test results object of this class. 1461 """ 1462 logging.info("==========> %s <==========", self.test_module_name) 1463 # Devise the actual test cases to run in the test class. 1464 tests = self.getTests(test_names) 1465 1466 if not self.run_as_vts_self_test: 1467 self.results.requested = [ 1468 records.TestResultRecord(test_name, self.test_module_name) 1469 for test_name,_ in tests if test_name.startswith(STR_TEST) 1470 ] 1471 1472 self.runTestsWithRetry(tests) 1473 return self.results 1474 1475 def cleanUp(self): 1476 """A function that is executed upon completion of all tests cases 1477 selected in the test class. 1478 1479 This function should clean up objects initialized in the constructor by 1480 user. 1481 """ 1482 1483 def DumpBugReport(self, prefix=''): 1484 """Get device bugreport through adb command. 1485 1486 Args: 1487 prefix: string, file name prefix. Usually in format of 1488 <test_module>-<test_case> 1489 """ 1490 event = tfi.Begin('dump Bugreport', 1491 tfi.categories.FAILED_TEST_CASE_PROCESSING) 1492 if prefix: 1493 prefix = re.sub('[^\w\-_\. ]', '_', prefix) + '_' 1494 1495 parent_dir = os.path.join(logging.log_path, 'bugreport') 1496 1497 if not file_util.Mkdir(parent_dir): 1498 logging.error('Failed to create bugreport output directory %s', parent_dir) 1499 return 1500 1501 for device in self.android_devices: 1502 if device.fatal_error: continue 1503 file_name = (_BUG_REPORT_FILE_PREFIX 1504 + prefix 1505 + '_%s' % device.serial 1506 + _BUG_REPORT_FILE_EXTENSION) 1507 1508 file_path = os.path.join(parent_dir, file_name) 1509 1510 logging.info('Dumping bugreport %s...' % file_path) 1511 device.adb.bugreport(file_path) 1512 event.End() 1513 1514 def skipAllTestsIf(self, condition, msg): 1515 """Skip all test cases if the given condition is true. 1516 1517 This method is usually called in setup functions when a precondition 1518 to the test module is not met. 1519 1520 Args: 1521 condition: object that can be evaluated by bool(), a condition that 1522 will trigger skipAllTests if evaluated to be True. 1523 msg: string, reason why tests are skipped. If set to None or empty 1524 string, a default message will be used (not recommended) 1525 """ 1526 if condition: 1527 self.skipAllTests(msg) 1528 1529 def skipAllTests(self, msg): 1530 """Skip all test cases. 1531 1532 This method is usually called in setup functions when a precondition 1533 to the test module is not met. 1534 1535 Args: 1536 msg: string, reason why tests are skipped. If set to None or empty 1537 string, a default message will be used (not recommended) 1538 """ 1539 if not msg: 1540 msg = "No reason provided." 1541 1542 setattr(self, _REASON_TO_SKIP_ALL_TESTS, msg) 1543 1544 def isSkipAllTests(self): 1545 """Returns whether all tests are set to be skipped. 1546 1547 Note: If all tests are being skipped not due to skipAllTests 1548 being called, or there is no tests defined, this method will 1549 still return False (since skipAllTests is not called.) 1550 1551 Returns: 1552 bool, True if skipAllTests has been called; False otherwise. 1553 """ 1554 return self.getSkipAllTestsReason() is not None 1555 1556 def getSkipAllTestsReason(self): 1557 """Returns the reason why all tests are skipped. 1558 1559 Note: If all tests are being skipped not due to skipAllTests 1560 being called, or there is no tests defined, this method will 1561 still return None (since skipAllTests is not called.) 1562 1563 Returns: 1564 String, reason why tests are skipped. None if skipAllTests 1565 is not called. 1566 """ 1567 return getattr(self, _REASON_TO_SKIP_ALL_TESTS, None) 1568 1569 def DumpLogcat(self, prefix=''): 1570 """Dumps device logcat outputs to log directory. 1571 1572 Args: 1573 prefix: string, file name prefix. Usually in format of 1574 <test_module>-<test_case> 1575 """ 1576 event = tfi.Begin('dump logcat', 1577 tfi.categories.FAILED_TEST_CASE_PROCESSING) 1578 if prefix: 1579 prefix = re.sub('[^\w\-_\. ]', '_', prefix) + '_' 1580 1581 parent_dir = os.path.join(logging.log_path, 'logcat') 1582 1583 if not file_util.Mkdir(parent_dir): 1584 logging.error('Failed to create bugreport output directory %s', parent_dir) 1585 return 1586 1587 for device in self.android_devices: 1588 if (not device.isAdbLogcatOn) or device.fatal_error: 1589 continue 1590 for buffer in LOGCAT_BUFFERS: 1591 file_name = (_LOGCAT_FILE_PREFIX 1592 + prefix 1593 + '_%s_' % buffer 1594 + device.serial 1595 + _LOGCAT_FILE_EXTENSION) 1596 1597 file_path = os.path.join(parent_dir, file_name) 1598 1599 logging.info('Dumping logcat %s...' % file_path) 1600 device.adb.logcat('-b', buffer, '-d', '>', file_path) 1601 event.End() 1602