1# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import contextlib, fcntl, logging, os, re, shutil 6 7import common 8from autotest_lib.client.bin import test, utils 9from autotest_lib.client.common_lib import error 10from autotest_lib.client.cros import constants, cros_logging 11 12 13class CrashTest(test.test): 14 """ 15 This class deals with running crash tests, which are tests which crash a 16 user-space program (or the whole machine) and generate a core dump. We 17 want to check that the correct crash dump is available and can be 18 retrieved. 19 20 Chromium OS has a crash sender which checks for new crash data and sends 21 it to a server. This crash data is used to track software quality and find 22 bugs. The system crash sender normally is always running, but can be paused 23 by creating _PAUSE_FILE. When crash sender sees this, it pauses operation. 24 25 The pid of the system crash sender is stored in _CRASH_SENDER_RUN_PATH so 26 we can use this to kill the system crash sender for when we want to run 27 our own. 28 29 For testing purposes we sometimes want to run the crash sender manually. 30 In this case we can set 'OVERRIDE_PAUSE_SENDING=1' in the environment and 31 run the crash sender manually (as a child process). 32 33 Also for testing we sometimes want to mock out the crash sender, and just 34 have it pretend to succeed or fail. The _MOCK_CRASH_SENDING file is used 35 for this. If it doesn't exist, then the crash sender runs normally. If 36 it exists but is empty, the crash sender will succeed (but actually do 37 nothing). If the file contains something, then the crash sender will fail. 38 39 If the user consents to sending crash tests, then the _CONSENT_FILE will 40 exist in the home directory. This test needs to create this file for the 41 crash sending to work. 42 43 Crash reports are rate limited to a certain number of reports each 24 44 hours. If the maximum number has already been sent then reports are held 45 until later. This is administered by a directory _CRASH_SENDER_RATE_DIR 46 which contains one temporary file for each time a report is sent. 47 48 The class provides the ability to push a consent file. This disables 49 consent for this test but allows it to be popped back at later. This 50 makes nested tests easier. If _automatic_consent_saving is True (the 51 default) then consent will be pushed at the start and popped at the end. 52 53 Interesting variables: 54 _log_reader: the log reader used for reading log files 55 _leave_crash_sending: True to enable crash sending on exit from the 56 test, False to disable it. (Default True). 57 _automatic_consent_saving: True to push the consent at the start of 58 the test and pop it afterwards. (Default True). 59 60 Useful places to look for more information are: 61 62 chromeos/src/platform/crash-reporter/crash_sender 63 - sender script which crash crash reporter to create reports, then 64 65 chromeos/src/platform/crash-reporter/ 66 - crash reporter program 67 """ 68 69 70 _CONSENT_FILE = '/home/chronos/Consent To Send Stats' 71 _CORE_PATTERN = '/proc/sys/kernel/core_pattern' 72 _CRASH_REPORTER_PATH = '/sbin/crash_reporter' 73 _CRASH_SENDER_PATH = '/sbin/crash_sender' 74 _CRASH_SENDER_RATE_DIR = '/var/lib/crash_sender' 75 _CRASH_SENDER_RUN_PATH = '/var/run/crash_sender.pid' 76 _CRASH_SENDER_LOCK_PATH = '/var/lock/crash_sender' 77 _CRASH_TEST_IN_PROGRESS = '/tmp/crash-test-in-progress' 78 _MOCK_CRASH_SENDING = '/tmp/mock-crash-sending' 79 _PAUSE_FILE = '/var/lib/crash_sender_paused' 80 _SYSTEM_CRASH_DIR = '/var/spool/crash' 81 _FALLBACK_USER_CRASH_DIR = '/home/chronos/crash' 82 _USER_CRASH_DIRS = '/home/chronos/u-*/crash' 83 84 # Use the same file format as crash does normally: 85 # <basename>.#.#.#.meta 86 _FAKE_TEST_BASENAME = 'fake.1.2.3' 87 88 def _set_system_sending(self, is_enabled): 89 """Sets whether or not the system crash_sender is allowed to run. 90 91 This is done by creating or removing _PAUSE_FILE. 92 93 crash_sender may still be allowed to run if _set_child_sending is 94 called with True and it is run as a child process. 95 96 @param is_enabled: True to enable crash_sender, False to disable it. 97 """ 98 if is_enabled: 99 if os.path.exists(self._PAUSE_FILE): 100 os.remove(self._PAUSE_FILE) 101 else: 102 utils.system('touch ' + self._PAUSE_FILE) 103 104 105 def _set_child_sending(self, is_enabled): 106 """Overrides crash sending enabling for child processes. 107 108 When the system crash sender is disabled this test can manually run 109 the crash sender as a child process. Normally this would do nothing, 110 but this function sets up crash_sender to ignore its disabled status 111 and do its job. 112 113 @param is_enabled: True to enable crash sending for child processes. 114 """ 115 if is_enabled: 116 os.environ['OVERRIDE_PAUSE_SENDING'] = "1" 117 else: 118 del os.environ['OVERRIDE_PAUSE_SENDING'] 119 120 121 def _set_force_official(self, is_enabled): 122 """Sets whether or not reports will upload for unofficial versions. 123 124 Normally, crash reports are only uploaded for official build 125 versions. If the override is set, however, they will also be 126 uploaded for unofficial versions. 127 128 @param is_enabled: True to enable uploading for unofficial versions. 129 """ 130 if is_enabled: 131 os.environ['FORCE_OFFICIAL'] = "1" 132 elif os.environ.get('FORCE_OFFICIAL'): 133 del os.environ['FORCE_OFFICIAL'] 134 135 136 def _set_mock_developer_mode(self, is_enabled): 137 """Sets whether or not we should pretend we booted in developer mode. 138 139 @param is_enabled: True to pretend we are in developer mode. 140 """ 141 if is_enabled: 142 os.environ['MOCK_DEVELOPER_MODE'] = "1" 143 elif os.environ.get('MOCK_DEVELOPER_MODE'): 144 del os.environ['MOCK_DEVELOPER_MODE'] 145 146 147 def _reset_rate_limiting(self): 148 """Reset the count of crash reports sent today. 149 150 This clears the contents of the rate limiting directory which has 151 the effect of reseting our count of crash reports sent. 152 """ 153 utils.system('rm -rf ' + self._CRASH_SENDER_RATE_DIR) 154 155 156 def _clear_spooled_crashes(self): 157 """Clears system and user crash directories. 158 159 This will remove all crash reports which are waiting to be sent. 160 """ 161 utils.system('rm -rf ' + self._SYSTEM_CRASH_DIR) 162 utils.system('rm -rf %s %s' % (self._USER_CRASH_DIRS, 163 self._FALLBACK_USER_CRASH_DIR)) 164 165 166 def _kill_running_sender(self): 167 """Kill the the crash_sender process if running. 168 169 We use the PID file to find the process ID, then kill it with signal 9. 170 """ 171 if not os.path.exists(self._CRASH_SENDER_RUN_PATH): 172 return 173 running_pid = int(utils.read_file(self._CRASH_SENDER_RUN_PATH)) 174 logging.warning('Detected running crash sender (%d), killing', 175 running_pid) 176 utils.system('kill -9 %d' % running_pid) 177 os.remove(self._CRASH_SENDER_RUN_PATH) 178 179 180 def _set_sending_mock(self, mock_enabled, send_success=True): 181 """Enables / disables mocking of the sending process. 182 183 This uses the _MOCK_CRASH_SENDING file to achieve its aims. See notes 184 at the top. 185 186 @param mock_enabled: If True, mocking is enabled, else it is disabled. 187 @param send_success: If mock_enabled this is True for the mocking to 188 indicate success, False to indicate failure. 189 """ 190 if mock_enabled: 191 if send_success: 192 data = '' 193 else: 194 data = '1' 195 logging.info('Setting sending mock') 196 utils.open_write_close(self._MOCK_CRASH_SENDING, data) 197 else: 198 utils.system('rm -f ' + self._MOCK_CRASH_SENDING) 199 200 201 def _set_consent(self, has_consent): 202 """Sets whether or not we have consent to send crash reports. 203 204 This creates or deletes the _CONSENT_FILE to control whether 205 crash_sender will consider that it has consent to send crash reports. 206 It also copies a policy blob with the proper policy setting. 207 208 @param has_consent: True to indicate consent, False otherwise 209 """ 210 autotest_cros_dir = os.path.join(os.path.dirname(__file__), '..') 211 if has_consent: 212 if os.path.isdir(constants.WHITELIST_DIR): 213 # Create policy file that enables metrics/consent. 214 shutil.copy('%s/mock_metrics_on.policy' % autotest_cros_dir, 215 constants.SIGNED_POLICY_FILE) 216 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 217 constants.OWNER_KEY_FILE) 218 # Create deprecated consent file. This is created *after* the 219 # policy file in order to avoid a race condition where chrome 220 # might remove the consent file if the policy's not set yet. 221 # We create it as a temp file first in order to make the creation 222 # of the consent file, owned by chronos, atomic. 223 # See crosbug.com/18413. 224 temp_file = self._CONSENT_FILE + '.tmp'; 225 utils.open_write_close(temp_file, 'test-consent') 226 utils.system('chown chronos:chronos "%s"' % (temp_file)) 227 shutil.move(temp_file, self._CONSENT_FILE) 228 logging.info('Created ' + self._CONSENT_FILE) 229 else: 230 if os.path.isdir(constants.WHITELIST_DIR): 231 # Create policy file that disables metrics/consent. 232 shutil.copy('%s/mock_metrics_off.policy' % autotest_cros_dir, 233 constants.SIGNED_POLICY_FILE) 234 shutil.copy('%s/mock_metrics_owner.key' % autotest_cros_dir, 235 constants.OWNER_KEY_FILE) 236 # Remove deprecated consent file. 237 utils.system('rm -f "%s"' % (self._CONSENT_FILE)) 238 239 240 def _set_crash_test_in_progress(self, in_progress): 241 if in_progress: 242 utils.open_write_close(self._CRASH_TEST_IN_PROGRESS, 'in-progress') 243 logging.info('Created ' + self._CRASH_TEST_IN_PROGRESS) 244 else: 245 utils.system('rm -f "%s"' % (self._CRASH_TEST_IN_PROGRESS)) 246 247 248 def _get_pushed_consent_file_path(self): 249 """Returns filename of the pushed consent file.""" 250 return os.path.join(self.bindir, 'pushed_consent') 251 252 253 def _get_pushed_policy_file_path(self): 254 """Returns filename of the pushed policy file.""" 255 return os.path.join(self.bindir, 'pushed_policy') 256 257 258 def _get_pushed_owner_key_file_path(self): 259 """Returns filename of the pushed owner.key file.""" 260 return os.path.join(self.bindir, 'pushed_owner_key') 261 262 263 def _push_consent(self): 264 """Push the consent file, thus disabling consent. 265 266 The consent files can be created in the new test if required. Call 267 _pop_consent() to restore the original state. 268 """ 269 if os.path.exists(self._CONSENT_FILE): 270 shutil.move(self._CONSENT_FILE, 271 self._get_pushed_consent_file_path()) 272 if os.path.exists(constants.SIGNED_POLICY_FILE): 273 shutil.move(constants.SIGNED_POLICY_FILE, 274 self._get_pushed_policy_file_path()) 275 if os.path.exists(constants.OWNER_KEY_FILE): 276 shutil.move(constants.OWNER_KEY_FILE, 277 self._get_pushed_owner_key_file_path()) 278 279 280 def _pop_consent(self): 281 """Pop the consent files, enabling/disabling consent as it was before 282 we pushed the consent.""" 283 if os.path.exists(self._get_pushed_consent_file_path()): 284 shutil.move(self._get_pushed_consent_file_path(), 285 self._CONSENT_FILE) 286 else: 287 utils.system('rm -f "%s"' % self._CONSENT_FILE) 288 if os.path.exists(self._get_pushed_policy_file_path()): 289 shutil.move(self._get_pushed_policy_file_path(), 290 constants.SIGNED_POLICY_FILE) 291 else: 292 utils.system('rm -f "%s"' % constants.SIGNED_POLICY_FILE) 293 if os.path.exists(self._get_pushed_owner_key_file_path()): 294 shutil.move(self._get_pushed_owner_key_file_path(), 295 constants.OWNER_KEY_FILE) 296 else: 297 utils.system('rm -f "%s"' % constants.OWNER_KEY_FILE) 298 299 300 def _get_crash_dir(self, username): 301 """Returns full path to the crash directory for a given username 302 303 This only really works (currently) when no one is logged in. That 304 is OK (currently) as the only test that uses this runs when no one 305 is actually logged in. 306 307 @param username: username to use: 308 'chronos': Returns user crash directory. 309 'root': Returns system crash directory. 310 """ 311 if username == 'chronos': 312 return self._FALLBACK_USER_CRASH_DIR 313 else: 314 return self._SYSTEM_CRASH_DIR 315 316 317 def _initialize_crash_reporter(self): 318 """Start up the crash reporter.""" 319 utils.system('%s --init' % self._CRASH_REPORTER_PATH) 320 # Completely disable crash_reporter from generating crash dumps 321 # while any tests are running, otherwise a crashy system can make 322 # these tests flaky. 323 self.enable_crash_filtering('none') 324 325 326 def get_crash_dir_name(self, name): 327 """Return the full path for |name| inside the system crash directory.""" 328 return os.path.join(self._SYSTEM_CRASH_DIR, name) 329 330 331 def write_crash_dir_entry(self, name, contents): 332 """Writes an empty file to the system crash directory. 333 334 This writes a file to _SYSTEM_CRASH_DIR with the given name. This is 335 used to insert new crash dump files for testing purposes. 336 337 @param name: Name of file to write. 338 @param contents: String to write to the file. 339 """ 340 entry = self.get_crash_dir_name(name) 341 if not os.path.exists(self._SYSTEM_CRASH_DIR): 342 os.makedirs(self._SYSTEM_CRASH_DIR) 343 utils.open_write_close(entry, contents) 344 return entry 345 346 347 def write_fake_meta(self, name, exec_name, payload, log=None, 348 complete=True): 349 """Writes a fake meta entry to the system crash directory. 350 351 @param name: Name of file to write. 352 @param exec_name: Value for exec_name item. 353 @param payload: Value for payload item. 354 @param log: Value for log item. 355 @param complete: True to close off the record, otherwise leave it 356 incomplete. 357 """ 358 last_line = '' 359 if complete: 360 last_line = 'done=1\n' 361 contents = ('exec_name=%s\n' 362 'ver=my_ver\n' 363 'payload=%s\n' 364 '%s' % (exec_name, payload, 365 last_line)) 366 if log: 367 contents = ('log=%s\n' % log) + contents 368 return self.write_crash_dir_entry(name, contents) 369 370 371 def _prepare_sender_one_crash(self, 372 send_success, 373 reports_enabled, 374 report): 375 """Create metadata for a fake crash report. 376 377 This enabled mocking of the crash sender, then creates a fake 378 crash report for testing purposes. 379 380 @param send_success: True to make the crash_sender success, False to 381 make it fail. 382 @param reports_enabled: True to enable consent to that reports will be 383 sent. 384 @param report: Report to use for crash, if None we create one. 385 """ 386 self._set_sending_mock(mock_enabled=True, send_success=send_success) 387 self._set_consent(reports_enabled) 388 if report is None: 389 # Use the same file format as crash does normally: 390 # <basename>.#.#.#.meta 391 payload = self.write_crash_dir_entry( 392 '%s.dmp' % self._FAKE_TEST_BASENAME, '') 393 report = self.write_fake_meta( 394 '%s.meta' % self._FAKE_TEST_BASENAME, 'fake', payload) 395 return report 396 397 398 def _parse_sender_output(self, output): 399 """Parse the log output from the crash_sender script. 400 401 This script can run on the logs from either a mocked or true 402 crash send. 403 404 @param output: output from the script 405 406 @returns A dictionary with these values: 407 error_type: an error type, if given 408 exec_name: name of executable which crashed 409 image_type: type of image ("dev","force-official",...), if given 410 boot_mode: current boot mode ("dev",...), if given 411 meta_path: path to the report metadata file 412 output: the output from the script, copied 413 report_kind: kind of report sent (minidump vs kernel) 414 send_attempt: did the script attempt to send a crash. 415 send_success: if it attempted, was the crash send successful. 416 sig: signature of the report, if given. 417 sleep_time: if it attempted, how long did it sleep before 418 sending (if mocked, how long would it have slept) 419 """ 420 sleep_match = re.search('Scheduled to send in (\d+)s', output) 421 send_attempt = sleep_match is not None 422 if send_attempt: 423 sleep_time = int(sleep_match.group(1)) 424 else: 425 sleep_time = None 426 427 meta_match = re.search('Metadata: (\S+) \((\S+)\)', output) 428 if meta_match: 429 meta_path = meta_match.group(1) 430 report_kind = meta_match.group(2) 431 else: 432 meta_path = None 433 report_kind = None 434 435 payload_match = re.search('Payload: (\S+)', output) 436 if payload_match: 437 report_payload = payload_match.group(1) 438 else: 439 report_payload = None 440 441 exec_name_match = re.search('Exec name: (\S+)', output) 442 if exec_name_match: 443 exec_name = exec_name_match.group(1) 444 else: 445 exec_name = None 446 447 sig_match = re.search('sig: (\S+)', output) 448 if sig_match: 449 sig = sig_match.group(1) 450 else: 451 sig = None 452 453 error_type_match = re.search('Error type: (\S+)', output) 454 if error_type_match: 455 error_type = error_type_match.group(1) 456 else: 457 error_type = None 458 459 image_type_match = re.search('Image type: (\S+)', output) 460 if image_type_match: 461 image_type = image_type_match.group(1) 462 else: 463 image_type = None 464 465 boot_mode_match = re.search('Boot mode: (\S+)', output) 466 if boot_mode_match: 467 boot_mode = boot_mode_match.group(1) 468 else: 469 boot_mode = None 470 471 send_success = 'Mocking successful send' in output 472 return {'exec_name': exec_name, 473 'report_kind': report_kind, 474 'meta_path': meta_path, 475 'report_payload': report_payload, 476 'send_attempt': send_attempt, 477 'send_success': send_success, 478 'sig': sig, 479 'error_type': error_type, 480 'image_type': image_type, 481 'boot_mode': boot_mode, 482 'sleep_time': sleep_time, 483 'output': output} 484 485 486 def wait_for_sender_completion(self): 487 """Wait for crash_sender to complete. 488 489 Wait for no crash_sender's last message to be placed in the 490 system log before continuing and for the process to finish. 491 Otherwise we might get only part of the output.""" 492 utils.poll_for_condition( 493 lambda: self._log_reader.can_find('crash_sender done.'), 494 timeout=60, 495 exception=error.TestError( 496 'Timeout waiting for crash_sender to emit done: ' + 497 self._log_reader.get_logs())) 498 utils.poll_for_condition( 499 lambda: utils.system('pgrep crash_sender', 500 ignore_status=True) != 0, 501 timeout=60, 502 exception=error.TestError( 503 'Timeout waiting for crash_sender to finish: ' + 504 self._log_reader.get_logs())) 505 506 507 def _call_sender_one_crash(self, 508 send_success=True, 509 reports_enabled=True, 510 username='root', 511 report=None, 512 should_fail=False): 513 """Call the crash sender script to mock upload one crash. 514 515 @param send_success: Mock a successful send if true 516 @param reports_enabled: Has the user consented to sending crash reports. 517 @param username: user to emulate a crash from 518 @param report: report to use for crash, if None we create one. 519 520 @returns a dictionary describing the result with the keys 521 from _parse_sender_output, as well as: 522 report_exists: does the minidump still exist after calling 523 send script 524 rate_count: how many crashes have been uploaded in the past 525 24 hours. 526 """ 527 report = self._prepare_sender_one_crash(send_success, 528 reports_enabled, 529 report) 530 self._log_reader.set_start_by_current() 531 script_output = "" 532 try: 533 script_output = utils.system_output( 534 '/bin/sh -c "%s" 2>&1' % self._CRASH_SENDER_PATH, 535 ignore_status=should_fail) 536 except error.CmdError as err: 537 raise error.TestFail('"%s" returned an unexpected non-zero ' 538 'value (%s).' 539 % (err.command, err.result_obj.exit_status)) 540 541 self.wait_for_sender_completion() 542 output = self._log_reader.get_logs() 543 logging.debug('Crash sender message output:\n' + output) 544 545 if script_output != '': 546 logging.debug('crash_sender stdout/stderr: ' + script_output) 547 548 if os.path.exists(report): 549 report_exists = True 550 os.remove(report) 551 else: 552 report_exists = False 553 if os.path.exists(self._CRASH_SENDER_RATE_DIR): 554 rate_count = len(os.listdir(self._CRASH_SENDER_RATE_DIR)) 555 else: 556 rate_count = 0 557 558 result = self._parse_sender_output(output) 559 result['report_exists'] = report_exists 560 result['rate_count'] = rate_count 561 562 # Show the result for debugging but remove 'output' key 563 # since it's large and earlier in debug output. 564 debug_result = dict(result) 565 del debug_result['output'] 566 logging.debug('Result of send (besides output): %s', debug_result) 567 568 return result 569 570 571 def _replace_crash_reporter_filter_in(self, new_parameter): 572 """Replaces the --filter_in= parameter of the crash reporter. 573 574 The kernel is set up to call the crash reporter with the core dump 575 as stdin when a process dies. This function adds a filter to the 576 command line used to call the crash reporter. This is used to ignore 577 crashes in which we have no interest. 578 579 This removes any --filter_in= parameter and optionally replaces it 580 with a new one. 581 582 @param new_parameter: This is parameter to add to the command line 583 instead of the --filter_in=... that was there. 584 """ 585 core_pattern = utils.read_file(self._CORE_PATTERN)[:-1] 586 core_pattern = re.sub('--filter_in=\S*\s*', '', 587 core_pattern).rstrip() 588 if new_parameter: 589 core_pattern += ' ' + new_parameter 590 utils.system('echo "%s" > %s' % (core_pattern, self._CORE_PATTERN)) 591 592 593 def enable_crash_filtering(self, name): 594 """Add a --filter_in argument to the kernel core dump cmdline. 595 596 @param name: Filter text to use. This is passed as a --filter_in 597 argument to the crash reporter. 598 """ 599 self._replace_crash_reporter_filter_in('--filter_in=' + name) 600 601 602 def disable_crash_filtering(self): 603 """Remove the --filter_in argument from the kernel core dump cmdline. 604 605 Next time the crash reporter is invoked (due to a crash) it will not 606 receive a --filter_in paramter.""" 607 self._replace_crash_reporter_filter_in('') 608 609 610 @contextlib.contextmanager 611 def hold_crash_lock(self): 612 """A context manager to hold the crash sender lock.""" 613 with open(self._CRASH_SENDER_LOCK_PATH, 'w+') as f: 614 fcntl.flock(f.fileno(), fcntl.LOCK_EX) 615 try: 616 yield 617 finally: 618 fcntl.flock(f.fileno(), fcntl.LOCK_UN) 619 620 621 def initialize(self): 622 """Initalize the test.""" 623 test.test.initialize(self) 624 self._log_reader = cros_logging.make_system_log_reader() 625 self._leave_crash_sending = True 626 self._automatic_consent_saving = True 627 self.enable_crash_filtering('none') 628 self._set_crash_test_in_progress(True) 629 630 631 def cleanup(self): 632 """Cleanup after the test. 633 634 We reset things back to the way we think they should be. This is 635 intended to allow the system to continue normal operation. 636 637 Some variables silently change the behavior: 638 _automatic_consent_saving: if True, we pop the consent file. 639 _leave_crash_sending: True to enable crash sending, False to 640 disable it 641 """ 642 self._reset_rate_limiting() 643 self._clear_spooled_crashes() 644 self._set_system_sending(self._leave_crash_sending) 645 self._set_sending_mock(mock_enabled=False) 646 if self._automatic_consent_saving: 647 self._pop_consent() 648 self.disable_crash_filtering() 649 self._set_crash_test_in_progress(False) 650 test.test.cleanup(self) 651 652 653 def run_crash_tests(self, 654 test_names, 655 initialize_crash_reporter=False, 656 clear_spool_first=True, 657 must_run_all=True): 658 """Run crash tests defined in this class. 659 660 @param test_names: Array of test names. 661 @param initialize_crash_reporter: Should set up crash reporter for every 662 run. 663 @param clear_spool_first: Clear all spooled user/system crashes before 664 starting the test. 665 @param must_run_all: Should make sure every test in this class is 666 mentioned in test_names. 667 """ 668 if self._automatic_consent_saving: 669 self._push_consent() 670 671 if must_run_all: 672 # Sanity check test_names is complete 673 for attr in dir(self): 674 if attr.find('_test_') == 0: 675 test_name = attr[6:] 676 if not test_name in test_names: 677 raise error.TestError('Test %s is missing' % test_name) 678 679 for test_name in test_names: 680 logging.info(('=' * 20) + ('Running %s' % test_name) + ('=' * 20)) 681 if initialize_crash_reporter: 682 self._initialize_crash_reporter() 683 # Disable crash_sender from running, kill off any running ones, but 684 # set environment so crash_sender may run as a child process. 685 self._set_system_sending(False) 686 self._set_child_sending(True) 687 self._kill_running_sender() 688 self._reset_rate_limiting() 689 # Default to not overriding for unofficial versions. 690 self._set_force_official(False) 691 # Default to not pretending we're in developer mode. 692 self._set_mock_developer_mode(False) 693 if clear_spool_first: 694 self._clear_spooled_crashes() 695 696 # Call the test function 697 getattr(self, '_test_' + test_name)() 698 699 # Clear the intentional crashes, so that the server won't automatically 700 # report crash as failure. 701 self._clear_spooled_crashes() 702