1# Lint as: python2, python3 2# Copyright 2015 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import absolute_import 7from __future__ import division 8from __future__ import print_function 9 10import errno 11import os 12import re 13import shutil 14import signal 15import stat 16import subprocess 17import sys 18import tempfile 19import threading 20 21import logging 22# Turn the logging level to INFO before importing other autotest 23# code, to avoid having failed import logging messages confuse the 24# test_that user. 25logging.basicConfig(level=logging.INFO) 26 27import common 28from autotest_lib.client.common_lib.cros import dev_server, retry 29from autotest_lib.client.common_lib import logging_manager 30from autotest_lib.server.cros.dynamic_suite import suite, constants 31from autotest_lib.server.cros import provision 32from autotest_lib.server.hosts import factory 33from autotest_lib.server.hosts import file_store 34from autotest_lib.server.hosts import host_info 35from autotest_lib.server import autoserv_utils 36from autotest_lib.server import server_logging_config 37from autotest_lib.server import utils 38from autotest_lib.utils import labellib 39from six.moves import range 40 41 42_autoserv_proc = None 43_sigint_handler_lock = threading.Lock() 44 45_AUTOSERV_SIGINT_TIMEOUT_SECONDS = 5 46NO_BOARD = 'ad_hoc_board' 47NO_BUILD = 'ad_hoc_build' 48NO_MODEL = 'ad_hoc_model' 49_SUITE_REGEX = r'suite:(.*)' 50 51_TEST_KEY_FILENAME = 'testing_rsa' 52TEST_KEY_PATH = ('/mnt/host/source/src/scripts/mod_for_test_scripts/' 53 'ssh_keys/%s' % _TEST_KEY_FILENAME) 54 55_LATEST_RESULTS_DIRECTORY = '/tmp/test_that_latest' 56_HOST_INFO_SUBDIR = 'host_info_store' 57 58 59class TestThatRunError(Exception): 60 """Raised if test_that encounters something unexpected while running.""" 61 62 63class TestThatProvisioningError(Exception): 64 """Raised when it fails to provision the DUT to the requested build.""" 65 66 67def add_common_args(parser): 68 """ 69 Add common arguments for both test_that and test_droid to their parser. 70 71 @param parser: argparse.ArgumentParser object to add arguments to. 72 """ 73 parser.add_argument('tests', nargs='+', metavar='TEST', 74 help='Run given test(s). Use suite:SUITE to specify ' 75 'test suite. Use e:[NAME_PATTERN] to specify a ' 76 'NAME-matching regular expression. Use ' 77 'f:[FILE_PATTERN] to specify a filename matching ' 78 'regular expression. Specified regular ' 79 'expressions will be implicitly wrapped in ' 80 '^ and $.') 81 parser.add_argument('--fast', action='store_true', dest='fast_mode', 82 default=False, 83 help='Enable fast mode. This will cause test_droid ' 84 'to skip time consuming steps like sysinfo and ' 85 'collecting crash information.') 86 parser.add_argument('--args', metavar='ARGS', 87 help='Whitespace separated argument string to pass ' 88 'through to test. Only supported for runs ' 89 'against a local DUT. ' 90 "e.g. --args='foo=bar cat=\"in a hat\"'.") 91 parser.add_argument('--results_dir', metavar='RESULTS_DIR', default=None, 92 help='Instead of storing results in a new subdirectory' 93 ' of /tmp , store results in RESULTS_DIR. If ' 94 'RESULTS_DIR already exists, it will be deleted.') 95 parser.add_argument('--pretend', action='store_true', default=False, 96 help='Print autoserv commands that would be run, ' 97 'rather than running them.') 98 parser.add_argument('--no-experimental', action='store_true', 99 default=False, dest='no_experimental', 100 help='When scheduling a suite, skip any tests marked ' 101 'as experimental. Applies only to tests scheduled' 102 ' via suite:[SUITE].') 103 parser.add_argument('--enforce-deps', action='store_true', 104 default=False, dest='enforce_deps', 105 help='Skip tests whose DEPENDENCIES can not ' 106 'be satisfied.') 107 parser.add_argument('--debug', action='store_true', 108 help='Include DEBUG level messages in stdout. Note: ' 109 'these messages will be included in output log ' 110 'file regardless. In addition, turn on autoserv ' 111 'verbosity.') 112 parser.add_argument('--iterations', action='store', type=int, default=1, 113 help='Number of times to run the tests specified.') 114 parser.add_argument('--ssh_verbosity', action='store', type=int, 115 choices=[0, 1, 2, 3], default=0, 116 help='Verbosity level for ssh, between 0 and 3 ' 117 'inclusive.') 118 parser.add_argument('--ssh_options', action='store', default=None, 119 help='A string giving additional options to be ' 120 'added to ssh commands.') 121 122 123class LocalSuite(suite.Suite): 124 """Subclass of Suite with methods for running locally""" 125 126 def handle_local_result(self, job_id, results_dir, record): 127 """ 128 Handle recording and/or retrying a completed job run locally. 129 130 @param job_id: int ID of job 131 @param results_dir: absolute path where test results were stored. 132 @param record: callable that records job status 133 134 @returns: new job_id if a job was scheduled for retry, None otherwise. 135 """ 136 logging.debug('Parsing test results for job %s',job_id) 137 code = generate_report(results_dir, just_status_code=True) 138 if not self._retry_handler: 139 return None 140 logging.debug('Handling result of job %s',job_id) 141 logging.debug(self._retry_handler._retry_map) 142 if code == 0: 143 logging.debug('All tests for job %s succeeded, no retry', job_id) 144 if self._retry_handler.job_present(job_id): 145 self._retry_handler.set_attempted(job_id) 146 return None 147 148 new_job_id = None 149 go_ahead = (self._job_retry and 150 self._retry_handler._should_retry_local_job(job_id)) 151 if go_ahead: 152 new_job_id = self._retry_local_result(job_id, record) 153 return new_job_id 154 155 def _retry_local_result(self, job_id, record): 156 """ 157 Retry a test job by id. 158 159 @param job_id: int ID of job 160 @param record: callable that records job status. 161 prototype: 162 record(base_job.status_log_entry) 163 164 @returns: new job_id if a job was scheduled for retry, None otherwise. 165 """ 166 test = self._jobs_to_tests[job_id] 167 logging.debug('Attempting to retry job %s, test %s', job_id, test.name) 168 test.fast = False 169 new_job = self._schedule_test( 170 record=record, test=test, retry_for=job_id) 171 if new_job: 172 return new_job.id 173 return None 174 175 def test_name_from_job(self, job_id): 176 """Find the name of the test run by a job with a given job ID.""" 177 if self._jobs_to_tests[job_id]: 178 return self._jobs_to_tests[job_id].name 179 180 181 182def fetch_local_suite(autotest_path, suite_predicate, afe, test_arg, remote, 183 build=NO_BUILD, board=NO_BOARD, 184 results_directory=None, no_experimental=False, 185 ignore_deps=True, job_retry=True): 186 """Create a suite from the given suite predicate. 187 188 Satisfaction of dependencies is enforced by Suite.schedule() if 189 ignore_deps is False. Note that this method assumes only one host, 190 i.e. |remote|, was added to afe. Suite.schedule() will not 191 schedule a job if none of the hosts in the afe (in our case, 192 just one host |remote|) has a label that matches a requested 193 test dependency. 194 195 @param autotest_path: Absolute path to autotest (in sysroot or 196 custom autotest directory set by --autotest_dir). 197 @param suite_predicate: callable that takes ControlData objects, and 198 returns True on those that should be in suite 199 @param afe: afe object to schedule against (typically a directAFE) 200 @param test_arg: String. An individual TEST command line argument, e.g. 201 'login_CryptohomeMounted' or 'suite:smoke'. 202 @param remote: String representing the IP of the remote host. 203 @param build: Build to schedule suite for. 204 @param board: Board to schedule suite for. 205 @param results_directory: Absolute path of directory to store results in. 206 (results will be stored in subdirectory of this). 207 @param no_experimental: Skip experimental tests when scheduling a suite. 208 @param ignore_deps: If True, test dependencies will be ignored. 209 @param job_retry: If False, tests will not be retried at all. 210 211 @returns: A LocalSuite object. 212 213 """ 214 fs_getter = suite.create_fs_getter(autotest_path) 215 devserver = dev_server.ImageServer('') 216 my_suite = LocalSuite.create_from_predicates( 217 [suite_predicate], 218 {provision.CROS_VERSION_PREFIX: build}, 219 constants.BOARD_PREFIX + board, 220 devserver, fs_getter, afe=afe, 221 ignore_deps=ignore_deps, 222 results_dir=results_directory, 223 forgiving_parser=False, 224 job_retry=job_retry 225 ) 226 if len(my_suite.tests) == 0: 227 (similarity_predicate, similarity_description) = ( 228 get_predicate_for_possible_test_arg(test_arg)) 229 logging.error('No test found, searching for possible tests with %s', 230 similarity_description) 231 possible_tests = suite.find_possible_tests(fs_getter, 232 similarity_predicate) 233 raise ValueError('Found no tests. Check your suite name, test name, ' 234 'or test matching wildcard.\nDid you mean any of ' 235 'following tests?\n %s' % '\n '.join(possible_tests)) 236 237 if not ignore_deps: 238 # Log tests whose dependencies can't be satisfied. 239 labels = [label.name for label in 240 afe.get_labels(host__hostname=remote)] 241 for test in my_suite.tests: 242 if test.experimental and no_experimental: 243 continue 244 unsatisfiable_deps = set(test.dependencies).difference(labels) 245 if unsatisfiable_deps: 246 logging.warning('%s will be skipped, unsatisfiable ' 247 'test dependencies: %s', test.name, 248 unsatisfiable_deps) 249 return my_suite 250 251 252def _run_autoserv(command, pretend=False): 253 """Run autoserv command. 254 255 Run the autoserv command and wait on it. Log the stdout. 256 Ensure that SIGINT signals are passed along to autoserv. 257 258 @param command: the autoserv command to run. 259 @returns: exit code of the command. 260 261 """ 262 if not pretend: 263 logging.debug('Running autoserv command: %s', command) 264 global _autoserv_proc 265 _autoserv_proc = subprocess.Popen(command, 266 stdout=subprocess.PIPE, 267 stderr=subprocess.STDOUT) 268 # This incantation forces unbuffered reading from stdout, 269 # so that autoserv output can be displayed to the user 270 # immediately. 271 for message in iter(_autoserv_proc.stdout.readline, b''): 272 logging.info('autoserv| %s', message.rstrip()) 273 274 _autoserv_proc.wait() 275 returncode = _autoserv_proc.returncode 276 _autoserv_proc = None 277 else: 278 logging.info('Pretend mode. Would run autoserv command: %s', 279 command) 280 returncode = 0 281 return returncode 282 283 284def run_provisioning_job(provision_label, host, info, autotest_path, 285 results_directory, fast_mode, 286 ssh_verbosity=0, ssh_options=None, 287 pretend=False, autoserv_verbose=False): 288 """Shell out to autoserv to run provisioning job. 289 290 @param provision_label: Label to provision the machine to. 291 @param host: Hostname of DUT. 292 @param info: A host_info.HostInfo for the remote host. 293 @param autotest_path: Absolute path of autotest directory. 294 @param results_directory: Absolute path of directory to store results in. 295 (results will be stored in subdirectory of this). 296 @param fast_mode: bool to use fast mode (disables slow autotest features). 297 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 298 @param ssh_options: Additional ssh options to be passed to autoserv_utils 299 @param pretend: If True, will print out autoserv commands rather than 300 running them. 301 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 302 303 @returns: Absolute path of directory where results were stored. 304 305 """ 306 # TODO(fdeng): When running against a local DUT, autoserv 307 # is still hitting the AFE in the lab. 308 # provision_QuickProvision checks the current build of DUT by 309 # retrieving build info from AFE. crosbug.com/295178 310 results_directory = os.path.join(results_directory, 'results-provision') 311 _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info) 312 command = autoserv_utils.autoserv_run_job_command( 313 os.path.join(autotest_path, 'server'), 314 machines=host, job=None, verbose=autoserv_verbose, 315 results_directory=results_directory, 316 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 317 ssh_options=ssh_options, 318 extra_args=['--provision', '--job-labels', provision_label], 319 no_console_prefix=True, 320 host_info_subdir=_HOST_INFO_SUBDIR) 321 if _run_autoserv(command, pretend) != 0: 322 raise TestThatProvisioningError('Command returns non-zero code: %s ' % 323 command) 324 return results_directory 325 326 327def run_job(job, host, info, autotest_path, results_directory, fast_mode, 328 id_digits=1, ssh_verbosity=0, ssh_options=None, 329 args=None, pretend=False, 330 autoserv_verbose=False): 331 """ 332 Shell out to autoserv to run an individual test job. 333 334 @param job: A Job object containing the control file contents and other 335 relevent metadata for this test. 336 @param host: Hostname of DUT to run test against. 337 @param info: a host_info.HostInfo for the remote host. 338 @param autotest_path: Absolute path of autotest directory. 339 @param results_directory: Absolute path of directory to store results in. 340 (results will be stored in subdirectory of this). 341 @param fast_mode: bool to use fast mode (disables slow autotest features). 342 @param id_digits: The minimum number of digits that job ids should be 343 0-padded to when formatting as a string for results 344 directory. 345 @param ssh_verbosity: SSH verbosity level, passed along to autoserv_utils 346 @param ssh_options: Additional ssh options to be passed to autoserv_utils 347 @param args: String that should be passed as args parameter to autoserv, 348 and then ultimitely to test itself. 349 @param pretend: If True, will print out autoserv commands rather than 350 running them. 351 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 352 353 @returns: a tuple, return code of the job and absolute path of directory 354 where results were stored. 355 """ 356 with tempfile.NamedTemporaryFile() as temp_file: 357 temp_file.write(job.control_file) 358 temp_file.flush() 359 name_tail = job.name.split('/')[-1] 360 results_directory = os.path.join(results_directory, 361 'results-%0*d-%s' % (id_digits, job.id, 362 name_tail)) 363 # Drop experimental keyval in the keval file in the job result folder. 364 os.makedirs(results_directory) 365 utils.write_keyval(results_directory, 366 {constants.JOB_EXPERIMENTAL_KEY: job.keyvals[ 367 constants.JOB_EXPERIMENTAL_KEY]}) 368 _write_host_info(results_directory, _HOST_INFO_SUBDIR, host, info) 369 extra_args = [temp_file.name] 370 if args: 371 extra_args.extend(['--args', args]) 372 373 command = autoserv_utils.autoserv_run_job_command( 374 os.path.join(autotest_path, 'server'), 375 machines=host, job=job, verbose=autoserv_verbose, 376 results_directory=results_directory, 377 fast_mode=fast_mode, ssh_verbosity=ssh_verbosity, 378 ssh_options=ssh_options, 379 extra_args=extra_args, 380 no_console_prefix=True, 381 use_packaging=False, 382 host_attributes=info.attributes, 383 host_info_subdir=_HOST_INFO_SUBDIR) 384 385 code = _run_autoserv(command, pretend) 386 return code, results_directory 387 388 389def setup_local_afe(): 390 """ 391 Setup a local afe database and return a direct_afe object to access it. 392 393 @returns: A autotest_lib.frontend.afe.direct_afe instance. 394 """ 395 # This import statement is delayed until now rather than running at 396 # module load time, because it kicks off a local sqlite :memory: backed 397 # database, and we don't need that unless we are doing a local run. 398 from autotest_lib.frontend import setup_django_lite_environment 399 from autotest_lib.frontend.afe import direct_afe 400 return direct_afe.directAFE() 401 402 403def get_predicate_for_test_arg(test): 404 """ 405 Gets a suite predicte function for a given command-line argument. 406 407 @param test: String. An individual TEST command line argument, e.g. 408 'login_CryptohomeMounted' or 'suite:smoke' 409 @returns: A (predicate, string) tuple with the necessary suite 410 predicate, and a description string of the suite that 411 this predicate will produce. 412 """ 413 suitematch = re.match(_SUITE_REGEX, test) 414 name_pattern_match = re.match(r'e:(.*)', test) 415 file_pattern_match = re.match(r'f:(.*)', test) 416 if suitematch: 417 suitename = suitematch.group(1) 418 return (suite.name_in_tag_predicate(suitename), 419 'suite named %s' % suitename) 420 if name_pattern_match: 421 pattern = '^%s$' % name_pattern_match.group(1) 422 return (suite.test_name_matches_pattern_predicate(pattern), 423 'suite to match name pattern %s' % pattern) 424 if file_pattern_match: 425 pattern = '^%s$' % file_pattern_match.group(1) 426 return (suite.test_file_matches_pattern_predicate(pattern), 427 'suite to match file name pattern %s' % pattern) 428 return (suite.test_name_equals_predicate(test), 429 'job named %s' % test) 430 431 432def get_predicate_for_possible_test_arg(test): 433 """ 434 Gets a suite predicte function to calculate the similarity of given test 435 and possible tests. 436 437 @param test: String. An individual TEST command line argument, e.g. 438 'login_CryptohomeMounted' or 'suite:smoke' 439 @returns: A (predicate, string) tuple with the necessary suite 440 predicate, and a description string of the suite that 441 this predicate will produce. 442 """ 443 suitematch = re.match(_SUITE_REGEX, test) 444 name_pattern_match = re.match(r'e:(.*)', test) 445 file_pattern_match = re.match(r'f:(.*)', test) 446 if suitematch: 447 suitename = suitematch.group(1) 448 return (suite.name_in_tag_similarity_predicate(suitename), 449 'suite name similar to %s' % suitename) 450 if name_pattern_match: 451 pattern = '^%s$' % name_pattern_match.group(1) 452 return (suite.test_name_similarity_predicate(pattern), 453 'job name similar to %s' % pattern) 454 if file_pattern_match: 455 pattern = '^%s$' % file_pattern_match.group(1) 456 return (suite.test_file_similarity_predicate(pattern), 457 'suite to match file name similar to %s' % pattern) 458 return (suite.test_name_similarity_predicate(test), 459 'job name similar to %s' % test) 460 461 462def add_ssh_identity(temp_directory, ssh_private_key=TEST_KEY_PATH): 463 """Add an ssh identity to the agent. 464 465 TODO (sbasi) b/26186193: Add support for test_droid and make TEST_KEY_PATH 466 not Chrome OS specific. 467 468 @param temp_directory: A directory to copy the |private key| into. 469 @param ssh_private_key: Path to the ssh private key to use for testing. 470 """ 471 # Add the testing key to the current ssh agent. 472 if 'SSH_AGENT_PID' in os.environ: 473 # Copy the testing key to the temp directory and make it NOT 474 # world-readable. Otherwise, ssh-add complains. 475 shutil.copy(ssh_private_key, temp_directory) 476 key_copy_path = os.path.join(temp_directory, 477 os.path.basename(ssh_private_key)) 478 os.chmod(key_copy_path, stat.S_IRUSR | stat.S_IWUSR) 479 p = subprocess.Popen(['ssh-add', key_copy_path], 480 stderr=subprocess.STDOUT, stdout=subprocess.PIPE) 481 p_out, _ = p.communicate() 482 for line in p_out.splitlines(): 483 logging.info(line) 484 else: 485 logging.warning('There appears to be no running ssh-agent. Attempting ' 486 'to continue without running ssh-add, but ssh commands ' 487 'may fail.') 488 489 490def _auto_detect_labels(afe, remote): 491 """Automatically detect host labels and add them to the host in afe. 492 493 Note that the label of board will not be auto-detected. 494 This method assumes the host |remote| has already been added to afe. 495 496 @param afe: A direct_afe object used to interact with local afe database. 497 @param remote: The hostname of the remote device. 498 499 @returns: the detected labels as a list of strings. 500 """ 501 cros_host = factory.create_host(remote) 502 labels_to_create = [label for label in cros_host.get_labels() 503 if not label.startswith(constants.BOARD_PREFIX)] 504 labels_to_add_to_afe_host = [] 505 for label in labels_to_create: 506 new_label = afe.create_label(label) 507 labels_to_add_to_afe_host.append(new_label.name) 508 hosts = afe.get_hosts(hostname=remote) 509 if not hosts: 510 raise TestThatRunError('Unexpected error: %s has not ' 511 'been added to afe.' % remote) 512 afe_host = hosts[0] 513 afe_host.add_labels(labels_to_add_to_afe_host) 514 return labels_to_add_to_afe_host 515 516 517def perform_local_run(afe, 518 autotest_path, 519 tests, 520 remote, 521 fast_mode, 522 build=NO_BUILD, 523 board=NO_BOARD, 524 model=NO_MODEL, 525 args=None, 526 pretend=False, 527 no_experimental=False, 528 ignore_deps=True, 529 results_directory=None, 530 ssh_verbosity=0, 531 ssh_options=None, 532 autoserv_verbose=False, 533 iterations=1, 534 host_attributes={}, 535 job_retry=True): 536 """Perform local run of tests. 537 538 This method enforces satisfaction of test dependencies for tests that are 539 run as a part of a suite. 540 541 @param afe: A direct_afe object used to interact with local afe database. 542 @param autotest_path: Absolute path of autotest installed in sysroot or 543 custom autotest path set by --autotest_dir. 544 @param tests: List of strings naming tests and suites to run. Suite strings 545 should be formed like "suite:smoke". 546 @param remote: Remote hostname. 547 @param fast_mode: bool to use fast mode (disables slow autotest features). 548 @param build: String specifying build for local run. 549 @param board: String specifying board for local run. 550 @param model: String specifying model for local run. 551 @param args: String that should be passed as args parameter to autoserv, 552 and then ultimitely to test itself. 553 @param pretend: If True, will print out autoserv commands rather than 554 running them. 555 @param no_experimental: Skip experimental tests when scheduling a suite. 556 @param ignore_deps: If True, test dependencies will be ignored. 557 @param results_directory: Directory to store results in. Defaults to None, 558 in which case results will be stored in a new 559 subdirectory of /tmp 560 @param ssh_verbosity: SSH verbosity level, passed through to 561 autoserv_utils. 562 @param ssh_options: Additional ssh options to be passed to autoserv_utils 563 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 564 @param iterations: int number of times to schedule tests. 565 @param host_attributes: Dict of host attributes to pass into autoserv. 566 @param job_retry: If False, tests will not be retried at all. 567 568 @returns: A list of return codes each job that has run. Or [1] if 569 provision failed prior to running any jobs. 570 """ 571 args = _set_default_servo_args(args) 572 # Create host in afe, add board and build labels. 573 cros_version_label = labellib.format_keyval_label( 574 labellib.KeyvalLabel(labellib.Key.CROS_VERSION, build)) 575 576 build_label = afe.create_label(cros_version_label) 577 board_label = afe.create_label(constants.BOARD_PREFIX + board) 578 model_label = afe.create_label(constants.MODEL_PREFIX + model) 579 labels = [build_label.name, board_label.name, model_label.name] 580 581 new_host = afe.create_host(remote) 582 new_host.add_labels(labels) 583 if not ignore_deps: 584 logging.info('Auto-detecting labels for %s', remote) 585 labels += _auto_detect_labels(afe, remote) 586 # Auto-detected labels may duplicate explicitly set ones. 587 labels = list(set(labels)) 588 589 info = host_info.HostInfo(labels, host_attributes) 590 591 # Provision the host to |build|. 592 if build != NO_BUILD: 593 logging.info('Provisioning %s...', cros_version_label) 594 try: 595 run_provisioning_job( 596 cros_version_label, 597 remote, 598 info, 599 autotest_path, 600 results_directory, 601 fast_mode, 602 ssh_verbosity, 603 ssh_options, 604 pretend, 605 autoserv_verbose, 606 ) 607 except TestThatProvisioningError as e: 608 logging.error('Provisioning %s to %s failed, tests are aborted, ' 609 'failure reason: %s', 610 remote, cros_version_label, e) 611 return [1] 612 613 # Create suites that will be scheduled. 614 suites_and_descriptions = [] 615 for test in tests: 616 (predicate, description) = get_predicate_for_test_arg(test) 617 logging.info('Fetching suite for %s...', description) 618 suite = fetch_local_suite(autotest_path, predicate, afe, test_arg=test, 619 remote=remote, 620 build=build, board=board, 621 results_directory=results_directory, 622 no_experimental=no_experimental, 623 ignore_deps=ignore_deps, 624 job_retry=job_retry) 625 suites_and_descriptions.append((suite, description)) 626 627 jobs_to_suites = {} 628 null_logger = lambda log_entry, log_in_subdir=False: None 629 # Schedule the suites, looping over iterations if necessary. 630 for iteration in range(iterations): 631 if iteration > 0: 632 logging.info('Repeating scheduling for iteration %d:', iteration) 633 634 for suite, description in suites_and_descriptions: 635 logging.info('Scheduling suite for %s...', description) 636 ntests = suite.schedule(null_logger) 637 logging.debug('jobs: %s nonzero job_retries: %s', 638 len(suite._jobs_to_tests), 639 len([True for (job_id, test) in 640 suite._jobs_to_tests.items()])) 641 logging.info('... scheduled %s job(s).', ntests) 642 for job in suite.jobs: 643 jobs_to_suites[job.id] = suite 644 645 if not afe.get_jobs(): 646 logging.info('No jobs scheduled. End of local run.') 647 return [] 648 649 last_job_id = afe.get_jobs()[-1].id 650 job_id_digits = len(str(last_job_id)) 651 codes = [] 652 job_queue = afe.get_jobs() 653 completed_job_ids = set() 654 while job_queue: 655 logging.info('%s jobs in job queue', len(job_queue)) 656 for job in job_queue: 657 suite = jobs_to_suites.get(job.id) 658 if not suite: 659 logging.error('Job %s not run, no associated suite.', job.id) 660 else: 661 logging.debug('Running job %s of test %s', job.id, 662 suite.test_name_from_job(job.id)) 663 code, abs_dir = run_job( 664 job, 665 remote, 666 info, 667 autotest_path, 668 results_directory, 669 fast_mode, 670 job_id_digits, 671 ssh_verbosity, 672 ssh_options, 673 args, 674 pretend, 675 autoserv_verbose, 676 ) 677 codes.append(code) 678 logging.debug("Code: %s, Results in %s", code, abs_dir) 679 new_id = suite.handle_local_result(job.id, abs_dir, 680 null_logger) 681 if new_id: 682 jobs_to_suites[new_id] = jobs_to_suites[job.id] 683 completed_job_ids.add(job.id) 684 all_jobs = afe.get_jobs(not_yet_run=True, running=True) 685 new_jobs = set(job for job in all_jobs 686 if job.id not in completed_job_ids) 687 logging.debug('%s incomplete jobs, %s jobs total', len(new_jobs), 688 len(all_jobs)) 689 job_queue = list(new_jobs) 690 return codes 691 692 693def _set_default_servo_args(args): 694 """Add default servo arguments for backward compatibitlity. 695 696 See crbug.com/881006 for context. Some servo related defaults were baked 697 into the autotest ServoHost code. These have now been deleted. A side effect 698 was that users of test_that relied on these defaults for some tests to work 699 magically in the chroot environment. 700 701 Current plan is to add back these defaults to test_that invocations for 702 backwards compatibility of these use cases. There is no planned removal date 703 for this hack. 704 705 @return modified args str. 706 """ 707 # args is a str with whitespace separated key=value arguments. 708 # Avoid parsing args here (to avoid adding another implicit constraint on 709 # the exact args format) by adding defaults only in the obvious cases where 710 # relevant keys are entirely missing. 711 if args is None: 712 args = '' 713 if 'servo_host' not in args: 714 args += ' servo_host=localhost' 715 if 'servo_port' not in args: 716 args += ' servo_port=9999' 717 return args 718 719 720def sigint_handler(signum, stack_frame): 721 #pylint: disable-msg=C0111 722 """Handle SIGINT or SIGTERM to a local test_that run. 723 724 This handler sends a SIGINT to the running autoserv process, 725 if one is running, giving it up to 5 seconds to clean up and exit. After 726 the timeout elapses, autoserv is killed. In either case, after autoserv 727 exits then this process exits with status 1. 728 """ 729 # If multiple signals arrive before handler is unset, ignore duplicates 730 if not _sigint_handler_lock.acquire(False): 731 return 732 try: 733 # Ignore future signals by unsetting handler. 734 signal.signal(signal.SIGINT, signal.SIG_IGN) 735 signal.signal(signal.SIGTERM, signal.SIG_IGN) 736 737 logging.warning('Received SIGINT or SIGTERM. Cleaning up and exiting.') 738 if _autoserv_proc: 739 logging.warning('Sending SIGINT to autoserv process. Waiting up ' 740 'to %s seconds for cleanup.', 741 _AUTOSERV_SIGINT_TIMEOUT_SECONDS) 742 _autoserv_proc.send_signal(signal.SIGINT) 743 timed_out, _ = retry.timeout(_autoserv_proc.wait, 744 timeout_sec=_AUTOSERV_SIGINT_TIMEOUT_SECONDS) 745 if timed_out: 746 _autoserv_proc.kill() 747 logging.warning('Timed out waiting for autoserv to handle ' 748 'SIGINT. Killed autoserv.') 749 finally: 750 _sigint_handler_lock.release() # this is not really necessary? 751 sys.exit(1) 752 753 754def create_results_directory(results_directory=None, board_name=None): 755 """Create a results directory. 756 757 If no directory is specified this method will create and return a 758 temp directory to hold results. If a directory name is specified this 759 method will create a directory at the given path, provided it doesn't 760 already exist. 761 762 @param results_directory: The path to the results_directory to create. 763 764 @return results_directory: A path to the results_directory, ready for use. 765 """ 766 if results_directory is None: 767 # Create a results_directory as subdir of /tmp 768 dirname_prefix='test_that_results_' 769 if board_name is not None: 770 dirname_prefix += (board_name + '_') 771 results_directory = tempfile.mkdtemp(prefix=dirname_prefix) 772 else: 773 # Delete results_directory if it already exists. 774 try: 775 shutil.rmtree(results_directory) 776 except OSError as e: 777 if e.errno != errno.ENOENT: 778 raise 779 780 # Create results_directory if it does not exist 781 try: 782 os.makedirs(results_directory) 783 except OSError as e: 784 if e.errno != errno.EEXIST: 785 raise 786 return results_directory 787 788def generate_report(directory, 789 allow_chrome_crashes=False, 790 just_status_code=False, 791 html_report=False): 792 """Parse the test result files in the given directory into a report 793 794 @param directory: string, the absolute path of the directory to look in 795 @param allow_chrome_crashes: boolean, ignore Chrome crashes in the 796 report. Default: False, report Chrome crashes. 797 @param just_status_code: boolean, skip the report and only parse the files 798 to determine whether there were failures. Default: False, generate report. 799 """ 800 test_report_command = [os.path.join(os.path.dirname(__file__), 801 'generate_test_report')] 802 # Experimental test results do not influence the exit code. 803 test_report_command.append('--ignore_experimental_tests') 804 if html_report: 805 test_report_command.append('--html') 806 test_report_command.append('--html-report-dir=%s' % directory) 807 if allow_chrome_crashes: 808 test_report_command.append('--allow_chrome_crashes') 809 if just_status_code: 810 test_report_command.append('--just_status_code') 811 test_report_command.append(directory) 812 status_code = subprocess.call(test_report_command) 813 if not just_status_code: 814 with open(os.path.join(directory, 'test_report.log'), 815 'w') as report_log: 816 subprocess.call(test_report_command, stdout=report_log) 817 return status_code 818 819 820def perform_run_from_autotest_root(autotest_path, 821 argv, 822 tests, 823 remote, 824 build=NO_BUILD, 825 board=NO_BOARD, 826 model=NO_MODEL, 827 args=None, 828 pretend=False, 829 no_experimental=False, 830 ignore_deps=True, 831 results_directory=None, 832 ssh_verbosity=0, 833 ssh_options=None, 834 iterations=1, 835 fast_mode=False, 836 debug=False, 837 allow_chrome_crashes=False, 838 host_attributes={}, 839 job_retry=True): 840 """ 841 Perform a test_that run, from the |autotest_path|. 842 843 This function is to be called from test_that/test_droid's main() script, 844 when tests are executed from the |autotest_path|. It handles all stages 845 of a test run that come after the bootstrap into |autotest_path|. 846 847 @param autotest_path: Full absolute path to the autotest root directory. 848 @param argv: The arguments list, as passed to main(...) 849 @param tests: List of strings naming tests and suites to run. Suite strings 850 should be formed like "suite:smoke". 851 @param remote: Remote hostname. 852 @param build: String specifying build for local run. 853 @param board: String specifying board for local run. 854 @param model: String specifying model for local run. 855 @param args: String that should be passed as args parameter to autoserv, 856 and then ultimitely to test itself. 857 @param pretend: If True, will print out autoserv commands rather than 858 running them. 859 @param no_experimental: Skip experimental tests when scheduling a suite. 860 @param ignore_deps: If True, test dependencies will be ignored. 861 @param results_directory: Directory to store results in. Defaults to None, 862 in which case results will be stored in a new 863 subdirectory of /tmp 864 @param ssh_verbosity: SSH verbosity level, passed through to 865 autoserv_utils. 866 @param ssh_options: Additional ssh options to be passed to autoserv_utils 867 @param autoserv_verbose: If true, pass the --verbose flag to autoserv. 868 @param iterations: int number of times to schedule tests. 869 @param fast_mode: bool to use fast mode (disables slow autotest features). 870 @param debug: Logging and autoserv verbosity. 871 @param allow_chrome_crashes: If True, allow chrome crashes. 872 @param host_attributes: Dict of host attributes to pass into autoserv. 873 @param job_retry: If False, tests will not be retried at all. 874 875 @return: A return code that test_that should exit with. 876 """ 877 if results_directory is None or not os.path.exists(results_directory): 878 raise ValueError('Expected valid results directory, got %s' % 879 results_directory) 880 881 logging_manager.configure_logging( 882 server_logging_config.ServerLoggingConfig(), 883 results_dir=results_directory, 884 use_console=True, 885 verbose=debug, 886 debug_log_name='test_that') 887 logging.info('Began logging to %s', results_directory) 888 889 logging.debug('test_that command line was: %s', argv) 890 891 signal.signal(signal.SIGINT, sigint_handler) 892 signal.signal(signal.SIGTERM, sigint_handler) 893 894 afe = setup_local_afe() 895 codes = perform_local_run(afe, 896 autotest_path, 897 tests, 898 remote, 899 fast_mode, 900 build, 901 board, 902 model, 903 args=args, 904 pretend=pretend, 905 no_experimental=no_experimental, 906 ignore_deps=ignore_deps, 907 results_directory=results_directory, 908 ssh_verbosity=ssh_verbosity, 909 ssh_options=ssh_options, 910 autoserv_verbose=debug, 911 iterations=iterations, 912 host_attributes=host_attributes, 913 job_retry=job_retry) 914 if pretend: 915 logging.info('Finished pretend run. Exiting.') 916 return 0 917 918 final_result = generate_report(results_directory, 919 allow_chrome_crashes=allow_chrome_crashes, 920 html_report=True) 921 try: 922 os.unlink(_LATEST_RESULTS_DIRECTORY) 923 except OSError: 924 pass 925 link_target = os.path.relpath(results_directory, 926 os.path.dirname(_LATEST_RESULTS_DIRECTORY)) 927 if any(codes): 928 logging.error('Autoserv encountered unexpected errors ' 929 'when executing jobs.') 930 final_result = final_result or 1 931 os.symlink(link_target, _LATEST_RESULTS_DIRECTORY) 932 logging.info('Finished running tests. Results can be found in %s or %s', 933 results_directory, _LATEST_RESULTS_DIRECTORY) 934 return final_result 935 936 937def _write_host_info(results_dir, host_info_subdir, hostname, info): 938 """ Write HostInfo to a FileStore to be used by autoserv. 939 940 @param results_dir: Path to he results directory. 941 @param host_info_subdir: Subdirectory of results directory for host info. 942 @param hostname: Hostname passed into autoserv. 943 @param info: hosts.HostInfo to write. 944 """ 945 d = os.path.join(results_dir, host_info_subdir) 946 os.makedirs(d) 947 store = file_store.FileStore(os.path.join(d, '%s.store' % hostname)) 948 store.commit(info) 949