1# Copyright 2016 The Chromium OS Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import json 6import logging 7import os 8 9from autotest_lib.client.common_lib import error 10from autotest_lib.client.common_lib import global_config 11from autotest_lib.client.common_lib.cros import dev_server 12from autotest_lib.server import adb_utils 13from autotest_lib.server import constants 14from autotest_lib.server.cros import dnsname_mangler 15from autotest_lib.server.hosts import adb_host 16 17DEFAULT_ACTS_INTERNAL_DIRECTORY = 'tools/test/connectivity/acts' 18 19CONFIG_FOLDER_LOCATION = global_config.global_config.get_config_value( 20 'ACTS', 'acts_config_folder', default='') 21 22TEST_DIR_NAME = 'tests' 23FRAMEWORK_DIR_NAME = 'framework' 24SETUP_FILE_NAME = 'setup.py' 25CONFIG_DIR_NAME = 'autotest_config' 26CAMPAIGN_DIR_NAME = 'autotest_campaign' 27LOG_DIR_NAME = 'logs' 28ACTS_EXECUTABLE_IN_FRAMEWORK = 'acts/bin/act.py' 29 30ACTS_TESTPATHS_ENV_KEY = 'ACTS_TESTPATHS' 31ACTS_LOGPATH_ENV_KEY = 'ACTS_LOGPATH' 32ACTS_PYTHONPATH_ENV_KEY = 'PYTHONPATH' 33 34 35def create_acts_package_from_current_artifact(test_station, job_repo_url, 36 target_zip_file): 37 """Creates an acts package from the build branch being used. 38 39 Creates an acts artifact from the build branch being used. This is 40 determined by the job_repo_url passed in. 41 42 @param test_station: The teststation that should be creating the package. 43 @param job_repo_url: The job_repo_url to get the build info from. 44 @param target_zip_file: The zip file to create form the artifact on the 45 test_station. 46 47 @returns An ActsPackage containing all the information about the zipped 48 artifact. 49 """ 50 build_info = adb_host.ADBHost.get_build_info_from_build_url(job_repo_url) 51 52 return create_acts_package_from_artifact( 53 test_station, build_info['branch'], build_info['target'], 54 build_info['build_id'], job_repo_url, target_zip_file) 55 56 57def create_acts_package_from_artifact(test_station, branch, target, build_id, 58 job_repo_url, target_zip_file): 59 """Creates an acts package from a specified branch. 60 61 Grabs the packaged acts artifact from the branch and places it on the 62 test_station. 63 64 @param test_station: The teststation that should be creating the package. 65 @param branch: The name of the branch where the artifact is to be pulled. 66 @param target: The name of the target where the artifact is to be pulled. 67 @param build_id: The build id to pull the artifact from. 68 @param job_repo_url: The job repo url for where to pull build from. 69 @param target_zip_file: The zip file to create on the teststation. 70 71 @returns An ActsPackage containing all the information about the zipped 72 artifact. 73 """ 74 devserver_url = dev_server.AndroidBuildServer.get_server_url(job_repo_url) 75 devserver = dev_server.AndroidBuildServer(devserver_url) 76 devserver.trigger_download( 77 target, build_id, branch, files='acts.zip', synchronous=True) 78 79 pull_base_url = devserver.get_pull_url(target, build_id, branch) 80 download_ulr = os.path.join(pull_base_url, 'acts.zip') 81 82 test_station.download_file(download_ulr, target_zip_file) 83 84 return ActsPackage(test_station, target_zip_file) 85 86 87def create_acts_package_from_zip(test_station, zip_location, target_zip_file): 88 """Creates an acts package from an existing zip. 89 90 Creates an acts package from a zip file that already sits on the drone. 91 92 @param test_station: The teststation to create the package on. 93 @param zip_location: The location of the zip on the drone. 94 @param target_zip_file: The zip file to create on the teststaiton. 95 96 @returns An ActsPackage containing all the information about the zipped 97 artifact. 98 """ 99 if not os.path.isabs(zip_location): 100 zip_location = os.path.join(CONFIG_FOLDER_LOCATION, 'acts_artifacts', 101 zip_location) 102 103 test_station.send_file(zip_location, target_zip_file) 104 105 return ActsPackage(test_station, target_zip_file) 106 107 108class ActsPackage(object): 109 """A packaged version of acts on a teststation.""" 110 111 def __init__(self, test_station, zip_file_path): 112 """ 113 @param test_station: The teststation this package is on. 114 @param zip_file_path: The path to the zip file on the test station that 115 holds the package on the teststation. 116 """ 117 self.test_station = test_station 118 self.zip_file = zip_file_path 119 120 def create_container(self, 121 container_directory, 122 internal_acts_directory=None): 123 """Unpacks this package into a container. 124 125 Unpacks this acts package into a container to interact with acts. 126 127 @param container_directory: The directory on the teststation to hold 128 the container. 129 @param internal_acts_directory: The directory inside of the package 130 that holds acts. 131 132 @returns: An ActsContainer with info on the unpacked acts container. 133 """ 134 self.test_station.run('unzip "%s" -x -d "%s"' % 135 (self.zip_file, container_directory)) 136 137 return ActsContainer( 138 self.test_station, 139 container_directory, 140 acts_directory=internal_acts_directory) 141 142 def create_enviroment(self, 143 container_directory, 144 testbed, 145 testbed_name=None, 146 internal_acts_directory=None): 147 """Unpacks this package into an acts testing enviroment. 148 149 Unpacks this acts package into a test enviroment to test with acts. 150 151 @param container_directory: The directory on the teststation to hold 152 the test enviroment. 153 @param testbed: The testbed that the test enviroment 154 will be testing on. 155 @param testbed_name: An overriding name for the testbed. 156 @param internal_acts_directory: The directory inside of the package 157 that holds acts. 158 159 @returns: An ActsTestingEnviroment with info on the unpacked 160 acts testing enviroment. 161 """ 162 if testbed.teststation != self.test_station: 163 raise error.TestError('Creating a contianer for a testbed on a ' 164 'different teststation is not allowed.') 165 166 self.test_station.run('unzip "%s" -x -d "%s"' % 167 (self.zip_file, container_directory)) 168 169 return ActsTestingEnviroment( 170 testbed=testbed, 171 container_directory=container_directory, 172 testbed_name=testbed_name, 173 acts_directory=internal_acts_directory) 174 175 176class AndroidTestingEnviroment(object): 177 """A container for testing android devices on a test station.""" 178 179 def __init__(self, testbed, testbed_name=None): 180 """Creates a new android testing enviroment. 181 182 @param testbed: The testbed to test on. 183 @param testbed_name: An overriding name for the testbed. 184 """ 185 self.testbed = testbed 186 187 if not testbed_name: 188 # If no override is given get the name from the hostname. 189 hostname = testbed.hostname 190 if dnsname_mangler.is_ip_address(hostname): 191 testbed_name = hostname 192 else: 193 testbed_name = hostname.split('.')[0] 194 195 self.testbed_name = testbed_name 196 197 def install_sl4a_apk(self, force_reinstall=True): 198 """Install sl4a to a test bed. 199 200 @param force_reinstall: If true the apk will be force to reinstall. 201 """ 202 for serial, adb_host in self.testbed.get_adb_devices().iteritems(): 203 adb_utils.install_apk_from_build( 204 adb_host, 205 constants.SL4A_APK, 206 constants.SL4A_ARTIFACT, 207 package_name=constants.SL4A_PACKAGE, 208 force_reinstall=force_reinstall) 209 210 def install_apk(self, apk_info, force_reinstall=True): 211 """Installs an additional apk on all adb devices. 212 213 @param apk_info: A dictionary contianing the apk info. This dictionary 214 should contain the keys: 215 apk="Name of the apk", 216 package="Name of the package". 217 artifact="Name of the artifact", if missing 218 the package name is used." 219 @param force_reinstall: If true the apk will be forced to reinstall. 220 """ 221 for serial, adb_host in self.testbed.get_adb_devices().iteritems(): 222 adb_utils.install_apk_from_build( 223 adb_host, 224 apk_info['apk'], 225 apk_info.get('artifact') or constants.SL4A_ARTIFACT, 226 package_name=apk_info['package'], 227 force_reinstall=force_reinstall) 228 229 230class ActsContainer(object): 231 """A container for working with acts.""" 232 233 def __init__(self, test_station, container_directory, acts_directory=None): 234 """ 235 @param test_station: The test station that the container is on. 236 @param container_directory: The directory on the teststation this 237 container operates out of. 238 @param acts_directory: The directory within the container that holds 239 acts. If none then it defaults to 240 DEFAULT_ACTS_INTERNAL_DIRECTORY. 241 """ 242 self.test_station = test_station 243 self.container_directory = container_directory 244 245 if not acts_directory: 246 acts_directory = DEFAULT_ACTS_INTERNAL_DIRECTORY 247 248 if not os.path.isabs(acts_directory): 249 self.acts_directory = os.path.join(container_directory, 250 acts_directory) 251 else: 252 self.acts_directory = acts_directory 253 254 self.tests_directory = os.path.join(self.acts_directory, TEST_DIR_NAME) 255 self.framework_directory = os.path.join(self.acts_directory, 256 FRAMEWORK_DIR_NAME) 257 258 self.acts_file = os.path.join(self.framework_directory, 259 ACTS_EXECUTABLE_IN_FRAMEWORK) 260 261 self.setup_file = os.path.join(self.framework_directory, 262 SETUP_FILE_NAME) 263 264 def get_test_paths(self): 265 """Get all test paths within this container. 266 267 Gets all paths that hold tests within the container. 268 269 @returns: A list of paths on the teststation that hold tests. 270 """ 271 get_test_paths_result = self.test_station.run('find %s -type d' % 272 self.tests_directory) 273 test_search_dirs = get_test_paths_result.stdout.splitlines() 274 return test_search_dirs 275 276 def get_python_path(self): 277 """Get the python path being used. 278 279 Gets the python path that will be set in the enviroment for this 280 container. 281 282 @returns: A string of the PYTHONPATH enviroment variable to be used. 283 """ 284 return '%s:$PYTHONPATH' % self.framework_directory 285 286 def get_enviroment(self): 287 """Gets the enviroment variables to be used for this container. 288 289 @returns: A dictionary of enviroment variables to be used by this 290 container. 291 """ 292 env = { 293 ACTS_TESTPATHS_ENV_KEY: ':'.join(self.get_test_paths()), 294 ACTS_LOGPATH_ENV_KEY: self.log_directory, 295 ACTS_PYTHONPATH_ENV_KEY: self.get_python_path() 296 } 297 298 return env 299 300 def upload_file(self, src, dst): 301 """Uploads a file to be used by the container. 302 303 Uploads a file from the drone to the test staiton to be used by the 304 test container. 305 306 @param src: The source file on the drone. If a relative path is given 307 it is assumed to exist in CONFIG_FOLDER_LOCATION. 308 @param dst: The destination on the teststation. If a relative path is 309 given it is assumed that it is within the container. 310 311 @returns: The full path on the teststation. 312 """ 313 if not os.path.isabs(src): 314 src = os.path.join(CONFIG_FOLDER_LOCATION, src) 315 316 if not os.path.isabs(dst): 317 dst = os.path.join(self.container_directory, dst) 318 319 path = os.path.dirname(dst) 320 result = self.test_station.run('mkdir "%s"' % path, ignore_status=True) 321 322 original_dst = dst 323 if os.path.basename(src) == os.path.basename(dst): 324 dst = os.path.dirname(dst) 325 326 self.test_station.send_file(src, dst) 327 328 return original_dst 329 330 def setup_enviroment(self, python_bin='python'): 331 """Sets up the teststation system enviroment so the container can run. 332 333 Prepares the remote system so that the container can run. This involves 334 uninstalling all versions of acts for the version of python being 335 used and installing all needed dependencies. 336 337 @param python_bin: The python binary to use. 338 """ 339 uninstall_command = '%s %s uninstall' % (python_bin, self.setup_file) 340 install_deps_command = '%s %s install_deps' % (python_bin, 341 self.setup_file) 342 343 self.test_station.run(uninstall_command) 344 self.test_station.run(install_deps_command) 345 346 347class ActsTestingEnviroment(ActsContainer, AndroidTestingEnviroment): 348 """A container for running acts tests with a contained version of acts.""" 349 350 def __init__(self, 351 container_directory, 352 testbed, 353 testbed_name=None, 354 acts_directory=None): 355 """ 356 @param testbed: The testbed to test on. 357 @param container_directory: The directory on the teststation this 358 container operates out of. 359 @param testbed_name: An overriding name for the testbed. 360 @param acts_directory: The directory within the container that holds 361 acts. If none then it defaults to 362 DEFAULT_ACTS_INTERNAL_DIRECTORY. 363 """ 364 AndroidTestingEnviroment.__init__( 365 self, testbed, testbed_name=testbed_name) 366 367 ActsContainer.__init__( 368 self, 369 testbed.teststation, 370 container_directory=container_directory, 371 acts_directory=acts_directory) 372 373 self.config_location = os.path.join(self.container_directory, 374 CONFIG_DIR_NAME) 375 376 self.log_directory = os.path.join(self.container_directory, 377 LOG_DIR_NAME) 378 379 self.acts_file = os.path.join(self.framework_directory, 380 ACTS_EXECUTABLE_IN_FRAMEWORK) 381 382 self.working_directory = os.path.join(container_directory, 383 CONFIG_DIR_NAME) 384 self.test_station.run('mkdir %s' % self.working_directory, 385 ignore_status=True) 386 387 self.configs = {} 388 self.campaigns = {} 389 390 def upload_config(self, config_file): 391 """Uploads a config file to the container. 392 393 Uploads a config file to the config folder in the container. 394 395 @param config_file: The config file to upload. This must be a file 396 within the autotest_config directory under the 397 CONFIG_FOLDER_LOCATION. 398 399 @returns: The full path of the config on the test staiton. 400 """ 401 full_name = os.path.join(CONFIG_DIR_NAME, config_file) 402 403 full_path = self.upload_file(full_name, full_name) 404 self.configs[config_file] = full_path 405 406 return full_path 407 408 def upload_campaign(self, campaign_file): 409 """Uploads a campaign file to the container. 410 411 Uploads a campaign file to the campaign folder in the container. 412 413 @param campaign_file: The campaign file to upload. This must be a file 414 within the autotest_campaign directory under the 415 CONFIG_FOLDER_LOCATION. 416 417 @returns: The full path of the campaign on the test staiton. 418 """ 419 full_name = os.path.join(CAMPAIGN_DIR_NAME, campaign_file) 420 421 full_path = self.upload_file(full_name, full_name) 422 self.campaigns[campaign_file] = full_path 423 424 return full_path 425 426 def run_test(self, 427 config, 428 campaign=None, 429 test_case=None, 430 extra_env={}, 431 python_bin='python', 432 timeout=7200, 433 additional_cmd_line_params=None): 434 """Runs a test within the container. 435 436 Runs a test within a container using the given settings. 437 438 @param config: The name of the config file to use as the main config. 439 This should have already been uploaded with 440 upload_config. The string passed into upload_config 441 should be used here. 442 @param campaign: The campaign file to use for this test. If none then 443 test_case is assumed. This file should have already 444 been uploaded with upload_campaign. The string passed 445 into upload_campaign should be used here. 446 @param test_case: The test case to run the test with. If none then the 447 campaign will be used. If multiple are given, 448 multiple will be run. 449 @param extra_env: Extra enviroment variables to run the test with. 450 @param python_bin: The python binary to execute the test with. 451 @param timeout: How many seconds to wait before timing out. 452 @param additional_cmd_line_params: Adds the ability to add any string 453 to the end of the acts.py command 454 line string. This is intended to 455 add acts command line flags however 456 this is unbounded so it could cause 457 errors if incorrectly set. 458 459 @returns: The results of the test run. 460 """ 461 if not config in self.configs: 462 # Check if the config has been uploaded and upload if it hasn't 463 self.upload_config(config) 464 465 full_config = self.configs[config] 466 467 if campaign: 468 # When given a campaign check if it's upload. 469 if not campaign in self.campaigns: 470 self.upload_campaign(campaign) 471 472 full_campaign = self.campaigns[campaign] 473 else: 474 full_campaign = None 475 476 full_env = self.get_enviroment() 477 478 # Setup enviroment variables. 479 if extra_env: 480 for k, v in extra_env.items(): 481 full_env[k] = extra_env 482 483 logging.info('Using env: %s', full_env) 484 exports = ('export %s=%s' % (k, v) for k, v in full_env.items()) 485 env_command = ';'.join(exports) 486 487 # Make sure to execute in the working directory. 488 command_setup = 'cd %s' % self.working_directory 489 490 if additional_cmd_line_params: 491 act_base_cmd = '%s %s -c %s -tb %s %s ' % ( 492 python_bin, self.acts_file, full_config, self.testbed_name, 493 additional_cmd_line_params) 494 else: 495 act_base_cmd = '%s %s -c %s -tb %s ' % ( 496 python_bin, self.acts_file, full_config, self.testbed_name) 497 498 # Format the acts command based on what type of test is being run. 499 if test_case and campaign: 500 raise error.TestError( 501 'campaign and test_file cannot both have a value.') 502 elif test_case: 503 if isinstance(test_case, str): 504 test_case = [test_case] 505 if len(test_case) < 1: 506 raise error.TestError('At least one test case must be given.') 507 508 tc_str = '' 509 for tc in test_case: 510 tc_str = '%s %s' % (tc_str, tc) 511 tc_str = tc_str.strip() 512 513 act_cmd = '%s -tc %s' % (act_base_cmd, tc_str) 514 elif campaign: 515 act_cmd = '%s -tf %s' % (act_base_cmd, full_campaign) 516 else: 517 raise error.TestFail('No tests was specified!') 518 519 # Format all commands into a single command. 520 command_list = [command_setup, env_command, act_cmd] 521 full_command = '; '.join(command_list) 522 523 try: 524 # Run acts on the remote machine. 525 act_result = self.test_station.run(full_command, timeout=timeout) 526 excep = None 527 except Exception as e: 528 # Catch any error to store in the results. 529 act_result = None 530 excep = e 531 532 return ActsTestResults( 533 str(test_case) or campaign, 534 self.testbed, 535 testbed_name=self.testbed_name, 536 run_result=act_result, 537 log_directory=self.log_directory, 538 exception=excep) 539 540 541class ActsTestResults(object): 542 """The packaged results of a test run.""" 543 acts_result_to_autotest = { 544 'PASS': 'GOOD', 545 'FAIL': 'FAIL', 546 'UNKNOWN': 'WARN', 547 'SKIP': 'ABORT' 548 } 549 550 def __init__(self, 551 name, 552 testbed, 553 testbed_name=None, 554 run_result=None, 555 log_directory=None, 556 exception=None): 557 """ 558 @param name: A name to identify the test run. 559 @param testbed: The testbed that ran the test. 560 @param testbed_name: The name the testbed was run with, if none the 561 default name of the testbed is used. 562 @param run_result: The raw i/o result of the test run. 563 @param log_directory: The directory that acts logged to. 564 @param exception: An exception that was thrown while running the test. 565 """ 566 self.name = name 567 self.run_result = run_result 568 self.exception = exception 569 self.log_directory = log_directory 570 self.test_station = testbed.teststation 571 572 self.testbed = testbed 573 if not testbed_name: 574 # If no override is given get the name from the hostname. 575 hostname = testbed.hostname 576 if dnsname_mangler.is_ip_address(hostname): 577 self.testbed_name = hostname 578 else: 579 self.testbed_name = hostname.split('.')[0] 580 else: 581 self.testbed_name = testbed_name 582 583 self.reported_to = set() 584 585 self.json_results = {} 586 self.results_dir = None 587 if self.log_directory: 588 self.results_dir = os.path.join(self.log_directory, 589 self.testbed_name, 'latest') 590 results_file = os.path.join(self.results_dir, 591 'test_run_summary.json') 592 cat_log_result = self.test_station.run('cat %s' % results_file, 593 ignore_status=True) 594 if not cat_log_result.exit_status: 595 self.json_results = json.loads(cat_log_result.stdout) 596 597 def log_output(self): 598 """Logs the output of the test.""" 599 if self.run_result: 600 logging.debug('ACTS Output:\n%s', self.run_result.stdout) 601 602 def save_test_info(self, test): 603 """Save info about the test. 604 605 @param test: The test to save. 606 """ 607 if self.testbed and self: 608 self.testbed.save_info(test.resultsdir) 609 610 def rethrow_exception(self): 611 """Re-throws the exception thrown during the test.""" 612 if self.exception: 613 raise self.exception 614 615 def upload_to_local(self, local_dir): 616 """Saves all acts results to a local directory. 617 618 @param local_dir: The directory on the local machine to save all results 619 to. 620 """ 621 if self.results_dir: 622 self.test_station.get_file(self.results_dir, local_dir) 623 624 def report_to_autotest(self, test): 625 """Reports the results to an autotest test object. 626 627 Reports the results to the test and saves all acts results under the 628 tests results directory. 629 630 @param test: The autotest test object to report to. If this test object 631 has already recived our report then this call will be 632 ignored. 633 """ 634 if test in self.reported_to: 635 return 636 637 if self.results_dir: 638 self.upload_to_local(test.resultsdir) 639 640 if not 'Results' in self.json_results: 641 return 642 643 results = self.json_results['Results'] 644 for result in results: 645 verdict = self.acts_result_to_autotest[result['Result']] 646 details = result['Details'] 647 test.job.record(verdict, None, self.name, status=(details or '')) 648 649 self.reported_to.add(test) 650