1#!/usr/bin/env python 2# 3# Copyright 2018, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Atest Argument Parser class for atest. 19""" 20 21# pylint: disable=line-too-long 22 23import argparse 24import pydoc 25 26import constants 27 28# Constants used for AtestArgParser and EPILOG_TEMPLATE 29HELP_DESC = ('A command line tool that allows users to build, install, and run ' 30 'Android tests locally, greatly speeding test re-runs without ' 31 'requiring knowledge of Trade Federation test harness command line' 32 ' options.') 33 34# Constants used for arg help message(sorted in alphabetic) 35ACLOUD_CREATE = 'Create AVD(s) via acloud command.' 36ALL_ABI = 'Set to run tests for all abis.' 37BUILD = 'Run a build.' 38CLEAR_CACHE = 'Wipe out the test_infos cache of the test.' 39COLLECT_TESTS_ONLY = ('Collect a list test cases of the instrumentation tests ' 40 'without testing them in real.') 41DISABLE_TEARDOWN = 'Disable test teardown and cleanup.' 42DRY_RUN = 'Dry run atest without building, installing and running tests in real.' 43ENABLE_FILE_PATTERNS = 'Enable FILE_PATTERNS in TEST_MAPPING.' 44FLAKES_INFO = 'Test result with flakes info.' 45HISTORY = ('Show test results in chronological order(with specified number or ' 46 'all by default).') 47HOST = ('Run the test completely on the host without a device. ' 48 '(Note: running a host test that requires a device without ' 49 '--host will fail.)') 50HOST_UNIT_TEST_ONLY = ('Run all host unit tests under the current directory.') 51INCLUDE_SUBDIRS = 'Search TEST_MAPPING files in subdirs as well.' 52INFO = 'Show module information.' 53INSTALL = 'Install an APK.' 54INSTANT = ('Run the instant_app version of the module if the module supports it. ' 55 'Note: Nothing\'s going to run if it\'s not an Instant App test and ' 56 '"--instant" is passed.') 57ITERATION = 'Loop-run tests until the max iteration is reached. (10 by default)' 58LATEST_RESULT = 'Print latest test result.' 59LIST_MODULES = 'List testable modules for the given suite.' 60NO_METRICS = 'Do not send metrics.' 61NO_MODULES_IN = ('Do not include MODULES-IN-* as build targets. Warning: This ' 62 'may result in missing dependencies issue.') 63REBUILD_MODULE_INFO = ('Forces a rebuild of the module-info.json file. ' 64 'This may be necessary following a repo sync or ' 65 'when writing a new test.') 66REQUEST_UPLOAD_RESULT = 'Request permission to upload test result or not.' 67RERUN_UNTIL_FAILURE = ('Rerun all tests until a failure occurs or the max ' 68 'iteration is reached. (10 by default)') 69RETRY_ANY_FAILURE = ('Rerun failed tests until passed or the max iteration ' 70 'is reached. (10 by default)') 71SERIAL = 'The device to run the test on.' 72SHARDING = 'Option to specify sharding count. The default value is 2' 73START_AVD = 'Automatically create an AVD and run tests on the virtual device.' 74TEST = ('Run the tests. WARNING: Many test configs force cleanup of device ' 75 'after test run. In this case, "-d" must be used in previous test run ' 76 'to disable cleanup for "-t" to work. Otherwise, device will need to ' 77 'be setup again with "-i".') 78TEST_MAPPING = 'Run tests defined in TEST_MAPPING files.' 79TEST_CONFIG_SELECTION = ('If multiple test config belong to same test module ' 80 'pop out a selection menu on console.') 81TF_DEBUG = ('Enable tradefed debug mode with a specify port. Default value is ' 82 '10888.') 83TF_EARLY_DEVICE_RELEASE = ('Tradefed flag to release the device as soon as ' 84 'done with it.') 85TF_TEMPLATE = ('Add extra tradefed template for ATest suite, ' 86 'e.g. atest <test> --tf-template <template_key>=<template_path>') 87UPDATE_CMD_MAPPING = ('Update the test command of input tests. Warning: result ' 88 'will be saved under ' 89 'tools/asuite/atest/test_data.') 90USER_TYPE = ('Run test with specific user type, e.g. atest <test> --user-type ' 91 'secondary_user') 92VERBOSE = 'Display DEBUG level logging.' 93VERIFY_CMD_MAPPING = 'Verify the test command of input tests.' 94VERSION = 'Display version string.' 95WAIT_FOR_DEBUGGER = ('Wait for debugger prior to execution (Instrumentation ' 96 'tests only).') 97 98def _positive_int(value): 99 """Verify value by whether or not a positive integer. 100 101 Args: 102 value: A string of a command-line argument. 103 104 Returns: 105 int of value, if it is an positive integer. 106 Otherwise, raise argparse.ArgumentTypeError. 107 """ 108 err_msg = "invalid positive int value: '%s'" % value 109 try: 110 converted_value = int(value) 111 if converted_value < 1: 112 raise argparse.ArgumentTypeError(err_msg) 113 return converted_value 114 except ValueError as value_err: 115 raise argparse.ArgumentTypeError(err_msg) from value_err 116 117 118class AtestArgParser(argparse.ArgumentParser): 119 """Atest wrapper of ArgumentParser.""" 120 121 def __init__(self): 122 """Initialise an ArgumentParser instance.""" 123 super().__init__(description=HELP_DESC, add_help=False) 124 125 # pylint: disable=too-many-statements 126 def add_atest_args(self): 127 """A function that does ArgumentParser.add_argument()""" 128 self.add_argument('tests', nargs='*', help='Tests to build and/or run.') 129 # Options that to do with testing. 130 self.add_argument('-a', '--all-abi', action='store_true', help=ALL_ABI) 131 self.add_argument('-b', '--build', action='append_const', dest='steps', 132 const=constants.BUILD_STEP, help=BUILD) 133 self.add_argument('-d', '--disable-teardown', action='store_true', 134 help=DISABLE_TEARDOWN) 135 self.add_argument('--host', action='store_true', help=HOST) 136 self.add_argument('-i', '--install', action='append_const', 137 dest='steps', const=constants.INSTALL_STEP, 138 help=INSTALL) 139 self.add_argument('-m', constants.REBUILD_MODULE_INFO_FLAG, 140 action='store_true', help=REBUILD_MODULE_INFO) 141 self.add_argument('--no-modules-in', help=NO_MODULES_IN, 142 action='store_true') 143 self.add_argument('--sharding', nargs='?', const=2, 144 type=_positive_int, default=0, 145 help=SHARDING) 146 self.add_argument('-t', '--test', action='append_const', dest='steps', 147 const=constants.TEST_STEP, help=TEST) 148 self.add_argument('-w', '--wait-for-debugger', action='store_true', 149 help=WAIT_FOR_DEBUGGER) 150 self.add_argument('--request-upload-result', action='store_true', 151 help=REQUEST_UPLOAD_RESULT) 152 153 # Options related to Test Mapping 154 self.add_argument('-p', '--test-mapping', action='store_true', 155 help=TEST_MAPPING) 156 self.add_argument('--include-subdirs', action='store_true', 157 help=INCLUDE_SUBDIRS) 158 # TODO(146980564): Remove enable-file-patterns when support 159 # file-patterns in TEST_MAPPING by default. 160 self.add_argument('--enable-file-patterns', action='store_true', 161 help=ENABLE_FILE_PATTERNS) 162 163 # Options related to Host Unit Test. 164 self.add_argument('--host-unit-test-only', action='store_true', 165 help=HOST_UNIT_TEST_ONLY) 166 167 # Options for information queries and dry-runs: 168 # A group of options for dry-runs. They are mutually exclusive 169 # in a command line. 170 group = self.add_mutually_exclusive_group() 171 group.add_argument('--collect-tests-only', action='store_true', 172 help=COLLECT_TESTS_ONLY) 173 group.add_argument('--dry-run', action='store_true', help=DRY_RUN) 174 self.add_argument('-h', '--help', action='store_true', 175 help='Print this help message.') 176 self.add_argument('--info', action='store_true', help=INFO) 177 self.add_argument('-L', '--list-modules', help=LIST_MODULES) 178 self.add_argument('-v', '--verbose', action='store_true', help=VERBOSE) 179 self.add_argument('-V', '--version', action='store_true', help=VERSION) 180 181 # Options that to do with acloud/AVDs. 182 agroup = self.add_mutually_exclusive_group() 183 agroup.add_argument('--acloud-create', nargs=argparse.REMAINDER, type=str, 184 help=ACLOUD_CREATE) 185 agroup.add_argument('--start-avd', action='store_true', 186 help=START_AVD) 187 agroup.add_argument('-s', '--serial', help=SERIAL) 188 189 # Options that to query flakes info in test result 190 self.add_argument('--flakes-info', action='store_true', 191 help=FLAKES_INFO) 192 193 # Options for tradefed to release test device earlier. 194 self.add_argument('--tf-early-device-release', action='store_true', 195 help=TF_EARLY_DEVICE_RELEASE) 196 197 # Options to enable selection menu is multiple test config belong to 198 # same test module. 199 self.add_argument('--test-config-select', action='store_true', 200 help=TEST_CONFIG_SELECTION) 201 202 # Obsolete options that will be removed soon. 203 self.add_argument('--generate-baseline', nargs='?', 204 type=int, const=5, default=0, 205 help='Generate baseline metrics, run 5 iterations by' 206 'default. Provide an int argument to specify ' 207 '# iterations.') 208 self.add_argument('--generate-new-metrics', nargs='?', 209 type=int, const=5, default=0, 210 help='Generate new metrics, run 5 iterations by ' 211 'default. Provide an int argument to specify ' 212 '# iterations.') 213 self.add_argument('--detect-regression', nargs='*', 214 help='Run regression detection algorithm. Supply ' 215 'path to baseline and/or new metrics folders.') 216 217 # Options related to module parameterization 218 self.add_argument('--instant', action='store_true', help=INSTANT) 219 self.add_argument('--user-type', help=USER_TYPE) 220 221 # Option for dry-run command mapping result and cleaning cache. 222 self.add_argument('-c', '--clear-cache', action='store_true', 223 help=CLEAR_CACHE) 224 self.add_argument('-u', '--update-cmd-mapping', action='store_true', 225 help=UPDATE_CMD_MAPPING) 226 self.add_argument('-y', '--verify-cmd-mapping', action='store_true', 227 help=VERIFY_CMD_MAPPING) 228 # Options for Tradefed debug mode. 229 self.add_argument('-D', '--tf-debug', nargs='?', const=10888, 230 type=_positive_int, default=0, 231 help=TF_DEBUG) 232 # Options for Tradefed customization related. 233 self.add_argument('--tf-template', action='append', 234 help=TF_TEMPLATE) 235 236 # A group of options for rerun strategy. They are mutually exclusive 237 # in a command line. 238 group = self.add_mutually_exclusive_group() 239 # Option for rerun tests for the specified number iterations. 240 group.add_argument('--iterations', nargs='?', 241 type=_positive_int, const=10, default=0, 242 metavar='MAX_ITERATIONS', help=ITERATION) 243 group.add_argument('--rerun-until-failure', nargs='?', 244 type=_positive_int, const=10, default=0, 245 metavar='MAX_ITERATIONS', help=RERUN_UNTIL_FAILURE) 246 group.add_argument('--retry-any-failure', nargs='?', 247 type=_positive_int, const=10, default=0, 248 metavar='MAX_ITERATIONS', help=RETRY_ANY_FAILURE) 249 250 # A group of options for history. They are mutually exclusive 251 # in a command line. 252 history_group = self.add_mutually_exclusive_group() 253 # History related options. 254 history_group.add_argument('--latest-result', action='store_true', 255 help=LATEST_RESULT) 256 history_group.add_argument('--history', nargs='?', const='99999', 257 help=HISTORY) 258 259 # Options for disabling collecting data for metrics. 260 self.add_argument(constants.NO_METRICS_ARG, action='store_true', 261 help=NO_METRICS) 262 263 # This arg actually doesn't consume anything, it's primarily used for 264 # the help description and creating custom_args in the NameSpace object. 265 self.add_argument('--', dest='custom_args', nargs='*', 266 help='Specify custom args for the test runners. ' 267 'Everything after -- will be consumed as ' 268 'custom args.') 269 270 def get_args(self): 271 """This method is to get args from actions and return optional args. 272 273 Returns: 274 A list of optional arguments. 275 """ 276 argument_list = [] 277 # The output of _get_optional_actions(): [['-t', '--test'], [--info]] 278 # return an argument list: ['-t', '--test', '--info'] 279 for arg in self._get_optional_actions(): 280 argument_list.extend(arg.option_strings) 281 return argument_list 282 283 284def print_epilog_text(): 285 """Pagination print EPILOG_TEXT. 286 287 Returns: 288 STDOUT from pydoc.pager(). 289 """ 290 epilog_text = EPILOG_TEMPLATE.format( 291 ACLOUD_CREATE=ACLOUD_CREATE, 292 ALL_ABI=ALL_ABI, 293 BUILD=BUILD, 294 CLEAR_CACHE=CLEAR_CACHE, 295 COLLECT_TESTS_ONLY=COLLECT_TESTS_ONLY, 296 DISABLE_TEARDOWN=DISABLE_TEARDOWN, 297 DRY_RUN=DRY_RUN, 298 ENABLE_FILE_PATTERNS=ENABLE_FILE_PATTERNS, 299 FLAKES_INFO=FLAKES_INFO, 300 HELP_DESC=HELP_DESC, 301 HISTORY=HISTORY, 302 HOST=HOST, 303 HOST_UNIT_TEST_ONLY=HOST_UNIT_TEST_ONLY, 304 INCLUDE_SUBDIRS=INCLUDE_SUBDIRS, 305 INFO=INFO, 306 INSTALL=INSTALL, 307 INSTANT=INSTANT, 308 ITERATION=ITERATION, 309 LATEST_RESULT=LATEST_RESULT, 310 LIST_MODULES=LIST_MODULES, 311 NO_METRICS=NO_METRICS, 312 NO_MODULES_IN=NO_MODULES_IN, 313 REBUILD_MODULE_INFO=REBUILD_MODULE_INFO, 314 REQUEST_UPLOAD_RESULT=REQUEST_UPLOAD_RESULT, 315 RERUN_UNTIL_FAILURE=RERUN_UNTIL_FAILURE, 316 RETRY_ANY_FAILURE=RETRY_ANY_FAILURE, 317 SERIAL=SERIAL, 318 SHARDING=SHARDING, 319 START_AVD=START_AVD, 320 TEST=TEST, 321 TEST_CONFIG_SELECTION=TEST_CONFIG_SELECTION, 322 TEST_MAPPING=TEST_MAPPING, 323 TF_DEBUG=TF_DEBUG, 324 TF_EARLY_DEVICE_RELEASE=TF_EARLY_DEVICE_RELEASE, 325 TF_TEMPLATE=TF_TEMPLATE, 326 USER_TYPE=USER_TYPE, 327 UPDATE_CMD_MAPPING=UPDATE_CMD_MAPPING, 328 VERBOSE=VERBOSE, 329 VERSION=VERSION, 330 VERIFY_CMD_MAPPING=VERIFY_CMD_MAPPING, 331 WAIT_FOR_DEBUGGER=WAIT_FOR_DEBUGGER) 332 return pydoc.pager(epilog_text) 333 334 335EPILOG_TEMPLATE = r'''ATEST(1) ASuite/ATest 336 337NAME 338 atest - {HELP_DESC} 339 340 341SYNOPSIS 342 atest [OPTION]... [TEST_TARGET]... -- [CUSTOM_ARGS]... 343 344 345OPTIONS 346 Below arguments are catagorised by features and purposes. Arguments marked with default will apply even the user does not pass it explicitly. 347 348 [ Testing ] 349 -a, --all-abi 350 {ALL_ABI} 351 352 If only need to run tests for a specific abi, please use: 353 atest <test> -- --abi arm64-v8a # ARM 64-bit 354 atest <test> -- --abi armeabi-v7a # ARM 32-bit 355 356 -b, --build: 357 {BUILD} (default) 358 359 -d, --disable-teardown 360 {DISABLE_TEARDOWN} 361 362 -D, --tf-debug 363 {TF_DEBUG} 364 365 --host 366 {HOST} 367 368 --host-unit-test-only 369 {HOST_UNIT_TEST_ONLY} 370 371 -i, --install 372 {INSTALL} (default) 373 374 -m, --rebuild-module-info 375 {REBUILD_MODULE_INFO} (default) 376 377 --no-modules-in 378 {NO_MODULES_IN} 379 380 -s, --serial 381 {SERIAL} 382 383 --sharding 384 {SHARDING} 385 386 -t, --test 387 {TEST} (default) 388 389 --test-config-select 390 {TEST_CONFIG_SELECTION} 391 392 --tf-early-device-release 393 {TF_EARLY_DEVICE_RELEASE} 394 395 --tf-template 396 {TF_TEMPLATE} 397 398 -w, --wait-for-debugger 399 {WAIT_FOR_DEBUGGER} 400 401 --request-upload-result 402 {REQUEST_UPLOAD_RESULT} 403 404 [ Test Mapping ] 405 -p, --test-mapping 406 {TEST_MAPPING} 407 408 --include-subdirs 409 {INCLUDE_SUBDIRS} 410 411 --enable-file-patterns 412 {ENABLE_FILE_PATTERNS} 413 414 415 [ Information/Queries ] 416 --collect-tests-only 417 {COLLECT_TESTS_ONLY} 418 419 --history 420 {HISTORY} 421 422 --info 423 {INFO} 424 425 -L, --list-modules 426 {LIST_MODULES} 427 428 --latest-result 429 {LATEST_RESULT} 430 431 -v, --verbose 432 {VERBOSE} 433 434 -V, --version 435 {VERSION} 436 437 438 [ Dry-Run and Caching ] 439 --dry-run 440 {DRY_RUN} 441 442 -c, --clear-cache 443 {CLEAR_CACHE} 444 445 -u, --update-cmd-mapping 446 {UPDATE_CMD_MAPPING} 447 448 -y, --verify-cmd-mapping 449 {VERIFY_CMD_MAPPING} 450 451 452 [ Module Parameterization ] 453 --instant 454 {INSTANT} 455 456 --user-type 457 {USER_TYPE} 458 459 460 [ Iteration Testing ] 461 --iterations 462 {ITERATION} 463 464 --rerun-until-failure 465 {RERUN_UNTIL_FAILURE} 466 467 --retry-any-failure 468 {RETRY_ANY_FAILURE} 469 470 471 [ Testing With AVDs ] 472 --start-avd 473 {START_AVD} 474 475 --acloud-create 476 {ACLOUD_CREATE} 477 478 479 [ Testing With Flakes Info ] 480 --flakes-info 481 {FLAKES_INFO} 482 483 484 [ Metrics ] 485 --no-metrics 486 {NO_METRICS} 487 488 489EXAMPLES 490 - - - - - - - - - 491 IDENTIFYING TESTS 492 - - - - - - - - - 493 494 The positional argument <tests> should be a reference to one or more of the tests you'd like to run. Multiple tests can be run in one command by separating test references with spaces. 495 496 Usage template: atest <reference_to_test_1> <reference_to_test_2> 497 498 A <reference_to_test> can be satisfied by the test's MODULE NAME, MODULE:CLASS, CLASS NAME, TF INTEGRATION TEST, FILE PATH or PACKAGE NAME. Explanations and examples of each follow. 499 500 501 < MODULE NAME > 502 503 Identifying a test by its module name will run the entire module. Input the name as it appears in the LOCAL_MODULE or LOCAL_PACKAGE_NAME variables in that test's Android.mk or Android.bp file. 504 505 Note: Use < TF INTEGRATION TEST > to run non-module tests integrated directly into TradeFed. 506 507 Examples: 508 atest FrameworksServicesTests 509 atest CtsJankDeviceTestCases 510 511 512 < MODULE:CLASS > 513 514 Identifying a test by its class name will run just the tests in that class and not the whole module. MODULE:CLASS is the preferred way to run a single class. MODULE is the same as described above. CLASS is the name of the test class in the .java file. It can either be the fully qualified class name or just the basic name. 515 516 Examples: 517 atest FrameworksServicesTests:ScreenDecorWindowTests 518 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 519 atest CtsJankDeviceTestCases:CtsDeviceJankUi 520 521 522 < CLASS NAME > 523 524 A single class can also be run by referencing the class name without the module name. 525 526 Examples: 527 atest ScreenDecorWindowTests 528 atest CtsDeviceJankUi 529 530 However, this will take more time than the equivalent MODULE:CLASS reference, so we suggest using a MODULE:CLASS reference whenever possible. Examples below are ordered by performance from the fastest to the slowest: 531 532 Examples: 533 atest FrameworksServicesTests:com.android.server.wm.ScreenDecorWindowTests 534 atest FrameworksServicesTests:ScreenDecorWindowTests 535 atest ScreenDecorWindowTests 536 537 < TF INTEGRATION TEST > 538 539 To run tests that are integrated directly into TradeFed (non-modules), input the name as it appears in the output of the "tradefed.sh list configs" cmd. 540 541 Examples: 542 atest example/reboot 543 atest native-benchmark 544 545 546 < FILE PATH > 547 548 Both module-based tests and integration-based tests can be run by inputting the path to their test file or dir as appropriate. A single class can also be run by inputting the path to the class's java file. 549 550 Both relative and absolute paths are supported. 551 552 Example - 2 ways to run the `CtsJankDeviceTestCases` module via path: 553 1. run module from android <repo root>: 554 atest cts/tests/jank/jank 555 556 2. from <android root>/cts/tests/jank: 557 atest . 558 559 Example - run a specific class within CtsJankDeviceTestCases module from <android repo> root via path: 560 atest cts/tests/jank/src/android/jank/cts/ui/CtsDeviceJankUi.java 561 562 Example - run an integration test from <android repo> root via path: 563 atest tools/tradefederation/contrib/res/config/example/reboot.xml 564 565 566 < PACKAGE NAME > 567 568 Atest supports searching tests from package name as well. 569 570 Examples: 571 atest com.android.server.wm 572 atest android.jank.cts 573 574 575 - - - - - - - - - - - - - - - - - - - - - - - - - - 576 SPECIFYING INDIVIDUAL STEPS: BUILD, INSTALL OR RUN 577 - - - - - - - - - - - - - - - - - - - - - - - - - - 578 579 The -b, -i and -t options allow you to specify which steps you want to run. If none of those options are given, then all steps are run. If any of these options are provided then only the listed steps are run. 580 581 Note: -i alone is not currently support and can only be included with -t. 582 Both -b and -t can be run alone. 583 584 Examples: 585 atest -b <test> (just build targets) 586 atest -t <test> (run tests only) 587 atest -it <test> (install apk and run tests) 588 atest -bt <test> (build targets, run tests, but skip installing apk) 589 590 591 Atest now has the ability to force a test to skip its cleanup/teardown step. Many tests, e.g. CTS, cleanup the device after the test is run, so trying to rerun your test with -t will fail without having the --disable-teardown parameter. Use -d before -t to skip the test clean up step and test iteratively. 592 593 atest -d <test> (disable installing apk and cleanning up device) 594 atest -t <test> 595 596 Note that -t disables both setup/install and teardown/cleanup of the device. So you can continue to rerun your test with just 597 598 atest -t <test> 599 600 as many times as you want. 601 602 603 - - - - - - - - - - - - - 604 RUNNING SPECIFIC METHODS 605 - - - - - - - - - - - - - 606 607 It is possible to run only specific methods within a test class. To run only specific methods, identify the class in any of the ways supported for identifying a class (MODULE:CLASS, FILE PATH, etc) and then append the name of the method or method using the following template: 608 609 <reference_to_class>#<method1> 610 611 Multiple methods can be specified with commas: 612 613 <reference_to_class>#<method1>,<method2>,<method3>... 614 615 Examples: 616 atest com.android.server.wm.ScreenDecorWindowTests#testMultipleDecors 617 618 atest FrameworksServicesTests:ScreenDecorWindowTests#testFlagChange,testRemoval 619 620 621 - - - - - - - - - - - - - 622 RUNNING MULTIPLE CLASSES 623 - - - - - - - - - - - - - 624 625 To run multiple classes, deliminate them with spaces just like you would when running multiple tests. Atest will handle building and running classes in the most efficient way possible, so specifying a subset of classes in a module will improve performance over running the whole module. 626 627 628 Examples: 629 - two classes in same module: 630 atest FrameworksServicesTests:ScreenDecorWindowTests FrameworksServicesTests:DimmerTests 631 632 - two classes, different modules: 633 atest FrameworksServicesTests:ScreenDecorWindowTests CtsJankDeviceTestCases:CtsDeviceJankUi 634 635 636 - - - - - - - - - - - 637 RUNNING NATIVE TESTS 638 - - - - - - - - - - - 639 640 Atest can run native test. 641 642 Example: 643 - Input tests: 644 atest -a libinput_tests inputflinger_tests 645 646 Use -a|--all-abi to run the tests for all available device architectures, which in this example is armeabi-v7a (ARM 32-bit) and arm64-v8a (ARM 64-bit). 647 648 To select a specific native test to run, use colon (:) to specify the test name and hashtag (#) to further specify an individual method. For example, for the following test definition: 649 650 TEST_F(InputDispatcherTest, InjectInputEvent_ValidatesKeyEvents) 651 652 You can run the entire test using: 653 654 atest inputflinger_tests:InputDispatcherTest 655 656 or an individual test method using: 657 658 atest inputflinger_tests:InputDispatcherTest#InjectInputEvent_ValidatesKeyEvents 659 660 661 - - - - - - - - - - - - - - 662 RUNNING TESTS IN ITERATION 663 - - - - - - - - - - - - - - 664 665 To run tests in iterations, simply pass --iterations argument. No matter pass or fail, atest won't stop testing until the max iteration is reached. 666 667 Example: 668 atest <test> --iterations # 10 iterations(by default). 669 atest <test> --iterations 5 # run <test> 5 times. 670 671 Two approaches that assist users to detect flaky tests: 672 673 1) Run all tests until a failure occurs or the max iteration is reached. 674 675 Example: 676 - 10 iterations(by default). 677 atest <test> --rerun-until-failure 678 - stop when failed or reached the 20th run. 679 atest <test> --rerun-until-failure 20 680 681 2) Run failed tests until passed or the max iteration is reached. 682 683 Example: 684 - 10 iterations(by default). 685 atest <test> --retry-any-failure 686 - stop when passed or reached the 20th run. 687 atest <test> --retry-any-failure 20 688 689 690 - - - - - - - - - - - - 691 RUNNING TESTS ON AVD(s) 692 - - - - - - - - - - - - 693 694 Atest is able to run tests with the newly created AVD. Atest can build and 'acloud create' simultanously, and run tests after the AVD has been created successfully. 695 696 Examples: 697 - Start an AVD before running tests on that newly created device. 698 699 acloud create && atest <test> 700 701 can be simplified by: 702 703 atest <test> --start-avd 704 705 - Start AVD(s) by specifing 'acloud create' arguments and run tests on that newly created device. 706 707 atest <test> --acloud-create "--build-id 6509363 --build-target aosp_cf_x86_phone-userdebug --branch aosp_master" 708 709 To know detail about the argument, please run 'acloud create --help'. 710 711 [WARNING] 712 * --acloud-create must be the LAST optional argument: the remainder args will be consumed as its positional args. 713 * --acloud-create/--start-avd do not delete newly created AVDs. The users will be deleting them manually. 714 715 716 - - - - - - - - - - - - - - - - 717 REGRESSION DETECTION (obsolete) 718 - - - - - - - - - - - - - - - - 719 720 ********************** Warning ********************** 721 Please STOP using arguments below -- they are obsolete and will be removed in a near future: 722 --detect-regression 723 --generate-baseline 724 --generate-new-metrics 725 726 Please check RUNNING TESTS IN ITERATION out for alternatives. 727 ****************************************************** 728 729 Generate pre-patch or post-patch metrics without running regression detection: 730 731 Example: 732 atest <test> --generate-baseline <optional iter> 733 atest <test> --generate-new-metrics <optional iter> 734 735 Local regression detection can be run in three options: 736 737 1) Provide a folder containing baseline (pre-patch) metrics (generated previously). Atest will run the tests n (default 5) iterations, generate a new set of post-patch metrics, and compare those against existing metrics. 738 739 Example: 740 atest <test> --detect-regression </path/to/baseline> --generate-new-metrics <optional iter> 741 742 2) Provide a folder containing post-patch metrics (generated previously). Atest will run the tests n (default 5) iterations, generate a new set of pre-patch metrics, and compare those against those provided. Note: the developer needs to revert the device/tests to pre-patch state to generate baseline metrics. 743 744 Example: 745 atest <test> --detect-regression </path/to/new> --generate-baseline <optional iter> 746 747 3) Provide 2 folders containing both pre-patch and post-patch metrics. Atest will run no tests but the regression detection algorithm. 748 749 Example: 750 atest --detect-regression </path/to/baseline> </path/to/new> 751 752 753 - - - - - - - - - - - - 754 TESTS IN TEST MAPPING 755 - - - - - - - - - - - - 756 757 Atest can run tests in TEST_MAPPING files: 758 759 1) Run presubmit tests in TEST_MAPPING files in current and parent 760 directories. You can also specify a target directory. 761 762 Example: 763 atest (run presubmit tests in TEST_MAPPING files in current and parent directories) 764 atest --test-mapping </path/to/project> 765 (run presubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 766 767 2) Run a specified test group in TEST_MAPPING files. 768 769 Example: 770 atest :postsubmit 771 (run postsubmit tests in TEST_MAPPING files in current and parent directories) 772 atest :all 773 (Run tests from all groups in TEST_MAPPING files) 774 atest --test-mapping </path/to/project>:postsubmit 775 (run postsubmit tests in TEST_MAPPING files in </path/to/project> and its parent directories) 776 777 3) Run tests in TEST_MAPPING files including sub directories 778 779 By default, atest will only search for tests in TEST_MAPPING files in current (or given directory) and its parent directories. If you want to run tests in TEST_MAPPING files in the sub-directories, you can use option --include-subdirs to force atest to include those tests too. 780 781 Example: 782 atest --include-subdirs [optional </path/to/project>:<test_group_name>] 783 (run presubmit tests in TEST_MAPPING files in current, sub and parent directories) 784 A path can be provided optionally if you want to search for tests in a given directory, with optional test group name. By default, the test group is presubmit. 785 786 787 - - - - - - - - - - - - - - 788 ADDITIONAL ARGS TO TRADEFED 789 - - - - - - - - - - - - - - 790 791 When trying to pass custom arguments for the test runners, everything after '--' 792 will be consumed as custom args. 793 794 Example: 795 atest -v <test> -- <custom_args1> <custom_args2> 796 797 Examples of passing options to the modules: 798 atest <test> -- --module-arg <module-name>:<option-name>:<option-value> 799 atest GtsPermissionTestCases -- --module-arg GtsPermissionTestCases:ignore-business-logic-failure:true 800 801 Examples of passing options to the runner type or class: 802 atest <test> -- --test-arg <test-class>:<option-name>:<option-value> 803 atest CtsVideoTestCases -- --test-arg com.android.tradefed.testtype.JarHosttest:collect-tests-only:true 804 805 806 2021-04-22 807''' 808