1#!/usr/bin/env python
2#
3# Copyright 2008, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Command line utility for running Android tests
18
19runtest helps automate the instructions for building and running tests
20- It builds the corresponding test package for the code you want to test
21- It pushes the test package to your device or emulator
22- It launches InstrumentationTestRunner (or similar) to run the tests you
23specify.
24
25runtest supports running tests whose attributes have been pre-defined in
26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
27system path to the test to run (runtest --path <path>).
28
29Do runtest --help to see full list of options.
30"""
31
32# Python imports
33import glob
34import optparse
35import os
36import re
37from sets import Set
38import sys
39import time
40
41# local imports
42import adb_interface
43import android_build
44from coverage import coverage
45import errors
46import logger
47import make_tree
48import run_command
49from test_defs import test_defs
50from test_defs import test_walker
51
52
53class TestRunner(object):
54  """Command line utility class for running pre-defined Android test(s)."""
55
56  _TEST_FILE_NAME = "test_defs.xml"
57
58  # file path to android core platform tests, relative to android build root
59  # TODO move these test data files to another directory
60  _CORE_TEST_PATH = os.path.join("development", "testrunner",
61                                 _TEST_FILE_NAME)
62
63  # vendor glob file path patterns to tests, relative to android
64  # build root
65  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
66                                   _TEST_FILE_NAME)
67
68  _RUNTEST_USAGE = (
69      "usage: runtest.py [options] short-test-name[s]\n\n"
70      "The runtest script works in two ways.  You can query it "
71      "for a list of tests, or you can launch one or more tests.")
72
73  # default value for make -jX
74  _DEFAULT_JOBS = 16
75
76  _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags"
77  _DALVIK_VERIFIER_OFF_VALUE = "v=n"
78  _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE)
79
80  # regular expression to match path to artifacts to install in make output
81  _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$')
82
83  def __init__(self):
84    # disable logging of timestamp
85    self._root_path = android_build.GetTop()
86    out_base_name = os.path.basename(android_build.GetOutDir())
87    # regular expression to find remote device path from a file path relative
88    # to build root
89    pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$'
90    self._re_make_install_path = re.compile(pattern)
91    logger.SetTimestampLogging(False)
92    self._adb = None
93    self._known_tests = None
94    self._options = None
95    self._test_args = None
96    self._tests_to_run = None
97
98  def _ProcessOptions(self):
99    """Processes command-line options."""
100    # TODO error messages on once-only or mutually-exclusive options.
101    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
102                                     self._TEST_FILE_NAME)
103
104    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
105
106    parser.add_option("-l", "--list-tests", dest="only_list_tests",
107                      default=False, action="store_true",
108                      help="To view the list of tests")
109    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
110                      action="store_true", help="Skip build - just launch")
111    parser.add_option("-j", "--jobs", dest="make_jobs",
112                      metavar="X", default=self._DEFAULT_JOBS,
113                      help="Number of make jobs to use when building")
114    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
115                      action="store_true",
116                      help="Do not execute, just preview commands")
117    parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False,
118                      action="store_true",
119                      help="Do not execute, build tests and install to device only")
120    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
121                      action="store_true",
122                      help="Raw mode (for output to other tools)")
123    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
124                      default=False, action="store_true",
125                      help="Suite assignment (for details & usage see "
126                      "InstrumentationTestRunner)")
127    parser.add_option("-v", "--verbose", dest="verbose", default=False,
128                      action="store_true",
129                      help="Increase verbosity of %s" % sys.argv[0])
130    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
131                      default=False, action="store_true",
132                      help="Wait for debugger before launching tests")
133    parser.add_option("-c", "--test-class", dest="test_class",
134                      help="Restrict test to a specific class")
135    parser.add_option("-m", "--test-method", dest="test_method",
136                      help="Restrict test to a specific method")
137    parser.add_option("-p", "--test-package", dest="test_package",
138                      help="Restrict test to a specific java package")
139    parser.add_option("-z", "--size", dest="test_size",
140                      help="Restrict test to a specific test size")
141    parser.add_option("--annotation", dest="test_annotation",
142                      help="Include only those tests tagged with a specific"
143                      " annotation")
144    parser.add_option("--not-annotation", dest="test_not_annotation",
145                      help="Exclude any tests tagged with a specific"
146                      " annotation")
147    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
148                      metavar="FILE", default=user_test_default,
149                      help="Alternate source of user test definitions")
150    parser.add_option("-o", "--coverage", dest="coverage",
151                      default=False, action="store_true",
152                      help="Generate code coverage metrics for test(s)")
153    parser.add_option("--coverage-target", dest="coverage_target_path",
154                      default=None,
155                      help="Path to app to collect code coverage target data for.")
156    parser.add_option("-k", "--skip-permissions", dest="skip_permissions",
157                      default=False, action="store_true",
158                      help="Do not grant runtime permissions during test package"
159                      " installation.")
160    parser.add_option("-x", "--path", dest="test_path",
161                      help="Run test(s) at given file system path")
162    parser.add_option("-t", "--all-tests", dest="all_tests",
163                      default=False, action="store_true",
164                      help="Run all defined tests")
165    parser.add_option("--continuous", dest="continuous_tests",
166                      default=False, action="store_true",
167                      help="Run all tests defined as part of the continuous "
168                      "test set")
169    parser.add_option("--timeout", dest="timeout",
170                      default=300, help="Set a timeout limit (in sec) for "
171                      "running native tests on a device (default: 300 secs)")
172    parser.add_option("--suite", dest="suite",
173                      help="Run all tests defined as part of the "
174                      "the given test suite")
175    parser.add_option("--user", dest="user",
176                      help="The user that test apks are installing to."
177                      " This is the integer user id, e.g. 0 or 10."
178                      " If no user is specified, apk will be installed with"
179                      " adb's default behavior, which is currently all users.")
180    parser.add_option("--install-filter", dest="filter_re",
181                      help="Regular expression which generated apks have to"
182                      " match to be installed to target device. Default is None"
183                      " and will install all packages built.  This is"
184                      " useful when the test path has a lot of apks but you"
185                      " only care about one.")
186    parser.add_option("--no-hidden-api-checks", dest="no_hidden_api_checks",
187                      default=False, action="store_true",
188                      help="Disable hidden API checks in instrumentation"
189                      " tests.")
190    group = optparse.OptionGroup(
191        parser, "Targets", "Use these options to direct tests to a specific "
192        "Android target")
193    group.add_option("-e", "--emulator", dest="emulator", default=False,
194                     action="store_true", help="use emulator")
195    group.add_option("-d", "--device", dest="device", default=False,
196                     action="store_true", help="use device")
197    group.add_option("-s", "--serial", dest="serial",
198                     help="use specific serial")
199    parser.add_option_group(group)
200    self._options, self._test_args = parser.parse_args()
201
202    if (not self._options.only_list_tests
203        and not self._options.all_tests
204        and not self._options.continuous_tests
205        and not self._options.suite
206        and not self._options.test_path
207        and len(self._test_args) < 1):
208      parser.print_help()
209      logger.SilentLog("at least one test name must be specified")
210      raise errors.AbortError
211
212    self._adb = adb_interface.AdbInterface()
213    if self._options.emulator:
214      self._adb.SetEmulatorTarget()
215    elif self._options.device:
216      self._adb.SetDeviceTarget()
217    elif self._options.serial is not None:
218      self._adb.SetTargetSerial(self._options.serial)
219    if self._options.verbose:
220      logger.SetVerbose(True)
221
222    if self._options.coverage_target_path:
223      self._options.coverage = True
224
225    self._known_tests = self._ReadTests()
226
227    self._options.host_lib_path = android_build.GetHostLibraryPath()
228    self._options.test_data_path = android_build.GetTestAppPath()
229
230  def _ReadTests(self):
231    """Parses the set of test definition data.
232
233    Returns:
234      A TestDefinitions object that contains the set of parsed tests.
235    Raises:
236      AbortError: If a fatal error occurred when parsing the tests.
237    """
238    try:
239      known_tests = test_defs.TestDefinitions()
240      # only read tests when not in path mode
241      if not self._options.test_path:
242        core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
243        if os.path.isfile(core_test_path):
244          known_tests.Parse(core_test_path)
245        # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
246        vendor_tests_pattern = os.path.join(self._root_path,
247                                            self._VENDOR_TEST_PATH)
248        test_file_paths = glob.glob(vendor_tests_pattern)
249        for test_file_path in test_file_paths:
250          known_tests.Parse(test_file_path)
251        if os.path.isfile(self._options.user_tests_file):
252          known_tests.Parse(self._options.user_tests_file)
253      return known_tests
254    except errors.ParseError:
255      raise errors.AbortError
256
257  def _DumpTests(self):
258    """Prints out set of defined tests."""
259    print "The following tests are currently defined:\n"
260    print "%-25s %-40s %s" % ("name", "build path", "description")
261    print "-" * 80
262    for test in self._known_tests:
263      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
264                                test.GetDescription())
265    print "\nSee %s for more information" % self._TEST_FILE_NAME
266
267  def _DoBuild(self):
268    logger.SilentLog("Building tests...")
269    tests = self._GetTestsToRun()
270
271    # Build and install tests that do not get granted permissions
272    self._DoPermissionAwareBuild(tests, False)
273
274    # Build and install tests that require granted permissions
275    self._DoPermissionAwareBuild(tests, True)
276
277  def _DoPermissionAwareBuild(self, tests, test_requires_permissions):
278    # turn off dalvik verifier if necessary
279    # TODO: skip turning off verifier for now, since it puts device in bad
280    # state b/14088982
281    #self._TurnOffVerifier(tests)
282    self._DoFullBuild(tests, test_requires_permissions)
283
284    target_tree = make_tree.MakeTree()
285
286    extra_args_set = []
287    for test_suite in tests:
288      if test_suite.IsGrantedPermissions() == test_requires_permissions:
289        self._AddBuildTarget(test_suite, target_tree, extra_args_set)
290
291    if not self._options.preview:
292      self._adb.EnableAdbRoot()
293    else:
294      logger.Log("adb root")
295
296    if not target_tree.IsEmpty():
297      if self._options.coverage:
298        coverage.EnableCoverageBuild()
299        target_tree.AddPath("external/emma")
300
301      target_list = target_tree.GetPrunedMakeList()
302      target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list]
303      target_build_string = " ".join(target_list)
304      target_dir_build_string = " ".join(target_dir_list)
305      extra_args_string = " ".join(extra_args_set)
306
307      install_path_goals = []
308      mmma_goals = []
309      for d in target_dir_list:
310        if d.startswith("./"):
311          d = d[2:]
312        if d.endswith("/"):
313          d = d[:-1]
314        install_path_goals.append("GET-INSTALL-PATH-IN-" + d.replace("/","-"))
315        mmma_goals.append("MODULES-IN-" + d.replace("/","-"))
316      # mmm cannot be used from python, so perform a similar operation using
317      # ONE_SHOT_MAKEFILE
318      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" %s %s %s' % (
319          target_build_string, self._options.make_jobs, self._root_path,
320          " ".join(install_path_goals), " ".join(mmma_goals), extra_args_string)
321      # mmma cannot be used from python, so perform a similar operation
322      alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % (
323              self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals))
324
325      logger.Log(cmd)
326      if not self._options.preview:
327        run_command.SetAbortOnError()
328        try:
329          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
330          ## Chances are this failed because it didn't build the dependencies
331        except errors.AbortError:
332          logger.Log("make failed. Trying to rebuild all dependencies.")
333          logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string))
334          # Try again with mma equivalent, which will build the dependencies
335          run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600)
336          # Run mmm again to get the install paths only
337          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
338        run_command.SetAbortOnError(False)
339        logger.SilentLog(output)
340        filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None
341
342        self._DoInstall(output, test_requires_permissions, filter_re=filter_re)
343
344  def _DoInstall(self, make_output, test_requires_permissions, filter_re=None):
345    """Install artifacts from build onto device.
346
347    Looks for 'install:' text from make output to find artifacts to install.
348
349    Files with the .apk extension get 'adb install'ed, all other files
350    get 'adb push'ed onto the device.
351
352    Args:
353      make_output: stdout from make command
354    """
355    for line in make_output.split("\n"):
356      m = self._RE_MAKE_INSTALL.match(line)
357      if m:
358        # strip the 'INSTALL: <name>' from the left hand side
359        # the remaining string is a space-separated list of build-generated files
360        install_paths = m.group(2)
361        for install_path in re.split(r'\s+', install_paths):
362          if filter_re and not filter_re.match(install_path):
363            continue
364          if install_path.endswith(".apk"):
365            abs_install_path = os.path.join(self._root_path, install_path)
366            extra_flags = ""
367            if test_requires_permissions and not self._options.skip_permissions:
368              extra_flags = "-g"
369            if self._options.user:
370              extra_flags += " --user " + self._options.user
371            logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path))
372            logger.Log(self._adb.Install(abs_install_path, extra_flags))
373          else:
374            self._PushInstallFileToDevice(install_path)
375
376  def _PushInstallFileToDevice(self, install_path):
377    m = self._re_make_install_path.match(install_path)
378    if m:
379      remote_path = m.group(1)
380      remote_dir = os.path.dirname(remote_path)
381      logger.Log("adb shell mkdir -p %s" % remote_dir)
382      self._adb.SendShellCommand("mkdir -p %s" % remote_dir)
383      abs_install_path = os.path.join(self._root_path, install_path)
384      logger.Log("adb push %s %s" % (abs_install_path, remote_path))
385      self._adb.Push(abs_install_path, remote_path)
386    else:
387      logger.Log("Error: Failed to recognize path of file to install %s" % install_path)
388
389  def _DoFullBuild(self, tests, test_requires_permissions):
390    """If necessary, run a full 'make' command for the tests that need it."""
391    extra_args_set = Set()
392
393    for test in tests:
394      if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions:
395        if test.GetExtraBuildArgs():
396          # extra args contains the args to pass to 'make'
397          extra_args_set.add(test.GetExtraBuildArgs())
398        else:
399          logger.Log("Warning: test %s needs a full build but does not specify"
400                     " extra_build_args" % test.GetName())
401
402    # check if there is actually any tests that required a full build
403    if extra_args_set:
404      cmd = ('make -j%s %s' % (self._options.make_jobs,
405                               ' '.join(list(extra_args_set))))
406      logger.Log(cmd)
407      if not self._options.preview:
408        old_dir = os.getcwd()
409        os.chdir(self._root_path)
410        output = run_command.RunCommand(cmd, return_output=True)
411        logger.SilentLog(output)
412        os.chdir(old_dir)
413        self._DoInstall(output, test_requires_permissions)
414
415  def _AddBuildTarget(self, test_suite, target_tree, extra_args_set):
416    if not test_suite.IsFullMake():
417      build_dir = test_suite.GetBuildPath()
418      if self._AddBuildTargetPath(build_dir, target_tree):
419        extra_args_set.append(test_suite.GetExtraBuildArgs())
420      for path in test_suite.GetBuildDependencies(self._options):
421        self._AddBuildTargetPath(path, target_tree)
422
423  def _AddBuildTargetPath(self, build_dir, target_tree):
424    if build_dir is not None:
425      target_tree.AddPath(build_dir)
426      return True
427    return False
428
429  def _GetTestsToRun(self):
430    """Get a list of TestSuite objects to run, based on command line args."""
431    if self._tests_to_run:
432      return self._tests_to_run
433
434    self._tests_to_run = []
435    if self._options.all_tests:
436      self._tests_to_run = self._known_tests.GetTests()
437    elif self._options.continuous_tests:
438      self._tests_to_run = self._known_tests.GetContinuousTests()
439    elif self._options.suite:
440      self._tests_to_run = \
441          self._known_tests.GetTestsInSuite(self._options.suite)
442    elif self._options.test_path:
443      walker = test_walker.TestWalker()
444      self._tests_to_run = walker.FindTests(self._options.test_path)
445
446    for name in self._test_args:
447      test = self._known_tests.GetTest(name)
448      if test is None:
449        logger.Log("Error: Could not find test %s" % name)
450        self._DumpTests()
451        raise errors.AbortError
452      self._tests_to_run.append(test)
453    return self._tests_to_run
454
455  def _TurnOffVerifier(self, test_list):
456    """Turn off the dalvik verifier if needed by given tests.
457
458    If one or more tests needs dalvik verifier off, and it is not already off,
459    turns off verifier and reboots device to allow change to take effect.
460    """
461    # hack to check if these are frameworks/base tests. If so, turn off verifier
462    # to allow framework tests to access private/protected/package-private framework api
463    framework_test = False
464    for test in test_list:
465      if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]):
466        framework_test = True
467    if framework_test:
468      # check if verifier is off already - to avoid the reboot if not
469      # necessary
470      output = self._adb.SendShellCommand("cat /data/local.prop")
471      if not self._DALVIK_VERIFIER_OFF_PROP in output:
472
473        # Read the existing dalvik verifier flags.
474        old_prop_value = self._adb.SendShellCommand("getprop %s" \
475            %(self._DALVIK_VERIFIER_PROP))
476        old_prop_value = old_prop_value.strip() if old_prop_value else ""
477
478        # Append our verifier flags to existing flags
479        new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value)
480
481        # Update property now, as /data/local.prop is not read until reboot
482        logger.Log("adb shell setprop %s '%s'" \
483            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
484        if not self._options.preview:
485          self._adb.SendShellCommand("setprop %s '%s'" \
486            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
487
488        # Write prop to /data/local.prop
489        # Every time device is booted, it will pick up this value
490        new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value)
491        if self._options.preview:
492          logger.Log("adb shell \"echo %s >> /data/local.prop\""
493                     % new_prop_assignment)
494          logger.Log("adb shell chmod 644 /data/local.prop")
495        else:
496          logger.Log("Turning off dalvik verifier and rebooting")
497          self._adb.SendShellCommand("\"echo %s >> /data/local.prop\""
498                                     % new_prop_assignment)
499
500        # Reset runtime so that dalvik picks up new verifier flags from prop
501        self._ChmodRuntimeReset()
502      elif not self._options.preview:
503        # check the permissions on the file
504        permout = self._adb.SendShellCommand("ls -l /data/local.prop")
505        if not "-rw-r--r--" in permout:
506          logger.Log("Fixing permissions on /data/local.prop and rebooting")
507          self._ChmodRuntimeReset()
508
509  def _ChmodRuntimeReset(self):
510    """Perform a chmod of /data/local.prop and reset the runtime.
511    """
512    logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r")
513    if not self._options.preview:
514      self._adb.SendShellCommand("chmod 644 /data/local.prop")
515
516    self._adb.RuntimeReset(preview_only=self._options.preview)
517
518    if not self._options.preview:
519      self._adb.EnableAdbRoot()
520
521
522  def RunTests(self):
523    """Main entry method - executes the tests according to command line args."""
524    try:
525      run_command.SetAbortOnError()
526      self._ProcessOptions()
527      if self._options.only_list_tests:
528        self._DumpTests()
529        return
530
531      if not self._options.skip_build:
532        self._DoBuild()
533
534      if self._options.build_install_only:
535        logger.Log("Skipping test execution (due to --build-install-only flag)")
536        return
537
538      for test_suite in self._GetTestsToRun():
539        try:
540          test_suite.Run(self._options, self._adb)
541        except errors.WaitForResponseTimedOutError:
542          logger.Log("Timed out waiting for response")
543
544    except KeyboardInterrupt:
545      logger.Log("Exiting...")
546    except errors.AbortError, error:
547      logger.Log(error.msg)
548      logger.SilentLog("Exiting due to AbortError...")
549    except errors.WaitForResponseTimedOutError:
550      logger.Log("Timed out waiting for response")
551
552
553def RunTests():
554  runner = TestRunner()
555  runner.RunTests()
556
557if __name__ == "__main__":
558  RunTests()
559