1#!/usr/bin/env python
2#
3# Copyright 2008, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Command line utility for running Android tests
18
19runtest helps automate the instructions for building and running tests
20- It builds the corresponding test package for the code you want to test
21- It pushes the test package to your device or emulator
22- It launches InstrumentationTestRunner (or similar) to run the tests you
23specify.
24
25runtest supports running tests whose attributes have been pre-defined in
26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
27system path to the test to run (runtest --path <path>).
28
29Do runtest --help to see full list of options.
30"""
31
32# Python imports
33import glob
34import optparse
35import os
36import re
37from sets import Set
38import sys
39import time
40
41# local imports
42import adb_interface
43import android_build
44from coverage import coverage
45import errors
46import logger
47import make_tree
48import run_command
49from test_defs import test_defs
50from test_defs import test_walker
51
52
53class TestRunner(object):
54  """Command line utility class for running pre-defined Android test(s)."""
55
56  _TEST_FILE_NAME = "test_defs.xml"
57
58  # file path to android core platform tests, relative to android build root
59  # TODO move these test data files to another directory
60  _CORE_TEST_PATH = os.path.join("development", "testrunner",
61                                 _TEST_FILE_NAME)
62
63  # vendor glob file path patterns to tests, relative to android
64  # build root
65  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
66                                   _TEST_FILE_NAME)
67
68  _RUNTEST_USAGE = (
69      "usage: runtest.py [options] short-test-name[s]\n\n"
70      "The runtest script works in two ways.  You can query it "
71      "for a list of tests, or you can launch one or more tests.")
72
73  # default value for make -jX
74  _DEFAULT_JOBS = 16
75
76  _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags"
77  _DALVIK_VERIFIER_OFF_VALUE = "v=n"
78  _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE)
79
80  # regular expression to match path to artifacts to install in make output
81  _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$')
82
83  def __init__(self):
84    # disable logging of timestamp
85    self._root_path = android_build.GetTop()
86    out_base_name = os.path.basename(android_build.GetOutDir())
87    # regular expression to find remote device path from a file path relative
88    # to build root
89    pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$'
90    self._re_make_install_path = re.compile(pattern)
91    logger.SetTimestampLogging(False)
92    self._adb = None
93    self._known_tests = None
94    self._options = None
95    self._test_args = None
96    self._tests_to_run = None
97
98  def _ProcessOptions(self):
99    """Processes command-line options."""
100    # TODO error messages on once-only or mutually-exclusive options.
101    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
102                                     self._TEST_FILE_NAME)
103
104    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
105
106    parser.add_option("-l", "--list-tests", dest="only_list_tests",
107                      default=False, action="store_true",
108                      help="To view the list of tests")
109    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
110                      action="store_true", help="Skip build - just launch")
111    parser.add_option("-j", "--jobs", dest="make_jobs",
112                      metavar="X", default=self._DEFAULT_JOBS,
113                      help="Number of make jobs to use when building")
114    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
115                      action="store_true",
116                      help="Do not execute, just preview commands")
117    parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False,
118                      action="store_true",
119                      help="Do not execute, build tests and install to device only")
120    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
121                      action="store_true",
122                      help="Raw mode (for output to other tools)")
123    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
124                      default=False, action="store_true",
125                      help="Suite assignment (for details & usage see "
126                      "InstrumentationTestRunner)")
127    parser.add_option("-v", "--verbose", dest="verbose", default=False,
128                      action="store_true",
129                      help="Increase verbosity of %s" % sys.argv[0])
130    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
131                      default=False, action="store_true",
132                      help="Wait for debugger before launching tests")
133    parser.add_option("-c", "--test-class", dest="test_class",
134                      help="Restrict test to a specific class")
135    parser.add_option("-m", "--test-method", dest="test_method",
136                      help="Restrict test to a specific method")
137    parser.add_option("-p", "--test-package", dest="test_package",
138                      help="Restrict test to a specific java package")
139    parser.add_option("-z", "--size", dest="test_size",
140                      help="Restrict test to a specific test size")
141    parser.add_option("--annotation", dest="test_annotation",
142                      help="Include only those tests tagged with a specific"
143                      " annotation")
144    parser.add_option("--not-annotation", dest="test_not_annotation",
145                      help="Exclude any tests tagged with a specific"
146                      " annotation")
147    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
148                      metavar="FILE", default=user_test_default,
149                      help="Alternate source of user test definitions")
150    parser.add_option("-o", "--coverage", dest="coverage",
151                      default=False, action="store_true",
152                      help="Generate code coverage metrics for test(s)")
153    parser.add_option("--coverage-target", dest="coverage_target_path",
154                      default=None,
155                      help="Path to app to collect code coverage target data for.")
156    parser.add_option("-k", "--skip-permissions", dest="skip_permissions",
157                      default=False, action="store_true",
158                      help="Do not grant runtime permissions during test package"
159                      " installation.")
160    parser.add_option("-x", "--path", dest="test_path",
161                      help="Run test(s) at given file system path")
162    parser.add_option("-t", "--all-tests", dest="all_tests",
163                      default=False, action="store_true",
164                      help="Run all defined tests")
165    parser.add_option("--continuous", dest="continuous_tests",
166                      default=False, action="store_true",
167                      help="Run all tests defined as part of the continuous "
168                      "test set")
169    parser.add_option("--timeout", dest="timeout",
170                      default=300, help="Set a timeout limit (in sec) for "
171                      "running native tests on a device (default: 300 secs)")
172    parser.add_option("--suite", dest="suite",
173                      help="Run all tests defined as part of the "
174                      "the given test suite")
175    parser.add_option("--user", dest="user",
176                      help="The user that test apks are installing to."
177                      " This is the integer user id, e.g. 0 or 10."
178                      " If no user is specified, apk will be installed with"
179                      " adb's default behavior, which is currently all users.")
180    parser.add_option("--install-filter", dest="filter_re",
181                      help="Regular expression which generated apks have to"
182                      " match to be installed to target device. Default is None"
183                      " and will install all packages built.  This is"
184                      " useful when the test path has a lot of apks but you"
185                      " only care about one.")
186    group = optparse.OptionGroup(
187        parser, "Targets", "Use these options to direct tests to a specific "
188        "Android target")
189    group.add_option("-e", "--emulator", dest="emulator", default=False,
190                     action="store_true", help="use emulator")
191    group.add_option("-d", "--device", dest="device", default=False,
192                     action="store_true", help="use device")
193    group.add_option("-s", "--serial", dest="serial",
194                     help="use specific serial")
195    parser.add_option_group(group)
196    self._options, self._test_args = parser.parse_args()
197
198    if (not self._options.only_list_tests
199        and not self._options.all_tests
200        and not self._options.continuous_tests
201        and not self._options.suite
202        and not self._options.test_path
203        and len(self._test_args) < 1):
204      parser.print_help()
205      logger.SilentLog("at least one test name must be specified")
206      raise errors.AbortError
207
208    self._adb = adb_interface.AdbInterface()
209    if self._options.emulator:
210      self._adb.SetEmulatorTarget()
211    elif self._options.device:
212      self._adb.SetDeviceTarget()
213    elif self._options.serial is not None:
214      self._adb.SetTargetSerial(self._options.serial)
215    if self._options.verbose:
216      logger.SetVerbose(True)
217
218    if self._options.coverage_target_path:
219      self._options.coverage = True
220
221    self._known_tests = self._ReadTests()
222
223    self._options.host_lib_path = android_build.GetHostLibraryPath()
224    self._options.test_data_path = android_build.GetTestAppPath()
225
226  def _ReadTests(self):
227    """Parses the set of test definition data.
228
229    Returns:
230      A TestDefinitions object that contains the set of parsed tests.
231    Raises:
232      AbortError: If a fatal error occurred when parsing the tests.
233    """
234    try:
235      known_tests = test_defs.TestDefinitions()
236      # only read tests when not in path mode
237      if not self._options.test_path:
238        core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
239        if os.path.isfile(core_test_path):
240          known_tests.Parse(core_test_path)
241        # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
242        vendor_tests_pattern = os.path.join(self._root_path,
243                                            self._VENDOR_TEST_PATH)
244        test_file_paths = glob.glob(vendor_tests_pattern)
245        for test_file_path in test_file_paths:
246          known_tests.Parse(test_file_path)
247        if os.path.isfile(self._options.user_tests_file):
248          known_tests.Parse(self._options.user_tests_file)
249      return known_tests
250    except errors.ParseError:
251      raise errors.AbortError
252
253  def _DumpTests(self):
254    """Prints out set of defined tests."""
255    print "The following tests are currently defined:\n"
256    print "%-25s %-40s %s" % ("name", "build path", "description")
257    print "-" * 80
258    for test in self._known_tests:
259      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
260                                test.GetDescription())
261    print "\nSee %s for more information" % self._TEST_FILE_NAME
262
263  def _DoBuild(self):
264    logger.SilentLog("Building tests...")
265    tests = self._GetTestsToRun()
266
267    # Build and install tests that do not get granted permissions
268    self._DoPermissionAwareBuild(tests, False)
269
270    # Build and install tests that require granted permissions
271    self._DoPermissionAwareBuild(tests, True)
272
273  def _DoPermissionAwareBuild(self, tests, test_requires_permissions):
274    # turn off dalvik verifier if necessary
275    # TODO: skip turning off verifier for now, since it puts device in bad
276    # state b/14088982
277    #self._TurnOffVerifier(tests)
278    self._DoFullBuild(tests, test_requires_permissions)
279
280    target_tree = make_tree.MakeTree()
281
282    extra_args_set = []
283    for test_suite in tests:
284      if test_suite.IsGrantedPermissions() == test_requires_permissions:
285        self._AddBuildTarget(test_suite, target_tree, extra_args_set)
286
287    if not self._options.preview:
288      self._adb.EnableAdbRoot()
289    else:
290      logger.Log("adb root")
291
292    if not target_tree.IsEmpty():
293      if self._options.coverage:
294        coverage.EnableCoverageBuild()
295        target_tree.AddPath("external/emma")
296
297      target_list = target_tree.GetPrunedMakeList()
298      target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list]
299      target_build_string = " ".join(target_list)
300      target_dir_build_string = " ".join(target_dir_list)
301      extra_args_string = " ".join(extra_args_set)
302
303      # mmm cannot be used from python, so perform a similar operation using
304      # ONE_SHOT_MAKEFILE
305      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" GET-INSTALL-PATH all_modules %s' % (
306          target_build_string, self._options.make_jobs, self._root_path,
307          extra_args_string)
308      # mmma equivalent, used when regular mmm fails
309      mmma_goals = []
310      for d in target_dir_list:
311        if d.startswith("./"):
312          d = d[2:]
313        if d.endswith("/"):
314          d = d[:-1]
315        mmma_goals.append("MODULES-IN-" + d.replace("/","-"))
316      alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % (
317              self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals))
318
319      logger.Log(cmd)
320      if not self._options.preview:
321        run_command.SetAbortOnError()
322        try:
323          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
324          ## Chances are this failed because it didn't build the dependencies
325        except errors.AbortError:
326          logger.Log("make failed. Trying to rebuild all dependencies.")
327          logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string))
328          # Try again with mma equivalent, which will build the dependencies
329          run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600)
330          # Run mmm again to get the install paths only
331          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
332        run_command.SetAbortOnError(False)
333        logger.SilentLog(output)
334        filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None
335
336        self._DoInstall(output, test_requires_permissions, filter_re=filter_re)
337
338  def _DoInstall(self, make_output, test_requires_permissions, filter_re=None):
339    """Install artifacts from build onto device.
340
341    Looks for 'install:' text from make output to find artifacts to install.
342
343    Files with the .apk extension get 'adb install'ed, all other files
344    get 'adb push'ed onto the device.
345
346    Args:
347      make_output: stdout from make command
348    """
349    for line in make_output.split("\n"):
350      m = self._RE_MAKE_INSTALL.match(line)
351      if m:
352        # strip the 'INSTALL: <name>' from the left hand side
353        # the remaining string is a space-separated list of build-generated files
354        install_paths = m.group(2)
355        for install_path in re.split(r'\s+', install_paths):
356          if filter_re and not filter_re.match(install_path):
357            continue
358          if install_path.endswith(".apk"):
359            abs_install_path = os.path.join(self._root_path, install_path)
360            extra_flags = ""
361            if test_requires_permissions and not self._options.skip_permissions:
362              extra_flags = "-g"
363            if self._options.user:
364              extra_flags += " --user " + self._options.user
365            logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path))
366            logger.Log(self._adb.Install(abs_install_path, extra_flags))
367          else:
368            self._PushInstallFileToDevice(install_path)
369
370  def _PushInstallFileToDevice(self, install_path):
371    m = self._re_make_install_path.match(install_path)
372    if m:
373      remote_path = m.group(1)
374      remote_dir = os.path.dirname(remote_path)
375      logger.Log("adb shell mkdir -p %s" % remote_dir)
376      self._adb.SendShellCommand("mkdir -p %s" % remote_dir)
377      abs_install_path = os.path.join(self._root_path, install_path)
378      logger.Log("adb push %s %s" % (abs_install_path, remote_path))
379      self._adb.Push(abs_install_path, remote_path)
380    else:
381      logger.Log("Error: Failed to recognize path of file to install %s" % install_path)
382
383  def _DoFullBuild(self, tests, test_requires_permissions):
384    """If necessary, run a full 'make' command for the tests that need it."""
385    extra_args_set = Set()
386
387    for test in tests:
388      if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions:
389        if test.GetExtraBuildArgs():
390          # extra args contains the args to pass to 'make'
391          extra_args_set.add(test.GetExtraBuildArgs())
392        else:
393          logger.Log("Warning: test %s needs a full build but does not specify"
394                     " extra_build_args" % test.GetName())
395
396    # check if there is actually any tests that required a full build
397    if extra_args_set:
398      cmd = ('make -j%s %s' % (self._options.make_jobs,
399                               ' '.join(list(extra_args_set))))
400      logger.Log(cmd)
401      if not self._options.preview:
402        old_dir = os.getcwd()
403        os.chdir(self._root_path)
404        output = run_command.RunCommand(cmd, return_output=True)
405        logger.SilentLog(output)
406        os.chdir(old_dir)
407        self._DoInstall(output, test_requires_permissions)
408
409  def _AddBuildTarget(self, test_suite, target_tree, extra_args_set):
410    if not test_suite.IsFullMake():
411      build_dir = test_suite.GetBuildPath()
412      if self._AddBuildTargetPath(build_dir, target_tree):
413        extra_args_set.append(test_suite.GetExtraBuildArgs())
414      for path in test_suite.GetBuildDependencies(self._options):
415        self._AddBuildTargetPath(path, target_tree)
416
417  def _AddBuildTargetPath(self, build_dir, target_tree):
418    if build_dir is not None:
419      target_tree.AddPath(build_dir)
420      return True
421    return False
422
423  def _GetTestsToRun(self):
424    """Get a list of TestSuite objects to run, based on command line args."""
425    if self._tests_to_run:
426      return self._tests_to_run
427
428    self._tests_to_run = []
429    if self._options.all_tests:
430      self._tests_to_run = self._known_tests.GetTests()
431    elif self._options.continuous_tests:
432      self._tests_to_run = self._known_tests.GetContinuousTests()
433    elif self._options.suite:
434      self._tests_to_run = \
435          self._known_tests.GetTestsInSuite(self._options.suite)
436    elif self._options.test_path:
437      walker = test_walker.TestWalker()
438      self._tests_to_run = walker.FindTests(self._options.test_path)
439
440    for name in self._test_args:
441      test = self._known_tests.GetTest(name)
442      if test is None:
443        logger.Log("Error: Could not find test %s" % name)
444        self._DumpTests()
445        raise errors.AbortError
446      self._tests_to_run.append(test)
447    return self._tests_to_run
448
449  def _TurnOffVerifier(self, test_list):
450    """Turn off the dalvik verifier if needed by given tests.
451
452    If one or more tests needs dalvik verifier off, and it is not already off,
453    turns off verifier and reboots device to allow change to take effect.
454    """
455    # hack to check if these are frameworks/base tests. If so, turn off verifier
456    # to allow framework tests to access private/protected/package-private framework api
457    framework_test = False
458    for test in test_list:
459      if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]):
460        framework_test = True
461    if framework_test:
462      # check if verifier is off already - to avoid the reboot if not
463      # necessary
464      output = self._adb.SendShellCommand("cat /data/local.prop")
465      if not self._DALVIK_VERIFIER_OFF_PROP in output:
466
467        # Read the existing dalvik verifier flags.
468        old_prop_value = self._adb.SendShellCommand("getprop %s" \
469            %(self._DALVIK_VERIFIER_PROP))
470        old_prop_value = old_prop_value.strip() if old_prop_value else ""
471
472        # Append our verifier flags to existing flags
473        new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value)
474
475        # Update property now, as /data/local.prop is not read until reboot
476        logger.Log("adb shell setprop %s '%s'" \
477            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
478        if not self._options.preview:
479          self._adb.SendShellCommand("setprop %s '%s'" \
480            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
481
482        # Write prop to /data/local.prop
483        # Every time device is booted, it will pick up this value
484        new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value)
485        if self._options.preview:
486          logger.Log("adb shell \"echo %s >> /data/local.prop\""
487                     % new_prop_assignment)
488          logger.Log("adb shell chmod 644 /data/local.prop")
489        else:
490          logger.Log("Turning off dalvik verifier and rebooting")
491          self._adb.SendShellCommand("\"echo %s >> /data/local.prop\""
492                                     % new_prop_assignment)
493
494        # Reset runtime so that dalvik picks up new verifier flags from prop
495        self._ChmodRuntimeReset()
496      elif not self._options.preview:
497        # check the permissions on the file
498        permout = self._adb.SendShellCommand("ls -l /data/local.prop")
499        if not "-rw-r--r--" in permout:
500          logger.Log("Fixing permissions on /data/local.prop and rebooting")
501          self._ChmodRuntimeReset()
502
503  def _ChmodRuntimeReset(self):
504    """Perform a chmod of /data/local.prop and reset the runtime.
505    """
506    logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r")
507    if not self._options.preview:
508      self._adb.SendShellCommand("chmod 644 /data/local.prop")
509
510    self._adb.RuntimeReset(preview_only=self._options.preview)
511
512    if not self._options.preview:
513      self._adb.EnableAdbRoot()
514
515
516  def RunTests(self):
517    """Main entry method - executes the tests according to command line args."""
518    try:
519      run_command.SetAbortOnError()
520      self._ProcessOptions()
521      if self._options.only_list_tests:
522        self._DumpTests()
523        return
524
525      if not self._options.skip_build:
526        self._DoBuild()
527
528      if self._options.build_install_only:
529        logger.Log("Skipping test execution (due to --build-install-only flag)")
530        return
531
532      for test_suite in self._GetTestsToRun():
533        try:
534          test_suite.Run(self._options, self._adb)
535        except errors.WaitForResponseTimedOutError:
536          logger.Log("Timed out waiting for response")
537
538    except KeyboardInterrupt:
539      logger.Log("Exiting...")
540    except errors.AbortError, error:
541      logger.Log(error.msg)
542      logger.SilentLog("Exiting due to AbortError...")
543    except errors.WaitForResponseTimedOutError:
544      logger.Log("Timed out waiting for response")
545
546
547def RunTests():
548  runner = TestRunner()
549  runner.RunTests()
550
551if __name__ == "__main__":
552  RunTests()
553