1#!/usr/bin/env python
2#
3# Copyright 2008, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Command line utility for running Android tests
18
19runtest helps automate the instructions for building and running tests
20- It builds the corresponding test package for the code you want to test
21- It pushes the test package to your device or emulator
22- It launches InstrumentationTestRunner (or similar) to run the tests you
23specify.
24
25runtest supports running tests whose attributes have been pre-defined in
26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
27system path to the test to run (runtest --path <path>).
28
29Do runtest --help to see full list of options.
30"""
31
32# Python imports
33import glob
34import optparse
35import os
36import re
37from sets import Set
38import sys
39import time
40
41# local imports
42import adb_interface
43import android_build
44from coverage import coverage
45import errors
46import logger
47import make_tree
48import run_command
49from test_defs import test_defs
50from test_defs import test_walker
51
52
53class TestRunner(object):
54  """Command line utility class for running pre-defined Android test(s)."""
55
56  _TEST_FILE_NAME = "test_defs.xml"
57
58  # file path to android core platform tests, relative to android build root
59  # TODO move these test data files to another directory
60  _CORE_TEST_PATH = os.path.join("development", "testrunner",
61                                 _TEST_FILE_NAME)
62
63  # vendor glob file path patterns to tests, relative to android
64  # build root
65  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
66                                   _TEST_FILE_NAME)
67
68  _RUNTEST_USAGE = (
69      "usage: runtest.py [options] short-test-name[s]\n\n"
70      "The runtest script works in two ways.  You can query it "
71      "for a list of tests, or you can launch one or more tests.")
72
73  # default value for make -jX
74  _DEFAULT_JOBS = 16
75
76  _DALVIK_VERIFIER_PROP = "dalvik.vm.dexopt-flags"
77  _DALVIK_VERIFIER_OFF_VALUE = "v=n"
78  _DALVIK_VERIFIER_OFF_PROP = "%s = %s" %(_DALVIK_VERIFIER_PROP, _DALVIK_VERIFIER_OFF_VALUE)
79
80  # regular expression to match path to artifacts to install in make output
81  _RE_MAKE_INSTALL = re.compile(r'INSTALL-PATH:\s([^\s]+)\s(.*)$')
82
83  def __init__(self):
84    # disable logging of timestamp
85    self._root_path = android_build.GetTop()
86    out_base_name = os.path.basename(android_build.GetOutDir())
87    # regular expression to find remote device path from a file path relative
88    # to build root
89    pattern = r'' + out_base_name + r'\/target\/product\/\w+\/(.+)$'
90    self._re_make_install_path = re.compile(pattern)
91    logger.SetTimestampLogging(False)
92    self._adb = None
93    self._known_tests = None
94    self._options = None
95    self._test_args = None
96    self._tests_to_run = None
97
98  def _ProcessOptions(self):
99    """Processes command-line options."""
100    # TODO error messages on once-only or mutually-exclusive options.
101    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
102                                     self._TEST_FILE_NAME)
103
104    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
105
106    parser.add_option("-l", "--list-tests", dest="only_list_tests",
107                      default=False, action="store_true",
108                      help="To view the list of tests")
109    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
110                      action="store_true", help="Skip build - just launch")
111    parser.add_option("-j", "--jobs", dest="make_jobs",
112                      metavar="X", default=self._DEFAULT_JOBS,
113                      help="Number of make jobs to use when building")
114    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
115                      action="store_true",
116                      help="Do not execute, just preview commands")
117    parser.add_option("-i", "--build-install-only", dest="build_install_only", default=False,
118                      action="store_true",
119                      help="Do not execute, build tests and install to device only")
120    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
121                      action="store_true",
122                      help="Raw mode (for output to other tools)")
123    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
124                      default=False, action="store_true",
125                      help="Suite assignment (for details & usage see "
126                      "InstrumentationTestRunner)")
127    parser.add_option("-v", "--verbose", dest="verbose", default=False,
128                      action="store_true",
129                      help="Increase verbosity of %s" % sys.argv[0])
130    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
131                      default=False, action="store_true",
132                      help="Wait for debugger before launching tests")
133    parser.add_option("-c", "--test-class", dest="test_class",
134                      help="Restrict test to a specific class")
135    parser.add_option("-m", "--test-method", dest="test_method",
136                      help="Restrict test to a specific method")
137    parser.add_option("-p", "--test-package", dest="test_package",
138                      help="Restrict test to a specific java package")
139    parser.add_option("-z", "--size", dest="test_size",
140                      help="Restrict test to a specific test size")
141    parser.add_option("--annotation", dest="test_annotation",
142                      help="Include only those tests tagged with a specific"
143                      " annotation")
144    parser.add_option("--not-annotation", dest="test_not_annotation",
145                      help="Exclude any tests tagged with a specific"
146                      " annotation")
147    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
148                      metavar="FILE", default=user_test_default,
149                      help="Alternate source of user test definitions")
150    parser.add_option("-o", "--coverage", dest="coverage",
151                      default=False, action="store_true",
152                      help="Generate code coverage metrics for test(s)")
153    parser.add_option("--coverage-target", dest="coverage_target_path",
154                      default=None,
155                      help="Path to app to collect code coverage target data for.")
156    parser.add_option("-k", "--skip-permissions", dest="skip_permissions",
157                      default=False, action="store_true",
158                      help="Do not grant runtime permissions during test package"
159                      " installation.")
160    parser.add_option("-x", "--path", dest="test_path",
161                      help="Run test(s) at given file system path")
162    parser.add_option("-t", "--all-tests", dest="all_tests",
163                      default=False, action="store_true",
164                      help="Run all defined tests")
165    parser.add_option("--continuous", dest="continuous_tests",
166                      default=False, action="store_true",
167                      help="Run all tests defined as part of the continuous "
168                      "test set")
169    parser.add_option("--timeout", dest="timeout",
170                      default=300, help="Set a timeout limit (in sec) for "
171                      "running native tests on a device (default: 300 secs)")
172    parser.add_option("--suite", dest="suite",
173                      help="Run all tests defined as part of the "
174                      "the given test suite")
175    parser.add_option("--user", dest="user",
176                      help="The user that test apks are installing to."
177                      " This is the integer user id, e.g. 0 or 10."
178                      " If no user is specified, apk will be installed with"
179                      " adb's default behavior, which is currently all users.")
180    parser.add_option("--install-filter", dest="filter_re",
181                      help="Regular expression which generated apks have to"
182                      " match to be installed to target device. Default is None"
183                      " and will install all packages built.  This is"
184                      " useful when the test path has a lot of apks but you"
185                      " only care about one.")
186    group = optparse.OptionGroup(
187        parser, "Targets", "Use these options to direct tests to a specific "
188        "Android target")
189    group.add_option("-e", "--emulator", dest="emulator", default=False,
190                     action="store_true", help="use emulator")
191    group.add_option("-d", "--device", dest="device", default=False,
192                     action="store_true", help="use device")
193    group.add_option("-s", "--serial", dest="serial",
194                     help="use specific serial")
195    parser.add_option_group(group)
196    self._options, self._test_args = parser.parse_args()
197
198    if (not self._options.only_list_tests
199        and not self._options.all_tests
200        and not self._options.continuous_tests
201        and not self._options.suite
202        and not self._options.test_path
203        and len(self._test_args) < 1):
204      parser.print_help()
205      logger.SilentLog("at least one test name must be specified")
206      raise errors.AbortError
207
208    self._adb = adb_interface.AdbInterface()
209    if self._options.emulator:
210      self._adb.SetEmulatorTarget()
211    elif self._options.device:
212      self._adb.SetDeviceTarget()
213    elif self._options.serial is not None:
214      self._adb.SetTargetSerial(self._options.serial)
215    if self._options.verbose:
216      logger.SetVerbose(True)
217
218    if self._options.coverage_target_path:
219      self._options.coverage = True
220
221    self._known_tests = self._ReadTests()
222
223    self._options.host_lib_path = android_build.GetHostLibraryPath()
224    self._options.test_data_path = android_build.GetTestAppPath()
225
226  def _ReadTests(self):
227    """Parses the set of test definition data.
228
229    Returns:
230      A TestDefinitions object that contains the set of parsed tests.
231    Raises:
232      AbortError: If a fatal error occurred when parsing the tests.
233    """
234    try:
235      known_tests = test_defs.TestDefinitions()
236      # only read tests when not in path mode
237      if not self._options.test_path:
238        core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
239        if os.path.isfile(core_test_path):
240          known_tests.Parse(core_test_path)
241        # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
242        vendor_tests_pattern = os.path.join(self._root_path,
243                                            self._VENDOR_TEST_PATH)
244        test_file_paths = glob.glob(vendor_tests_pattern)
245        for test_file_path in test_file_paths:
246          known_tests.Parse(test_file_path)
247        if os.path.isfile(self._options.user_tests_file):
248          known_tests.Parse(self._options.user_tests_file)
249      return known_tests
250    except errors.ParseError:
251      raise errors.AbortError
252
253  def _DumpTests(self):
254    """Prints out set of defined tests."""
255    print "The following tests are currently defined:\n"
256    print "%-25s %-40s %s" % ("name", "build path", "description")
257    print "-" * 80
258    for test in self._known_tests:
259      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
260                                test.GetDescription())
261    print "\nSee %s for more information" % self._TEST_FILE_NAME
262
263  def _DoBuild(self):
264    logger.SilentLog("Building tests...")
265    tests = self._GetTestsToRun()
266
267    # Build and install tests that do not get granted permissions
268    self._DoPermissionAwareBuild(tests, False)
269
270    # Build and install tests that require granted permissions
271    self._DoPermissionAwareBuild(tests, True)
272
273  def _DoPermissionAwareBuild(self, tests, test_requires_permissions):
274    # turn off dalvik verifier if necessary
275    # TODO: skip turning off verifier for now, since it puts device in bad
276    # state b/14088982
277    #self._TurnOffVerifier(tests)
278    self._DoFullBuild(tests, test_requires_permissions)
279
280    target_tree = make_tree.MakeTree()
281
282    extra_args_set = []
283    for test_suite in tests:
284      if test_suite.IsGrantedPermissions() == test_requires_permissions:
285        self._AddBuildTarget(test_suite, target_tree, extra_args_set)
286
287    if not self._options.preview:
288      self._adb.EnableAdbRoot()
289    else:
290      logger.Log("adb root")
291
292    if not target_tree.IsEmpty():
293      if self._options.coverage:
294        coverage.EnableCoverageBuild()
295        target_tree.AddPath("external/emma")
296
297      target_list = target_tree.GetPrunedMakeList()
298      target_dir_list = [re.sub(r'Android[.]mk$', r'', i) for i in target_list]
299      target_build_string = " ".join(target_list)
300      target_dir_build_string = " ".join(target_dir_list)
301      extra_args_string = " ".join(extra_args_set)
302
303      install_path_goals = []
304      mmma_goals = []
305      for d in target_dir_list:
306        if d.startswith("./"):
307          d = d[2:]
308        if d.endswith("/"):
309          d = d[:-1]
310        install_path_goals.append("GET-INSTALL-PATH-IN-" + d.replace("/","-"))
311        mmma_goals.append("MODULES-IN-" + d.replace("/","-"))
312      # mmm cannot be used from python, so perform a similar operation using
313      # ONE_SHOT_MAKEFILE
314      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" %s %s %s' % (
315          target_build_string, self._options.make_jobs, self._root_path,
316          " ".join(install_path_goals), " ".join(mmma_goals), extra_args_string)
317      # mmma cannot be used from python, so perform a similar operation
318      alt_cmd = 'make -j%s -C "%s" -f build/core/main.mk %s %s' % (
319              self._options.make_jobs, self._root_path, extra_args_string, " ".join(mmma_goals))
320
321      logger.Log(cmd)
322      if not self._options.preview:
323        run_command.SetAbortOnError()
324        try:
325          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
326          ## Chances are this failed because it didn't build the dependencies
327        except errors.AbortError:
328          logger.Log("make failed. Trying to rebuild all dependencies.")
329          logger.Log("mmma -j%s %s" %(self._options.make_jobs, target_dir_build_string))
330          # Try again with mma equivalent, which will build the dependencies
331          run_command.RunCommand(alt_cmd, return_output=False, timeout_time=600)
332          # Run mmm again to get the install paths only
333          output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
334        run_command.SetAbortOnError(False)
335        logger.SilentLog(output)
336        filter_re = re.compile(self._options.filter_re) if self._options.filter_re else None
337
338        self._DoInstall(output, test_requires_permissions, filter_re=filter_re)
339
340  def _DoInstall(self, make_output, test_requires_permissions, filter_re=None):
341    """Install artifacts from build onto device.
342
343    Looks for 'install:' text from make output to find artifacts to install.
344
345    Files with the .apk extension get 'adb install'ed, all other files
346    get 'adb push'ed onto the device.
347
348    Args:
349      make_output: stdout from make command
350    """
351    for line in make_output.split("\n"):
352      m = self._RE_MAKE_INSTALL.match(line)
353      if m:
354        # strip the 'INSTALL: <name>' from the left hand side
355        # the remaining string is a space-separated list of build-generated files
356        install_paths = m.group(2)
357        for install_path in re.split(r'\s+', install_paths):
358          if filter_re and not filter_re.match(install_path):
359            continue
360          if install_path.endswith(".apk"):
361            abs_install_path = os.path.join(self._root_path, install_path)
362            extra_flags = ""
363            if test_requires_permissions and not self._options.skip_permissions:
364              extra_flags = "-g"
365            if self._options.user:
366              extra_flags += " --user " + self._options.user
367            logger.Log("adb install -r %s %s" % (extra_flags, abs_install_path))
368            logger.Log(self._adb.Install(abs_install_path, extra_flags))
369          else:
370            self._PushInstallFileToDevice(install_path)
371
372  def _PushInstallFileToDevice(self, install_path):
373    m = self._re_make_install_path.match(install_path)
374    if m:
375      remote_path = m.group(1)
376      remote_dir = os.path.dirname(remote_path)
377      logger.Log("adb shell mkdir -p %s" % remote_dir)
378      self._adb.SendShellCommand("mkdir -p %s" % remote_dir)
379      abs_install_path = os.path.join(self._root_path, install_path)
380      logger.Log("adb push %s %s" % (abs_install_path, remote_path))
381      self._adb.Push(abs_install_path, remote_path)
382    else:
383      logger.Log("Error: Failed to recognize path of file to install %s" % install_path)
384
385  def _DoFullBuild(self, tests, test_requires_permissions):
386    """If necessary, run a full 'make' command for the tests that need it."""
387    extra_args_set = Set()
388
389    for test in tests:
390      if test.IsFullMake() and test.IsGrantedPermissions() == test_requires_permissions:
391        if test.GetExtraBuildArgs():
392          # extra args contains the args to pass to 'make'
393          extra_args_set.add(test.GetExtraBuildArgs())
394        else:
395          logger.Log("Warning: test %s needs a full build but does not specify"
396                     " extra_build_args" % test.GetName())
397
398    # check if there is actually any tests that required a full build
399    if extra_args_set:
400      cmd = ('make -j%s %s' % (self._options.make_jobs,
401                               ' '.join(list(extra_args_set))))
402      logger.Log(cmd)
403      if not self._options.preview:
404        old_dir = os.getcwd()
405        os.chdir(self._root_path)
406        output = run_command.RunCommand(cmd, return_output=True)
407        logger.SilentLog(output)
408        os.chdir(old_dir)
409        self._DoInstall(output, test_requires_permissions)
410
411  def _AddBuildTarget(self, test_suite, target_tree, extra_args_set):
412    if not test_suite.IsFullMake():
413      build_dir = test_suite.GetBuildPath()
414      if self._AddBuildTargetPath(build_dir, target_tree):
415        extra_args_set.append(test_suite.GetExtraBuildArgs())
416      for path in test_suite.GetBuildDependencies(self._options):
417        self._AddBuildTargetPath(path, target_tree)
418
419  def _AddBuildTargetPath(self, build_dir, target_tree):
420    if build_dir is not None:
421      target_tree.AddPath(build_dir)
422      return True
423    return False
424
425  def _GetTestsToRun(self):
426    """Get a list of TestSuite objects to run, based on command line args."""
427    if self._tests_to_run:
428      return self._tests_to_run
429
430    self._tests_to_run = []
431    if self._options.all_tests:
432      self._tests_to_run = self._known_tests.GetTests()
433    elif self._options.continuous_tests:
434      self._tests_to_run = self._known_tests.GetContinuousTests()
435    elif self._options.suite:
436      self._tests_to_run = \
437          self._known_tests.GetTestsInSuite(self._options.suite)
438    elif self._options.test_path:
439      walker = test_walker.TestWalker()
440      self._tests_to_run = walker.FindTests(self._options.test_path)
441
442    for name in self._test_args:
443      test = self._known_tests.GetTest(name)
444      if test is None:
445        logger.Log("Error: Could not find test %s" % name)
446        self._DumpTests()
447        raise errors.AbortError
448      self._tests_to_run.append(test)
449    return self._tests_to_run
450
451  def _TurnOffVerifier(self, test_list):
452    """Turn off the dalvik verifier if needed by given tests.
453
454    If one or more tests needs dalvik verifier off, and it is not already off,
455    turns off verifier and reboots device to allow change to take effect.
456    """
457    # hack to check if these are frameworks/base tests. If so, turn off verifier
458    # to allow framework tests to access private/protected/package-private framework api
459    framework_test = False
460    for test in test_list:
461      if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]):
462        framework_test = True
463    if framework_test:
464      # check if verifier is off already - to avoid the reboot if not
465      # necessary
466      output = self._adb.SendShellCommand("cat /data/local.prop")
467      if not self._DALVIK_VERIFIER_OFF_PROP in output:
468
469        # Read the existing dalvik verifier flags.
470        old_prop_value = self._adb.SendShellCommand("getprop %s" \
471            %(self._DALVIK_VERIFIER_PROP))
472        old_prop_value = old_prop_value.strip() if old_prop_value else ""
473
474        # Append our verifier flags to existing flags
475        new_prop_value = "%s %s" %(self._DALVIK_VERIFIER_OFF_VALUE, old_prop_value)
476
477        # Update property now, as /data/local.prop is not read until reboot
478        logger.Log("adb shell setprop %s '%s'" \
479            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
480        if not self._options.preview:
481          self._adb.SendShellCommand("setprop %s '%s'" \
482            %(self._DALVIK_VERIFIER_PROP, new_prop_value))
483
484        # Write prop to /data/local.prop
485        # Every time device is booted, it will pick up this value
486        new_prop_assignment = "%s = %s" %(self._DALVIK_VERIFIER_PROP, new_prop_value)
487        if self._options.preview:
488          logger.Log("adb shell \"echo %s >> /data/local.prop\""
489                     % new_prop_assignment)
490          logger.Log("adb shell chmod 644 /data/local.prop")
491        else:
492          logger.Log("Turning off dalvik verifier and rebooting")
493          self._adb.SendShellCommand("\"echo %s >> /data/local.prop\""
494                                     % new_prop_assignment)
495
496        # Reset runtime so that dalvik picks up new verifier flags from prop
497        self._ChmodRuntimeReset()
498      elif not self._options.preview:
499        # check the permissions on the file
500        permout = self._adb.SendShellCommand("ls -l /data/local.prop")
501        if not "-rw-r--r--" in permout:
502          logger.Log("Fixing permissions on /data/local.prop and rebooting")
503          self._ChmodRuntimeReset()
504
505  def _ChmodRuntimeReset(self):
506    """Perform a chmod of /data/local.prop and reset the runtime.
507    """
508    logger.Log("adb shell chmod 644 /data/local.prop ## u+w,a+r")
509    if not self._options.preview:
510      self._adb.SendShellCommand("chmod 644 /data/local.prop")
511
512    self._adb.RuntimeReset(preview_only=self._options.preview)
513
514    if not self._options.preview:
515      self._adb.EnableAdbRoot()
516
517
518  def RunTests(self):
519    """Main entry method - executes the tests according to command line args."""
520    try:
521      run_command.SetAbortOnError()
522      self._ProcessOptions()
523      if self._options.only_list_tests:
524        self._DumpTests()
525        return
526
527      if not self._options.skip_build:
528        self._DoBuild()
529
530      if self._options.build_install_only:
531        logger.Log("Skipping test execution (due to --build-install-only flag)")
532        return
533
534      for test_suite in self._GetTestsToRun():
535        try:
536          test_suite.Run(self._options, self._adb)
537        except errors.WaitForResponseTimedOutError:
538          logger.Log("Timed out waiting for response")
539
540    except KeyboardInterrupt:
541      logger.Log("Exiting...")
542    except errors.AbortError, error:
543      logger.Log(error.msg)
544      logger.SilentLog("Exiting due to AbortError...")
545    except errors.WaitForResponseTimedOutError:
546      logger.Log("Timed out waiting for response")
547
548
549def RunTests():
550  runner = TestRunner()
551  runner.RunTests()
552
553if __name__ == "__main__":
554  RunTests()
555