1#!/usr/bin/env python
2#
3# Copyright 2008, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Command line utility for running Android tests
18
19runtest helps automate the instructions for building and running tests
20- It builds the corresponding test package for the code you want to test
21- It pushes the test package to your device or emulator
22- It launches InstrumentationTestRunner (or similar) to run the tests you
23specify.
24
25runtest supports running tests whose attributes have been pre-defined in
26_TEST_FILE_NAME files, (runtest <testname>), or by specifying the file
27system path to the test to run (runtest --path <path>).
28
29Do runtest --help to see full list of options.
30"""
31
32# Python imports
33import glob
34import optparse
35import os
36import re
37from sets import Set
38import sys
39import time
40
41# local imports
42import adb_interface
43import android_build
44from coverage import coverage
45import errors
46import logger
47import make_tree
48import run_command
49from test_defs import test_defs
50from test_defs import test_walker
51
52
53class TestRunner(object):
54  """Command line utility class for running pre-defined Android test(s)."""
55
56  _TEST_FILE_NAME = "test_defs.xml"
57
58  # file path to android core platform tests, relative to android build root
59  # TODO move these test data files to another directory
60  _CORE_TEST_PATH = os.path.join("development", "testrunner",
61                                 _TEST_FILE_NAME)
62
63  # vendor glob file path patterns to tests, relative to android
64  # build root
65  _VENDOR_TEST_PATH = os.path.join("vendor", "*", "tests", "testinfo",
66                                   _TEST_FILE_NAME)
67
68  _RUNTEST_USAGE = (
69      "usage: runtest.py [options] short-test-name[s]\n\n"
70      "The runtest script works in two ways.  You can query it "
71      "for a list of tests, or you can launch one or more tests.")
72
73  # default value for make -jX
74  _DEFAULT_JOBS = 16
75
76  _DALVIK_VERIFIER_OFF_PROP = "dalvik.vm.dexopt-flags = v=n"
77
78  # regular expression to match install: statements in make output
79  _RE_MAKE_INSTALL = re.compile(r'Install:\s(.+)')
80
81  # regular expression to find remote device path from a file path relative
82  # to build root
83  _RE_MAKE_INSTALL_PATH = re.compile(r'out\/target\/product\/\w+\/(.+)$')
84
85  def __init__(self):
86    # disable logging of timestamp
87    self._root_path = android_build.GetTop()
88    logger.SetTimestampLogging(False)
89    self._adb = None
90    self._known_tests = None
91    self._options = None
92    self._test_args = None
93    self._tests_to_run = None
94
95  def _ProcessOptions(self):
96    """Processes command-line options."""
97    # TODO error messages on once-only or mutually-exclusive options.
98    user_test_default = os.path.join(os.environ.get("HOME"), ".android",
99                                     self._TEST_FILE_NAME)
100
101    parser = optparse.OptionParser(usage=self._RUNTEST_USAGE)
102
103    parser.add_option("-l", "--list-tests", dest="only_list_tests",
104                      default=False, action="store_true",
105                      help="To view the list of tests")
106    parser.add_option("-b", "--skip-build", dest="skip_build", default=False,
107                      action="store_true", help="Skip build - just launch")
108    parser.add_option("-j", "--jobs", dest="make_jobs",
109                      metavar="X", default=self._DEFAULT_JOBS,
110                      help="Number of make jobs to use when building")
111    parser.add_option("-n", "--skip_execute", dest="preview", default=False,
112                      action="store_true",
113                      help="Do not execute, just preview commands")
114    parser.add_option("-r", "--raw-mode", dest="raw_mode", default=False,
115                      action="store_true",
116                      help="Raw mode (for output to other tools)")
117    parser.add_option("-a", "--suite-assign", dest="suite_assign_mode",
118                      default=False, action="store_true",
119                      help="Suite assignment (for details & usage see "
120                      "InstrumentationTestRunner)")
121    parser.add_option("-v", "--verbose", dest="verbose", default=False,
122                      action="store_true",
123                      help="Increase verbosity of %s" % sys.argv[0])
124    parser.add_option("-w", "--wait-for-debugger", dest="wait_for_debugger",
125                      default=False, action="store_true",
126                      help="Wait for debugger before launching tests")
127    parser.add_option("-c", "--test-class", dest="test_class",
128                      help="Restrict test to a specific class")
129    parser.add_option("-m", "--test-method", dest="test_method",
130                      help="Restrict test to a specific method")
131    parser.add_option("-p", "--test-package", dest="test_package",
132                      help="Restrict test to a specific java package")
133    parser.add_option("-z", "--size", dest="test_size",
134                      help="Restrict test to a specific test size")
135    parser.add_option("--annotation", dest="test_annotation",
136                      help="Include only those tests tagged with a specific"
137                      " annotation")
138    parser.add_option("--not-annotation", dest="test_not_annotation",
139                      help="Exclude any tests tagged with a specific"
140                      " annotation")
141    parser.add_option("-u", "--user-tests-file", dest="user_tests_file",
142                      metavar="FILE", default=user_test_default,
143                      help="Alternate source of user test definitions")
144    parser.add_option("-o", "--coverage", dest="coverage",
145                      default=False, action="store_true",
146                      help="Generate code coverage metrics for test(s)")
147    parser.add_option("--coverage-target", dest="coverage_target_path",
148                      default=None,
149                      help="Path to app to collect code coverage target data for.")
150    parser.add_option("-x", "--path", dest="test_path",
151                      help="Run test(s) at given file system path")
152    parser.add_option("-t", "--all-tests", dest="all_tests",
153                      default=False, action="store_true",
154                      help="Run all defined tests")
155    parser.add_option("--continuous", dest="continuous_tests",
156                      default=False, action="store_true",
157                      help="Run all tests defined as part of the continuous "
158                      "test set")
159    parser.add_option("--timeout", dest="timeout",
160                      default=300, help="Set a timeout limit (in sec) for "
161                      "running native tests on a device (default: 300 secs)")
162    parser.add_option("--suite", dest="suite",
163                      help="Run all tests defined as part of the "
164                      "the given test suite")
165    group = optparse.OptionGroup(
166        parser, "Targets", "Use these options to direct tests to a specific "
167        "Android target")
168    group.add_option("-e", "--emulator", dest="emulator", default=False,
169                     action="store_true", help="use emulator")
170    group.add_option("-d", "--device", dest="device", default=False,
171                     action="store_true", help="use device")
172    group.add_option("-s", "--serial", dest="serial",
173                     help="use specific serial")
174    parser.add_option_group(group)
175    self._options, self._test_args = parser.parse_args()
176
177    if (not self._options.only_list_tests
178        and not self._options.all_tests
179        and not self._options.continuous_tests
180        and not self._options.suite
181        and not self._options.test_path
182        and len(self._test_args) < 1):
183      parser.print_help()
184      logger.SilentLog("at least one test name must be specified")
185      raise errors.AbortError
186
187    self._adb = adb_interface.AdbInterface()
188    if self._options.emulator:
189      self._adb.SetEmulatorTarget()
190    elif self._options.device:
191      self._adb.SetDeviceTarget()
192    elif self._options.serial is not None:
193      self._adb.SetTargetSerial(self._options.serial)
194
195    if self._options.verbose:
196      logger.SetVerbose(True)
197
198    if self._options.coverage_target_path:
199      self._options.coverage = True
200
201    self._known_tests = self._ReadTests()
202
203    self._options.host_lib_path = android_build.GetHostLibraryPath()
204    self._options.test_data_path = android_build.GetTestAppPath()
205
206  def _ReadTests(self):
207    """Parses the set of test definition data.
208
209    Returns:
210      A TestDefinitions object that contains the set of parsed tests.
211    Raises:
212      AbortError: If a fatal error occurred when parsing the tests.
213    """
214    try:
215      known_tests = test_defs.TestDefinitions()
216      # only read tests when not in path mode
217      if not self._options.test_path:
218        core_test_path = os.path.join(self._root_path, self._CORE_TEST_PATH)
219        if os.path.isfile(core_test_path):
220          known_tests.Parse(core_test_path)
221        # read all <android root>/vendor/*/tests/testinfo/test_defs.xml paths
222        vendor_tests_pattern = os.path.join(self._root_path,
223                                            self._VENDOR_TEST_PATH)
224        test_file_paths = glob.glob(vendor_tests_pattern)
225        for test_file_path in test_file_paths:
226          known_tests.Parse(test_file_path)
227        if os.path.isfile(self._options.user_tests_file):
228          known_tests.Parse(self._options.user_tests_file)
229      return known_tests
230    except errors.ParseError:
231      raise errors.AbortError
232
233  def _DumpTests(self):
234    """Prints out set of defined tests."""
235    print "The following tests are currently defined:\n"
236    print "%-25s %-40s %s" % ("name", "build path", "description")
237    print "-" * 80
238    for test in self._known_tests:
239      print "%-25s %-40s %s" % (test.GetName(), test.GetBuildPath(),
240                                test.GetDescription())
241    print "\nSee %s for more information" % self._TEST_FILE_NAME
242
243  def _DoBuild(self):
244    logger.SilentLog("Building tests...")
245
246    tests = self._GetTestsToRun()
247    # turn off dalvik verifier if necessary
248    self._TurnOffVerifier(tests)
249    self._DoFullBuild(tests)
250
251    target_tree = make_tree.MakeTree()
252
253    extra_args_set = []
254    for test_suite in tests:
255      self._AddBuildTarget(test_suite, target_tree, extra_args_set)
256
257    if not self._options.preview:
258      self._adb.EnableAdbRoot()
259    else:
260      logger.Log("adb root")
261
262    if not target_tree.IsEmpty():
263      if self._options.coverage:
264        coverage.EnableCoverageBuild()
265        target_tree.AddPath("external/emma")
266
267      target_list = target_tree.GetPrunedMakeList()
268      target_build_string = " ".join(target_list)
269      extra_args_string = " ".join(extra_args_set)
270
271      # mmm cannot be used from python, so perform a similar operation using
272      # ONE_SHOT_MAKEFILE
273      cmd = 'ONE_SHOT_MAKEFILE="%s" make -j%s -C "%s" all_modules %s' % (
274          target_build_string, self._options.make_jobs, self._root_path,
275          extra_args_string)
276      logger.Log(cmd)
277      if not self._options.preview:
278        output = run_command.RunCommand(cmd, return_output=True, timeout_time=600)
279        self._DoInstall(output)
280
281  def _DoInstall(self, make_output):
282    """Install artifacts from build onto device.
283
284    Looks for 'install:' text from make output to find artifacts to install.
285
286    Args:
287      make_output: stdout from make command
288    """
289    for line in make_output.split("\n"):
290      m = self._RE_MAKE_INSTALL.match(line)
291      if m:
292        install_path = m.group(1)
293        if install_path.endswith(".apk"):
294          abs_install_path = os.path.join(self._root_path, install_path)
295          logger.Log("adb install -r %s" % abs_install_path)
296          logger.Log(self._adb.Install(abs_install_path))
297        else:
298          self._PushInstallFileToDevice(install_path)
299
300  def _PushInstallFileToDevice(self, install_path):
301    m = self._RE_MAKE_INSTALL_PATH.match(install_path)
302    if m:
303      remote_path = m.group(1)
304      abs_install_path = os.path.join(self._root_path, install_path)
305      logger.Log("adb push %s %s" % (abs_install_path, remote_path))
306      self._adb.Push(abs_install_path, remote_path)
307    else:
308      logger.Log("Error: Failed to recognize path of file to install %s" % install_path)
309
310  def _DoFullBuild(self, tests):
311    """If necessary, run a full 'make' command for the tests that need it."""
312    extra_args_set = Set()
313
314    # hack to build cts dependencies
315    # TODO: remove this when cts dependencies are removed
316    if self._IsCtsTests(tests):
317      # need to use make since these fail building with ONE_SHOT_MAKEFILE
318      extra_args_set.add('CtsTestStubs')
319      extra_args_set.add('android.core.tests.runner')
320    for test in tests:
321      if test.IsFullMake():
322        if test.GetExtraBuildArgs():
323          # extra args contains the args to pass to 'make'
324          extra_args_set.add(test.GetExtraBuildArgs())
325        else:
326          logger.Log("Warning: test %s needs a full build but does not specify"
327                     " extra_build_args" % test.GetName())
328
329    # check if there is actually any tests that required a full build
330    if extra_args_set:
331      cmd = ('make -j%s %s' % (self._options.make_jobs,
332                               ' '.join(list(extra_args_set))))
333      logger.Log(cmd)
334      if not self._options.preview:
335        old_dir = os.getcwd()
336        os.chdir(self._root_path)
337        output = run_command.RunCommand(cmd, return_output=True)
338        os.chdir(old_dir)
339        self._DoInstall(output)
340
341  def _AddBuildTarget(self, test_suite, target_tree, extra_args_set):
342    if not test_suite.IsFullMake():
343      build_dir = test_suite.GetBuildPath()
344      if self._AddBuildTargetPath(build_dir, target_tree):
345        extra_args_set.append(test_suite.GetExtraBuildArgs())
346      for path in test_suite.GetBuildDependencies(self._options):
347        self._AddBuildTargetPath(path, target_tree)
348
349  def _AddBuildTargetPath(self, build_dir, target_tree):
350    if build_dir is not None:
351      target_tree.AddPath(build_dir)
352      return True
353    return False
354
355  def _GetTestsToRun(self):
356    """Get a list of TestSuite objects to run, based on command line args."""
357    if self._tests_to_run:
358      return self._tests_to_run
359
360    self._tests_to_run = []
361    if self._options.all_tests:
362      self._tests_to_run = self._known_tests.GetTests()
363    elif self._options.continuous_tests:
364      self._tests_to_run = self._known_tests.GetContinuousTests()
365    elif self._options.suite:
366      self._tests_to_run = \
367          self._known_tests.GetTestsInSuite(self._options.suite)
368    elif self._options.test_path:
369      walker = test_walker.TestWalker()
370      self._tests_to_run = walker.FindTests(self._options.test_path)
371
372    for name in self._test_args:
373      test = self._known_tests.GetTest(name)
374      if test is None:
375        logger.Log("Error: Could not find test %s" % name)
376        self._DumpTests()
377        raise errors.AbortError
378      self._tests_to_run.append(test)
379    return self._tests_to_run
380
381  def _IsCtsTests(self, test_list):
382    """Check if any cts tests are included in given list of tests to run."""
383    for test in test_list:
384      if test.GetSuite() == 'cts':
385        return True
386    return False
387
388  def _TurnOffVerifier(self, test_list):
389    """Turn off the dalvik verifier if needed by given tests.
390
391    If one or more tests needs dalvik verifier off, and it is not already off,
392    turns off verifier and reboots device to allow change to take effect.
393    """
394    # hack to check if these are frameworks/base tests. If so, turn off verifier
395    # to allow framework tests to access package-private framework api
396    framework_test = False
397    for test in test_list:
398      if os.path.commonprefix([test.GetBuildPath(), "frameworks/base"]):
399        framework_test = True
400    if framework_test:
401      # check if verifier is off already - to avoid the reboot if not
402      # necessary
403      output = self._adb.SendShellCommand("cat /data/local.prop")
404      if not self._DALVIK_VERIFIER_OFF_PROP in output:
405        if self._options.preview:
406          logger.Log("adb shell \"echo %s >> /data/local.prop\""
407                     % self._DALVIK_VERIFIER_OFF_PROP)
408          logger.Log("adb shell chmod 644 /data/local.prop")
409          logger.Log("adb reboot")
410          logger.Log("adb wait-for-device")
411        else:
412          logger.Log("Turning off dalvik verifier and rebooting")
413          self._adb.SendShellCommand("\"echo %s >> /data/local.prop\""
414                                     % self._DALVIK_VERIFIER_OFF_PROP)
415
416          self._ChmodReboot()
417      elif not self._options.preview:
418        # check the permissions on the file
419        permout = self._adb.SendShellCommand("ls -l /data/local.prop")
420        if not "-rw-r--r--" in permout:
421          logger.Log("Fixing permissions on /data/local.prop and rebooting")
422          self._ChmodReboot()
423
424  def _ChmodReboot(self):
425    """Perform a chmod of /data/local.prop and reboot.
426    """
427    self._adb.SendShellCommand("chmod 644 /data/local.prop")
428    self._adb.SendCommand("reboot")
429    # wait for device to go offline
430    time.sleep(10)
431    self._adb.SendCommand("wait-for-device", timeout_time=60,
432                          retry_count=3)
433    self._adb.EnableAdbRoot()
434
435
436  def RunTests(self):
437    """Main entry method - executes the tests according to command line args."""
438    try:
439      run_command.SetAbortOnError()
440      self._ProcessOptions()
441      if self._options.only_list_tests:
442        self._DumpTests()
443        return
444
445      if not self._options.skip_build:
446        self._DoBuild()
447
448      for test_suite in self._GetTestsToRun():
449        try:
450          test_suite.Run(self._options, self._adb)
451        except errors.WaitForResponseTimedOutError:
452          logger.Log("Timed out waiting for response")
453
454    except KeyboardInterrupt:
455      logger.Log("Exiting...")
456    except errors.AbortError, error:
457      logger.Log(error.msg)
458      logger.SilentLog("Exiting due to AbortError...")
459    except errors.WaitForResponseTimedOutError:
460      logger.Log("Timed out waiting for response")
461
462
463def RunTests():
464  runner = TestRunner()
465  runner.RunTests()
466
467if __name__ == "__main__":
468  RunTests()
469