test_runner.py revision f2477e01787aa58f445919b809d89e252beef54f 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import logging
6import os
7import re
8
9from pylib import android_commands
10from pylib import constants
11from pylib import pexpect
12from pylib.base import base_test_result
13from pylib.base import base_test_runner
14from pylib.perf import perf_control
15
16
17def _TestSuiteRequiresMockTestServer(suite_name):
18  """Returns True if the test suite requires mock test server."""
19  tests_require_net_test_server = ['unit_tests', 'net_unittests',
20                                   'content_unittests',
21                                   'content_browsertests']
22  return (suite_name in
23          tests_require_net_test_server)
24
25def _TestSuiteRequiresHighPerfMode(suite_name):
26  """Returns True if the test suite requires high performance mode."""
27  return 'perftests' in suite_name
28
29class TestRunner(base_test_runner.BaseTestRunner):
30  def __init__(self, test_options, device, test_package):
31    """Single test suite attached to a single device.
32
33    Args:
34      test_options: A GTestOptions object.
35      device: Device to run the tests.
36      test_package: An instance of TestPackage class.
37    """
38
39    super(TestRunner, self).__init__(device, test_options.tool,
40                                     test_options.push_deps,
41                                     test_options.cleanup_test_files)
42
43    self.test_package = test_package
44    self.test_package.tool = self.tool
45    self._test_arguments = test_options.test_arguments
46
47    timeout = test_options.timeout
48    if timeout == 0:
49      timeout = 60
50    # On a VM (e.g. chromium buildbots), this timeout is way too small.
51    if os.environ.get('BUILDBOT_SLAVENAME'):
52      timeout = timeout * 2
53
54    self._timeout = timeout * self.tool.GetTimeoutScale()
55    if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
56      self._perf_controller = perf_control.PerfControl(self.adb)
57
58  #override
59  def InstallTestPackage(self):
60    self.test_package.Install(self.adb)
61
62  def GetAllTests(self):
63    """Install test package and get a list of all tests."""
64    self.test_package.Install(self.adb)
65    return self.test_package.GetAllTests(self.adb)
66
67  #override
68  def PushDataDeps(self):
69    self.adb.WaitForSdCardReady(20)
70    self.tool.CopyFiles()
71    if os.path.exists(constants.ISOLATE_DEPS_DIR):
72      device_dir = self.adb.GetExternalStorage()
73      # TODO(frankf): linux_dumper_unittest_helper needs to be in the same dir
74      # as breakpad_unittests exe. Find a better way to do this.
75      if self.test_package.suite_name == 'breakpad_unittests':
76        device_dir = constants.TEST_EXECUTABLE_DIR
77      for p in os.listdir(constants.ISOLATE_DEPS_DIR):
78        self.adb.PushIfNeeded(
79            os.path.join(constants.ISOLATE_DEPS_DIR, p),
80            os.path.join(device_dir, p))
81
82  def _ParseTestOutput(self, p):
83    """Process the test output.
84
85    Args:
86      p: An instance of pexpect spawn class.
87
88    Returns:
89      A TestRunResults object.
90    """
91    results = base_test_result.TestRunResults()
92
93    # Test case statuses.
94    re_run = re.compile('\[ RUN      \] ?(.*)\r\n')
95    re_fail = re.compile('\[  FAILED  \] ?(.*)\r\n')
96    re_ok = re.compile('\[       OK \] ?(.*?) .*\r\n')
97
98    # Test run statuses.
99    re_passed = re.compile('\[  PASSED  \] ?(.*)\r\n')
100    re_runner_fail = re.compile('\[ RUNNER_FAILED \] ?(.*)\r\n')
101    # Signal handlers are installed before starting tests
102    # to output the CRASHED marker when a crash happens.
103    re_crash = re.compile('\[ CRASHED      \](.*)\r\n')
104
105    log = ''
106    try:
107      while True:
108        full_test_name = None
109        found = p.expect([re_run, re_passed, re_runner_fail],
110                         timeout=self._timeout)
111        if found == 1:  # re_passed
112          break
113        elif found == 2:  # re_runner_fail
114          break
115        else:  # re_run
116          full_test_name = p.match.group(1).replace('\r', '')
117          found = p.expect([re_ok, re_fail, re_crash], timeout=self._timeout)
118          log = p.before.replace('\r', '')
119          if found == 0:  # re_ok
120            if full_test_name == p.match.group(1).replace('\r', ''):
121              results.AddResult(base_test_result.BaseTestResult(
122                  full_test_name, base_test_result.ResultType.PASS,
123                  log=log))
124          elif found == 2:  # re_crash
125            results.AddResult(base_test_result.BaseTestResult(
126                full_test_name, base_test_result.ResultType.CRASH,
127                log=log))
128            break
129          else:  # re_fail
130            results.AddResult(base_test_result.BaseTestResult(
131                full_test_name, base_test_result.ResultType.FAIL, log=log))
132    except pexpect.EOF:
133      logging.error('Test terminated - EOF')
134      # We're here because either the device went offline, or the test harness
135      # crashed without outputting the CRASHED marker (crbug.com/175538).
136      if not self.adb.IsOnline():
137        raise android_commands.errors.DeviceUnresponsiveError(
138            'Device %s went offline.' % self.device)
139      if full_test_name:
140        results.AddResult(base_test_result.BaseTestResult(
141            full_test_name, base_test_result.ResultType.CRASH,
142            log=p.before.replace('\r', '')))
143    except pexpect.TIMEOUT:
144      logging.error('Test terminated after %d second timeout.',
145                    self._timeout)
146      if full_test_name:
147        results.AddResult(base_test_result.BaseTestResult(
148            full_test_name, base_test_result.ResultType.TIMEOUT,
149            log=p.before.replace('\r', '')))
150    finally:
151      p.close()
152
153    ret_code = self.test_package.GetGTestReturnCode(self.adb)
154    if ret_code:
155      logging.critical(
156          'gtest exit code: %d\npexpect.before: %s\npexpect.after: %s',
157          ret_code, p.before, p.after)
158
159    return results
160
161  #override
162  def RunTest(self, test):
163    test_results = base_test_result.TestRunResults()
164    if not test:
165      return test_results, None
166
167    try:
168      self.test_package.ClearApplicationState(self.adb)
169      self.test_package.CreateCommandLineFileOnDevice(
170          self.adb, test, self._test_arguments)
171      test_results = self._ParseTestOutput(
172          self.test_package.SpawnTestProcess(self.adb))
173    finally:
174      self.CleanupSpawningServerState()
175    # Calculate unknown test results.
176    all_tests = set(test.split(':'))
177    all_tests_ran = set([t.GetName() for t in test_results.GetAll()])
178    unknown_tests = all_tests - all_tests_ran
179    test_results.AddResults(
180        [base_test_result.BaseTestResult(t, base_test_result.ResultType.UNKNOWN)
181         for t in unknown_tests])
182    retry = ':'.join([t.GetName() for t in test_results.GetNotPass()])
183    return test_results, retry
184
185  #override
186  def SetUp(self):
187    """Sets up necessary test enviroment for the test suite."""
188    super(TestRunner, self).SetUp()
189    if _TestSuiteRequiresMockTestServer(self.test_package.suite_name):
190      self.LaunchChromeTestServerSpawner()
191    if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
192      self._perf_controller.SetHighPerfMode()
193    self.tool.SetupEnvironment()
194
195  #override
196  def TearDown(self):
197    """Cleans up the test enviroment for the test suite."""
198    if _TestSuiteRequiresHighPerfMode(self.test_package.suite_name):
199      self._perf_controller.RestoreOriginalPerfMode()
200    self.test_package.ClearApplicationState(self.adb)
201    self.tool.CleanUpEnvironment()
202    super(TestRunner, self).TearDown()
203