setup.py revision 424c4d7b64af9d0d8fd9624f381f469654d5e3d2 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Generates test runner factory and tests for GTests."""
6
7import fnmatch
8import glob
9import logging
10import os
11import shutil
12import sys
13
14from pylib import android_commands
15from pylib import cmd_helper
16from pylib import constants
17from pylib import ports
18
19import test_package_apk
20import test_package_exe
21import test_runner
22
23sys.path.insert(0,
24                os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
25                             'common'))
26import unittest_util
27
28
29_ISOLATE_FILE_PATHS = {
30    'base_unittests': 'base/base_unittests.isolate',
31    'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
32    'cc_perftests': 'cc/cc_perftests.isolate',
33    'components_unittests': 'components/components_unittests.isolate',
34    'content_browsertests': 'content/content_browsertests.isolate',
35    'content_unittests': 'content/content_unittests.isolate',
36    'media_unittests': 'media/media_unittests.isolate',
37    'net_unittests': 'net/net_unittests.isolate',
38    'ui_unittests': 'ui/ui_unittests.isolate',
39    'unit_tests': 'chrome/unit_tests.isolate',
40    'webkit_unit_tests':
41      'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
42}
43
44# Paths relative to third_party/webrtc/ (kept separate for readability).
45_WEBRTC_ISOLATE_FILE_PATHS = {
46    'audio_decoder_unittests':
47      'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
48    'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
49    'common_video_unittests': 'common_video/common_video_unittests.isolate',
50    'metrics_unittests': 'test/metrics_unittests.isolate',
51    'modules_tests': 'modules/modules_tests.isolate',
52    'modules_unittests': 'modules/modules_unittests.isolate',
53    'neteq_unittests': 'modules/audio_coding/neteq/neteq_unittests.isolate',
54    'system_wrappers_unittests':
55      'system_wrappers/source/system_wrappers_unittests.isolate',
56    'test_support_unittests': 'test/test_support_unittests.isolate',
57    'tools_unittests': 'tools/tools_unittests.isolate',
58    'video_engine_core_unittests':
59      'video_engine/video_engine_core_unittests.isolate',
60    'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
61}
62
63# Append the WebRTC tests with the full path from Chromium's src/ root.
64for test,isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items():
65  _ISOLATE_FILE_PATHS[test] = 'third_party/webrtc/%s' % isolate_path
66
67# Used for filtering large data deps at a finer grain than what's allowed in
68# isolate files since pushing deps to devices is expensive.
69# Wildcards are allowed.
70_DEPS_EXCLUSION_LIST = [
71    'chrome/test/data/extensions/api_test',
72    'chrome/test/data/extensions/secure_shell',
73    'chrome/test/data/firefox*',
74    'chrome/test/data/gpu',
75    'chrome/test/data/image_decoding',
76    'chrome/test/data/import',
77    'chrome/test/data/page_cycler',
78    'chrome/test/data/perf',
79    'chrome/test/data/pyauto_private',
80    'chrome/test/data/safari_import',
81    'chrome/test/data/scroll',
82    'chrome/test/data/third_party',
83    'third_party/hunspell_dictionaries/*.dic',
84    # crbug.com/258690
85    'webkit/data/bmp_decoder',
86    'webkit/data/ico_decoder',
87]
88
89_ISOLATE_SCRIPT = os.path.join(
90    constants.DIR_SOURCE_ROOT, 'tools', 'swarm_client', 'isolate.py')
91
92
93def _GenerateDepsDirUsingIsolate(suite_name):
94  """Generate the dependency dir for the test suite using isolate.
95
96  Args:
97    suite_name: Name of the test suite (e.g. base_unittests).
98  """
99  product_dir = os.path.join(cmd_helper.OutDirectory.get(),
100      constants.GetBuildType())
101  assert os.path.isabs(product_dir)
102
103  if os.path.isdir(constants.ISOLATE_DEPS_DIR):
104    shutil.rmtree(constants.ISOLATE_DEPS_DIR)
105
106  isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
107  if not isolate_rel_path:
108    logging.info('Did not find an isolate file for the test suite.')
109    return
110
111  isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
112  isolated_abs_path = os.path.join(
113      product_dir, '%s.isolated' % suite_name)
114  assert os.path.exists(isolate_abs_path)
115  isolate_cmd = [
116      'python', _ISOLATE_SCRIPT,
117      'remap',
118      '--isolate', isolate_abs_path,
119      '--isolated', isolated_abs_path,
120      '-V', 'PRODUCT_DIR=%s' % product_dir,
121      '-V', 'OS=android',
122      '--outdir', constants.ISOLATE_DEPS_DIR,
123  ]
124  assert not cmd_helper.RunCmd(isolate_cmd)
125
126  # We're relying on the fact that timestamps are preserved
127  # by the remap command (hardlinked). Otherwise, all the data
128  # will be pushed to the device once we move to using time diff
129  # instead of md5sum. Perform a sanity check here.
130  for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
131    if filenames:
132      linked_file = os.path.join(root, filenames[0])
133      orig_file = os.path.join(
134          constants.DIR_SOURCE_ROOT,
135          os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
136      if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
137        break
138      else:
139        raise Exception('isolate remap command did not use hardlinks.')
140
141  # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
142  old_cwd = os.getcwd()
143  try:
144    os.chdir(constants.ISOLATE_DEPS_DIR)
145    excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
146    if excluded_paths:
147      logging.info('Excluding the following from dependency list: %s',
148                   excluded_paths)
149    for p in excluded_paths:
150      if os.path.isdir(p):
151        shutil.rmtree(p)
152      else:
153        os.remove(p)
154  finally:
155    os.chdir(old_cwd)
156
157  # On Android, all pak files need to be in the top-level 'paks' directory.
158  paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
159  os.mkdir(paks_dir)
160  for root, _, filenames in os.walk(os.path.join(constants.ISOLATE_DEPS_DIR,
161                                                 'out')):
162    for filename in fnmatch.filter(filenames, '*.pak'):
163      shutil.move(os.path.join(root, filename), paks_dir)
164
165  # Move everything in PRODUCT_DIR to top level.
166  deps_product_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'out',
167      constants.GetBuildType())
168  if os.path.isdir(deps_product_dir):
169    for p in os.listdir(deps_product_dir):
170      shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
171    os.rmdir(deps_product_dir)
172    os.rmdir(os.path.join(constants.ISOLATE_DEPS_DIR, 'out'))
173
174
175def _GetDisabledTestsFilterFromFile(suite_name):
176  """Returns a gtest filter based on the *_disabled file.
177
178  Args:
179    suite_name: Name of the test suite (e.g. base_unittests).
180
181  Returns:
182    A gtest filter which excludes disabled tests.
183    Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
184  """
185  filter_file_path = os.path.join(
186      os.path.abspath(os.path.dirname(__file__)),
187      'filter', '%s_disabled' % suite_name)
188
189  if not filter_file_path or not os.path.exists(filter_file_path):
190    logging.info('No filter file found at %s', filter_file_path)
191    return '*'
192
193  filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
194             if x and x[0] != '#']
195  disabled_filter = '*-%s' % ':'.join(filters)
196  logging.info('Applying filter "%s" obtained from %s',
197               disabled_filter, filter_file_path)
198  return disabled_filter
199
200
201def _GetTestsFromDevice(runner_factory, devices):
202  """Get a list of tests from a device.
203
204  Args:
205    runner_factory: Callable that takes device and shard_index and returns
206        a TestRunner.
207    devices: A list of device ids.
208
209  Returns:
210    All the tests in the test suite.
211  """
212  for device in devices:
213    try:
214      logging.info('Obtaining tests from %s', device)
215      return runner_factory(device, 0).GetAllTests()
216    except (android_commands.errors.WaitForResponseTimedOutError,
217            android_commands.errors.DeviceUnresponsiveError), e:
218      logging.warning('Failed obtaining tests from %s with exception: %s',
219                      device, e)
220  raise Exception('No device available to get the list of tests.')
221
222
223def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
224  """Removes tests with disabled prefixes.
225
226  Args:
227    all_tests: List of tests to filter.
228    pre: If True, include tests with PRE_ prefix.
229    manual: If True, include tests with MANUAL_ prefix.
230
231  Returns:
232    List of tests remaining.
233  """
234  filtered_tests = []
235  filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
236
237  if not pre:
238    filter_prefixes.append('PRE_')
239
240  if not manual:
241    filter_prefixes.append('MANUAL_')
242
243  for t in all_tests:
244    test_case, test = t.split('.', 1)
245    if not any([test_case.startswith(prefix) or test.startswith(prefix) for
246                prefix in filter_prefixes]):
247      filtered_tests.append(t)
248  return filtered_tests
249
250
251def _GetTestsFiltered(suite_name, gtest_filter, runner_factory, devices):
252  """Get all tests in the suite and filter them.
253
254  Obtains a list of tests from the test package on the device, and
255  applies the following filters in order:
256    1. Remove tests with disabled prefixes.
257    2. Remove tests specified in the *_disabled files in the 'filter' dir
258    3. Applies |gtest_filter|.
259
260  Args:
261    suite_name: Name of the test suite (e.g. base_unittests).
262    gtest_filter: A filter including negative and/or positive patterns.
263    runner_factory: callable that takes a device and index and returns a
264      TestRunner object.
265    devices: List of devices.
266
267  Returns:
268    List of tests remaining.
269  """
270  tests = _GetTestsFromDevice(runner_factory, devices)
271  tests = _FilterTestsUsingPrefixes(
272      tests, bool(gtest_filter), bool(gtest_filter))
273  tests = unittest_util.FilterTestNames(
274      tests, _GetDisabledTestsFilterFromFile(suite_name))
275
276  if gtest_filter:
277    tests = unittest_util.FilterTestNames(tests, gtest_filter)
278
279  return tests
280
281
282def Setup(test_options, devices):
283  """Create the test runner factory and tests.
284
285  Args:
286    test_options: A GTestOptions object.
287    devices: A list of attached devices.
288
289  Returns:
290    A tuple of (TestRunnerFactory, tests).
291  """
292
293  if not ports.ResetTestServerPortAllocation():
294    raise Exception('Failed to reset test server port.')
295
296  test_package = test_package_apk.TestPackageApk(test_options.suite_name)
297  if not os.path.exists(test_package.suite_path):
298    test_package = test_package_exe.TestPackageExecutable(
299        test_options.suite_name)
300    if not os.path.exists(test_package.suite_path):
301      raise Exception(
302          'Did not find %s target. Ensure it has been built.'
303          % test_options.suite_name)
304  logging.warning('Found target %s', test_package.suite_path)
305
306  _GenerateDepsDirUsingIsolate(test_options.suite_name)
307
308  # Constructs a new TestRunner with the current options.
309  def TestRunnerFactory(device, shard_index):
310    return test_runner.TestRunner(
311        test_options,
312        device,
313        test_package)
314
315  tests = _GetTestsFiltered(test_options.suite_name, test_options.gtest_filter,
316                            TestRunnerFactory, devices)
317  # Coalesce unit tests into a single test per device
318  if test_options.suite_name != 'content_browsertests':
319    num_devices = len(devices)
320    tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
321    tests = [t for t in tests if t]
322
323  return (TestRunnerFactory, tests)
324