setup.py revision 0f1bc08d4cfcc34181b0b5cbf065c40f687bf740 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Generates test runner factory and tests for GTests."""
6
7import fnmatch
8import glob
9import logging
10import os
11import shutil
12import sys
13
14from pylib import android_commands
15from pylib import cmd_helper
16from pylib import constants
17from pylib import ports
18
19import test_package_apk
20import test_package_exe
21import test_runner
22
23sys.path.insert(0,
24                os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
25                             'common'))
26import unittest_util
27
28
29_ISOLATE_FILE_PATHS = {
30    'base_unittests': 'base/base_unittests.isolate',
31    'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
32    'cc_perftests': 'cc/cc_perftests.isolate',
33    'components_unittests': 'components/components_unittests.isolate',
34    'content_browsertests': 'content/content_browsertests.isolate',
35    'content_unittests': 'content/content_unittests.isolate',
36    'media_unittests': 'media/media_unittests.isolate',
37    'net_unittests': 'net/net_unittests.isolate',
38    'ui_unittests': 'ui/ui_unittests.isolate',
39    'unit_tests': 'chrome/unit_tests.isolate',
40    'webkit_unit_tests':
41      'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
42}
43
44# Paths relative to third_party/webrtc/ (kept separate for readability).
45_WEBRTC_ISOLATE_FILE_PATHS = {
46    'audio_decoder_unittests':
47      'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
48    'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
49    'common_video_unittests': 'common_video/common_video_unittests.isolate',
50    'metrics_unittests': 'test/metrics_unittests.isolate',
51    'modules_tests': 'modules/modules_tests.isolate',
52    'modules_unittests': 'modules/modules_unittests.isolate',
53    'neteq_unittests': 'modules/audio_coding/neteq/neteq_unittests.isolate',
54    'system_wrappers_unittests':
55      'system_wrappers/source/system_wrappers_unittests.isolate',
56    'test_support_unittests': 'test/test_support_unittests.isolate',
57    'tools_unittests': 'tools/tools_unittests.isolate',
58    'video_engine_core_unittests':
59      'video_engine/video_engine_core_unittests.isolate',
60    'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
61}
62
63# Append the WebRTC tests with the full path from Chromium's src/ root.
64for test, isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items():
65  _ISOLATE_FILE_PATHS[test] = 'third_party/webrtc/%s' % isolate_path
66
67# Used for filtering large data deps at a finer grain than what's allowed in
68# isolate files since pushing deps to devices is expensive.
69# Wildcards are allowed.
70_DEPS_EXCLUSION_LIST = [
71    'chrome/test/data/extensions/api_test',
72    'chrome/test/data/extensions/secure_shell',
73    'chrome/test/data/firefox*',
74    'chrome/test/data/gpu',
75    'chrome/test/data/image_decoding',
76    'chrome/test/data/import',
77    'chrome/test/data/page_cycler',
78    'chrome/test/data/perf',
79    'chrome/test/data/pyauto_private',
80    'chrome/test/data/safari_import',
81    'chrome/test/data/scroll',
82    'chrome/test/data/third_party',
83    'third_party/hunspell_dictionaries/*.dic',
84    # crbug.com/258690
85    'webkit/data/bmp_decoder',
86    'webkit/data/ico_decoder',
87]
88
89_ISOLATE_SCRIPT = os.path.join(
90    constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
91
92
93def _GenerateDepsDirUsingIsolate(suite_name):
94  """Generate the dependency dir for the test suite using isolate.
95
96  Args:
97    suite_name: Name of the test suite (e.g. base_unittests).
98  """
99  if os.path.isdir(constants.ISOLATE_DEPS_DIR):
100    shutil.rmtree(constants.ISOLATE_DEPS_DIR)
101
102  isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
103  if not isolate_rel_path:
104    logging.info('Did not find an isolate file for the test suite.')
105    return
106
107  isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
108  isolated_abs_path = os.path.join(
109      constants.GetOutDirectory(), '%s.isolated' % suite_name)
110  assert os.path.exists(isolate_abs_path)
111  isolate_cmd = [
112      'python', _ISOLATE_SCRIPT,
113      'remap',
114      '--isolate', isolate_abs_path,
115      '--isolated', isolated_abs_path,
116      '-V', 'PRODUCT_DIR=%s' % constants.GetOutDirectory(),
117      '-V', 'OS=android',
118      '--outdir', constants.ISOLATE_DEPS_DIR,
119  ]
120  assert not cmd_helper.RunCmd(isolate_cmd)
121
122  # We're relying on the fact that timestamps are preserved
123  # by the remap command (hardlinked). Otherwise, all the data
124  # will be pushed to the device once we move to using time diff
125  # instead of md5sum. Perform a sanity check here.
126  for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
127    if filenames:
128      linked_file = os.path.join(root, filenames[0])
129      orig_file = os.path.join(
130          constants.DIR_SOURCE_ROOT,
131          os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
132      if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
133        break
134      else:
135        raise Exception('isolate remap command did not use hardlinks.')
136
137  # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
138  old_cwd = os.getcwd()
139  try:
140    os.chdir(constants.ISOLATE_DEPS_DIR)
141    excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
142    if excluded_paths:
143      logging.info('Excluding the following from dependency list: %s',
144                   excluded_paths)
145    for p in excluded_paths:
146      if os.path.isdir(p):
147        shutil.rmtree(p)
148      else:
149        os.remove(p)
150  finally:
151    os.chdir(old_cwd)
152
153  # On Android, all pak files need to be in the top-level 'paks' directory.
154  paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
155  os.mkdir(paks_dir)
156  for root, _, filenames in os.walk(os.path.join(constants.ISOLATE_DEPS_DIR,
157                                                 'out')):
158    for filename in fnmatch.filter(filenames, '*.pak'):
159      shutil.move(os.path.join(root, filename), paks_dir)
160
161  # Move everything in PRODUCT_DIR to top level.
162  deps_product_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'out',
163      constants.GetBuildType())
164  if os.path.isdir(deps_product_dir):
165    for p in os.listdir(deps_product_dir):
166      shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
167    os.rmdir(deps_product_dir)
168    os.rmdir(os.path.join(constants.ISOLATE_DEPS_DIR, 'out'))
169
170
171def _GetDisabledTestsFilterFromFile(suite_name):
172  """Returns a gtest filter based on the *_disabled file.
173
174  Args:
175    suite_name: Name of the test suite (e.g. base_unittests).
176
177  Returns:
178    A gtest filter which excludes disabled tests.
179    Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
180  """
181  filter_file_path = os.path.join(
182      os.path.abspath(os.path.dirname(__file__)),
183      'filter', '%s_disabled' % suite_name)
184
185  if not filter_file_path or not os.path.exists(filter_file_path):
186    logging.info('No filter file found at %s', filter_file_path)
187    return '*'
188
189  filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
190             if x and x[0] != '#']
191  disabled_filter = '*-%s' % ':'.join(filters)
192  logging.info('Applying filter "%s" obtained from %s',
193               disabled_filter, filter_file_path)
194  return disabled_filter
195
196
197def _GetTestsFromDevice(runner_factory, devices):
198  """Get a list of tests from a device.
199
200  Args:
201    runner_factory: Callable that takes device and shard_index and returns
202        a TestRunner.
203    devices: A list of device ids.
204
205  Returns:
206    All the tests in the test suite.
207  """
208  for device in devices:
209    try:
210      logging.info('Obtaining tests from %s', device)
211      return runner_factory(device, 0).GetAllTests()
212    except (android_commands.errors.WaitForResponseTimedOutError,
213            android_commands.errors.DeviceUnresponsiveError), e:
214      logging.warning('Failed obtaining test list from %s with exception: %s',
215                      device, e)
216  raise Exception('Failed to obtain test list from devices.')
217
218
219def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
220  """Removes tests with disabled prefixes.
221
222  Args:
223    all_tests: List of tests to filter.
224    pre: If True, include tests with PRE_ prefix.
225    manual: If True, include tests with MANUAL_ prefix.
226
227  Returns:
228    List of tests remaining.
229  """
230  filtered_tests = []
231  filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
232
233  if not pre:
234    filter_prefixes.append('PRE_')
235
236  if not manual:
237    filter_prefixes.append('MANUAL_')
238
239  for t in all_tests:
240    test_case, test = t.split('.', 1)
241    if not any([test_case.startswith(prefix) or test.startswith(prefix) for
242                prefix in filter_prefixes]):
243      filtered_tests.append(t)
244  return filtered_tests
245
246
247def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
248  """Removes disabled tests from |tests|.
249
250  Applies the following filters in order:
251    1. Remove tests with disabled prefixes.
252    2. Remove tests specified in the *_disabled files in the 'filter' dir
253
254  Args:
255    tests: List of tests.
256    suite_name: Name of the test suite (e.g. base_unittests).
257    has_gtest_filter: Whether a gtest_filter is provided.
258
259  Returns:
260    List of tests remaining.
261  """
262  tests = _FilterTestsUsingPrefixes(
263      tests, has_gtest_filter, has_gtest_filter)
264  tests = unittest_util.FilterTestNames(
265      tests, _GetDisabledTestsFilterFromFile(suite_name))
266
267  return tests
268
269
270def Setup(test_options, devices):
271  """Create the test runner factory and tests.
272
273  Args:
274    test_options: A GTestOptions object.
275    devices: A list of attached devices.
276
277  Returns:
278    A tuple of (TestRunnerFactory, tests).
279  """
280
281  if not ports.ResetTestServerPortAllocation():
282    raise Exception('Failed to reset test server port.')
283
284  test_package = test_package_apk.TestPackageApk(test_options.suite_name)
285  if not os.path.exists(test_package.suite_path):
286    test_package = test_package_exe.TestPackageExecutable(
287        test_options.suite_name)
288    if not os.path.exists(test_package.suite_path):
289      raise Exception(
290          'Did not find %s target. Ensure it has been built.'
291          % test_options.suite_name)
292  logging.warning('Found target %s', test_package.suite_path)
293
294  _GenerateDepsDirUsingIsolate(test_options.suite_name)
295
296  # Constructs a new TestRunner with the current options.
297  def TestRunnerFactory(device, shard_index):
298    return test_runner.TestRunner(
299        test_options,
300        device,
301        test_package)
302
303  tests = _GetTestsFromDevice(TestRunnerFactory, devices)
304  if test_options.run_disabled:
305    test_options = test_options._replace(
306        test_arguments=('%s --gtest_also_run_disabled_tests' %
307                        test_options.test_arguments))
308  else:
309    tests = _FilterDisabledTests(tests, test_options.suite_name,
310                                 bool(test_options.gtest_filter))
311  if test_options.gtest_filter:
312    tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
313
314  # Coalesce unit tests into a single test per device
315  if test_options.suite_name != 'content_browsertests':
316    num_devices = len(devices)
317    tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
318    tests = [t for t in tests if t]
319
320  return (TestRunnerFactory, tests)
321