setup.py revision ab8f6f0bd665d3c1ff476eb06c58c42630e462d4 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Generates test runner factory and tests for GTests."""
6# pylint: disable=W0212
7
8import fnmatch
9import glob
10import logging
11import os
12import shutil
13import sys
14
15from pylib import cmd_helper
16from pylib import constants
17
18from pylib.base import base_test_result
19from pylib.base import test_dispatcher
20from pylib.gtest import test_package_apk
21from pylib.gtest import test_package_exe
22from pylib.gtest import test_runner
23
24sys.path.insert(0,
25                os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
26                             'common'))
27import unittest_util # pylint: disable=F0401
28
29
30_ISOLATE_FILE_PATHS = {
31    'base_unittests': 'base/base_unittests.isolate',
32    'blink_heap_unittests':
33      'third_party/WebKit/Source/platform/heap/BlinkHeapUnitTests.isolate',
34    'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
35    'cc_perftests': 'cc/cc_perftests.isolate',
36    'components_unittests': 'components/components_unittests.isolate',
37    'content_browsertests': 'content/content_browsertests.isolate',
38    'content_unittests': 'content/content_unittests.isolate',
39    'media_perftests': 'media/media_perftests.isolate',
40    'media_unittests': 'media/media_unittests.isolate',
41    'net_unittests': 'net/net_unittests.isolate',
42    'sql_unittests': 'sql/sql_unittests.isolate',
43    'ui_unittests': 'ui/base/ui_base_tests.isolate',
44    'unit_tests': 'chrome/unit_tests.isolate',
45    'webkit_unit_tests':
46      'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
47}
48
49# Used for filtering large data deps at a finer grain than what's allowed in
50# isolate files since pushing deps to devices is expensive.
51# Wildcards are allowed.
52_DEPS_EXCLUSION_LIST = [
53    'chrome/test/data/extensions/api_test',
54    'chrome/test/data/extensions/secure_shell',
55    'chrome/test/data/firefox*',
56    'chrome/test/data/gpu',
57    'chrome/test/data/image_decoding',
58    'chrome/test/data/import',
59    'chrome/test/data/page_cycler',
60    'chrome/test/data/perf',
61    'chrome/test/data/pyauto_private',
62    'chrome/test/data/safari_import',
63    'chrome/test/data/scroll',
64    'chrome/test/data/third_party',
65    'third_party/hunspell_dictionaries/*.dic',
66    # crbug.com/258690
67    'webkit/data/bmp_decoder',
68    'webkit/data/ico_decoder',
69]
70
71_ISOLATE_SCRIPT = os.path.join(
72    constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
73
74
75def _GenerateDepsDirUsingIsolate(suite_name, isolate_file_path=None):
76  """Generate the dependency dir for the test suite using isolate.
77
78  Args:
79    suite_name: Name of the test suite (e.g. base_unittests).
80    isolate_file_path: .isolate file path to use. If there is a default .isolate
81                       file path for the suite_name, this will override it.
82  """
83  if os.path.isdir(constants.ISOLATE_DEPS_DIR):
84    shutil.rmtree(constants.ISOLATE_DEPS_DIR)
85
86  if isolate_file_path:
87    if os.path.isabs(isolate_file_path):
88      isolate_abs_path = isolate_file_path
89    else:
90      isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT,
91                                      isolate_file_path)
92  else:
93    isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
94    if not isolate_rel_path:
95      logging.info('Did not find an isolate file for the test suite.')
96      return
97    isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
98
99  isolated_abs_path = os.path.join(
100      constants.GetOutDirectory(), '%s.isolated' % suite_name)
101  assert os.path.exists(isolate_abs_path), 'Cannot find %s' % isolate_abs_path
102  # This needs to be kept in sync with the cmd line options for isolate.py
103  # in src/build/isolate.gypi.
104  isolate_cmd = [
105      'python', _ISOLATE_SCRIPT,
106      'remap',
107      '--isolate', isolate_abs_path,
108      '--isolated', isolated_abs_path,
109      '--outdir', constants.ISOLATE_DEPS_DIR,
110
111      '--path-variable', 'DEPTH', constants.DIR_SOURCE_ROOT,
112      '--path-variable', 'PRODUCT_DIR', constants.GetOutDirectory(),
113
114      '--config-variable', 'OS', 'android',
115      '--config-variable', 'CONFIGURATION_NAME', constants.GetBuildType(),
116      '--config-variable', 'asan', '0',
117      '--config-variable', 'chromeos', '0',
118      '--config-variable', 'component', 'static_library',
119      '--config-variable', 'fastbuild', '0',
120      '--config-variable', 'icu_use_data_file_flag', '1',
121      '--config-variable', 'libpeer_target_type', 'static_library',
122      # TODO(maruel): This may not be always true.
123      '--config-variable', 'target_arch', 'arm',
124      '--config-variable', 'use_openssl', '0',
125      '--config-variable', 'use_ozone', '0',
126  ]
127  assert not cmd_helper.RunCmd(isolate_cmd)
128
129  # We're relying on the fact that timestamps are preserved
130  # by the remap command (hardlinked). Otherwise, all the data
131  # will be pushed to the device once we move to using time diff
132  # instead of md5sum. Perform a sanity check here.
133  for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
134    if filenames:
135      linked_file = os.path.join(root, filenames[0])
136      orig_file = os.path.join(
137          constants.DIR_SOURCE_ROOT,
138          os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
139      if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
140        break
141      else:
142        raise Exception('isolate remap command did not use hardlinks.')
143
144  # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
145  old_cwd = os.getcwd()
146  try:
147    os.chdir(constants.ISOLATE_DEPS_DIR)
148    excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
149    if excluded_paths:
150      logging.info('Excluding the following from dependency list: %s',
151                   excluded_paths)
152    for p in excluded_paths:
153      if os.path.isdir(p):
154        shutil.rmtree(p)
155      else:
156        os.remove(p)
157  finally:
158    os.chdir(old_cwd)
159
160  # On Android, all pak files need to be in the top-level 'paks' directory.
161  paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
162  os.mkdir(paks_dir)
163
164  deps_out_dir = os.path.join(
165      constants.ISOLATE_DEPS_DIR,
166      os.path.relpath(os.path.join(constants.GetOutDirectory(), os.pardir),
167                      constants.DIR_SOURCE_ROOT))
168  for root, _, filenames in os.walk(deps_out_dir):
169    for filename in fnmatch.filter(filenames, '*.pak'):
170      shutil.move(os.path.join(root, filename), paks_dir)
171
172  # Move everything in PRODUCT_DIR to top level.
173  deps_product_dir = os.path.join(deps_out_dir, constants.GetBuildType())
174  if os.path.isdir(deps_product_dir):
175    for p in os.listdir(deps_product_dir):
176      shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
177    os.rmdir(deps_product_dir)
178    os.rmdir(deps_out_dir)
179
180
181def _GetDisabledTestsFilterFromFile(suite_name):
182  """Returns a gtest filter based on the *_disabled file.
183
184  Args:
185    suite_name: Name of the test suite (e.g. base_unittests).
186
187  Returns:
188    A gtest filter which excludes disabled tests.
189    Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
190  """
191  filter_file_path = os.path.join(
192      os.path.abspath(os.path.dirname(__file__)),
193      'filter', '%s_disabled' % suite_name)
194
195  if not filter_file_path or not os.path.exists(filter_file_path):
196    logging.info('No filter file found at %s', filter_file_path)
197    return '*'
198
199  filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
200             if x and x[0] != '#']
201  disabled_filter = '*-%s' % ':'.join(filters)
202  logging.info('Applying filter "%s" obtained from %s',
203               disabled_filter, filter_file_path)
204  return disabled_filter
205
206
207def _GetTests(test_options, test_package, devices):
208  """Get a list of tests.
209
210  Args:
211    test_options: A GTestOptions object.
212    test_package: A TestPackageApk object.
213    devices: A list of attached devices.
214
215  Returns:
216    A list of all the tests in the test suite.
217  """
218  def TestListerRunnerFactory(device, _shard_index):
219    class TestListerRunner(test_runner.TestRunner):
220      #override
221      def PushDataDeps(self):
222        pass
223
224      #override
225      def RunTest(self, _test):
226        result = base_test_result.BaseTestResult(
227            'gtest_list_tests', base_test_result.ResultType.PASS)
228        self.test_package.Install(self.device)
229        result.test_list = self.test_package.GetAllTests(self.device)
230        results = base_test_result.TestRunResults()
231        results.AddResult(result)
232        return results, None
233    return TestListerRunner(test_options, device, test_package)
234
235  results, _no_retry = test_dispatcher.RunTests(
236      ['gtest_list_tests'], TestListerRunnerFactory, devices)
237  tests = []
238  for r in results.GetAll():
239    tests.extend(r.test_list)
240  return tests
241
242
243def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
244  """Removes tests with disabled prefixes.
245
246  Args:
247    all_tests: List of tests to filter.
248    pre: If True, include tests with PRE_ prefix.
249    manual: If True, include tests with MANUAL_ prefix.
250
251  Returns:
252    List of tests remaining.
253  """
254  filtered_tests = []
255  filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
256
257  if not pre:
258    filter_prefixes.append('PRE_')
259
260  if not manual:
261    filter_prefixes.append('MANUAL_')
262
263  for t in all_tests:
264    test_case, test = t.split('.', 1)
265    if not any([test_case.startswith(prefix) or test.startswith(prefix) for
266                prefix in filter_prefixes]):
267      filtered_tests.append(t)
268  return filtered_tests
269
270
271def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
272  """Removes disabled tests from |tests|.
273
274  Applies the following filters in order:
275    1. Remove tests with disabled prefixes.
276    2. Remove tests specified in the *_disabled files in the 'filter' dir
277
278  Args:
279    tests: List of tests.
280    suite_name: Name of the test suite (e.g. base_unittests).
281    has_gtest_filter: Whether a gtest_filter is provided.
282
283  Returns:
284    List of tests remaining.
285  """
286  tests = _FilterTestsUsingPrefixes(
287      tests, has_gtest_filter, has_gtest_filter)
288  tests = unittest_util.FilterTestNames(
289      tests, _GetDisabledTestsFilterFromFile(suite_name))
290
291  return tests
292
293
294def Setup(test_options, devices):
295  """Create the test runner factory and tests.
296
297  Args:
298    test_options: A GTestOptions object.
299    devices: A list of attached devices.
300
301  Returns:
302    A tuple of (TestRunnerFactory, tests).
303  """
304  test_package = test_package_apk.TestPackageApk(test_options.suite_name)
305  if not os.path.exists(test_package.suite_path):
306    exe_test_package = test_package_exe.TestPackageExecutable(
307        test_options.suite_name)
308    if not os.path.exists(exe_test_package.suite_path):
309      raise Exception(
310          'Did not find %s target. Ensure it has been built.\n'
311          '(not found at %s or %s)'
312          % (test_options.suite_name,
313             test_package.suite_path,
314             exe_test_package.suite_path))
315    test_package = exe_test_package
316  logging.warning('Found target %s', test_package.suite_path)
317
318  _GenerateDepsDirUsingIsolate(test_options.suite_name,
319                               test_options.isolate_file_path)
320
321  tests = _GetTests(test_options, test_package, devices)
322
323  # Constructs a new TestRunner with the current options.
324  def TestRunnerFactory(device, _shard_index):
325    return test_runner.TestRunner(
326        test_options,
327        device,
328        test_package)
329
330  if test_options.run_disabled:
331    test_options = test_options._replace(
332        test_arguments=('%s --gtest_also_run_disabled_tests' %
333                        test_options.test_arguments))
334  else:
335    tests = _FilterDisabledTests(tests, test_options.suite_name,
336                                 bool(test_options.gtest_filter))
337  if test_options.gtest_filter:
338    tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
339
340  # Coalesce unit tests into a single test per device
341  if test_options.suite_name != 'content_browsertests':
342    num_devices = len(devices)
343    tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
344    tests = [t for t in tests if t]
345
346  return (TestRunnerFactory, tests)
347