1#!/usr/bin/env python
2# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import optparse
7import os
8import subprocess
9import sys
10import time
11
12import build_projects
13import build_version
14import buildbot_common
15import parse_dsc
16
17from build_paths import OUT_DIR, SRC_DIR, SDK_SRC_DIR, SCRIPT_DIR
18
19sys.path.append(os.path.join(SDK_SRC_DIR, 'tools'))
20import getos
21platform = getos.GetPlatform()
22
23# TODO(binji): ugly hack -- can I get the browser in a cleaner way?
24sys.path.append(os.path.join(SRC_DIR, 'chrome', 'test', 'nacl_test_injection'))
25import find_chrome
26browser_path = find_chrome.FindChrome(SRC_DIR, ['Debug', 'Release'])
27
28
29pepper_ver = str(int(build_version.ChromeMajorVersion()))
30pepperdir = os.path.join(OUT_DIR, 'pepper_' + pepper_ver)
31
32browser_tester_py = os.path.join(SRC_DIR, 'ppapi', 'native_client', 'tools',
33    'browser_tester', 'browser_tester.py')
34
35
36ALL_CONFIGS = ['Debug', 'Release']
37ALL_TOOLCHAINS = ['newlib', 'glibc', 'pnacl', 'win', 'linux', 'mac']
38
39# Values you can filter by:
40#   name: The name of the test. (e.g. "pi_generator")
41#   config: See ALL_CONFIGS above.
42#   toolchain: See ALL_TOOLCHAINS above.
43#   platform: mac/win/linux.
44#
45# All keys must be matched, but any value that matches in a sequence is
46# considered a match for that key. For example:
47#
48#   {'name': ('pi_generator', 'input_event'), 'toolchain': ('newlib', 'pnacl')}
49#
50# Will match 8 tests:
51#   pi_generator.newlib_debug_test
52#   pi_generator.newlib_release_test
53#   input_event.newlib_debug_test
54#   input_event.newlib_release_test
55#   pi_generator.glibc_debug_test
56#   pi_generator.glibc_release_test
57#   input_event.glibc_debug_test
58#   input_event.glibc_release_test
59DISABLED_TESTS = [
60    # TODO(binji): Disable 3D examples on linux/win/mac. See
61    # http://crbug.com/262379.
62    {'name': 'graphics_3d', 'platform': ('win', 'linux', 'mac')},
63    {'name': 'video_decode', 'platform': ('win', 'linux', 'mac')},
64    # media_stream_audio uses audio input devices which are not supported.
65    {'name': 'media_stream_audio', 'platform': ('win', 'linux', 'mac')},
66    # media_stream_video uses 3D and webcam which are not supported.
67    {'name': 'media_stream_video', 'platform': ('win', 'linux', 'mac')},
68    # TODO(binji): These tests timeout on the trybots because the NEXEs take
69    # more than 40 seconds to load (!). See http://crbug.com/280753
70    {'name': 'nacl_io_test', 'platform': 'win', 'toolchain': 'glibc'},
71    # We don't test "getting_started/part1" because it would complicate the
72    # example.
73    # TODO(binji): figure out a way to inject the testing code without
74    # modifying the example; maybe an extension?
75    {'name': 'part1'},
76]
77
78def ValidateToolchains(toolchains):
79  invalid_toolchains = set(toolchains) - set(ALL_TOOLCHAINS)
80  if invalid_toolchains:
81    buildbot_common.ErrorExit('Invalid toolchain(s): %s' % (
82        ', '.join(invalid_toolchains)))
83
84
85def GetServingDirForProject(desc):
86  dest = desc['DEST']
87  path = os.path.join(pepperdir, *dest.split('/'))
88  return os.path.join(path, desc['NAME'])
89
90
91def GetRepoServingDirForProject(desc):
92  # This differs from GetServingDirForProject, because it returns the location
93  # within the Chrome repository of the project, not the "pepperdir".
94  return os.path.dirname(desc['FILEPATH'])
95
96
97def GetExecutableDirForProject(desc, toolchain, config):
98  return os.path.join(GetServingDirForProject(desc), toolchain, config)
99
100
101def GetBrowserTesterCommand(desc, toolchain, config):
102  if browser_path is None:
103    buildbot_common.ErrorExit('Failed to find chrome browser using FindChrome.')
104
105  args = [
106    sys.executable,
107    browser_tester_py,
108    '--browser_path', browser_path,
109    '--timeout', '30.0',  # seconds
110    # Prevent the infobar that shows up when requesting filesystem quota.
111    '--browser_flag', '--unlimited-storage',
112    '--enable_sockets',
113    # Prevent installing a new copy of PNaCl.
114    '--browser_flag', '--disable-component-update',
115  ]
116
117  args.extend(['--serving_dir', GetServingDirForProject(desc)])
118  # Fall back on the example directory in the Chromium repo, to find test.js.
119  args.extend(['--serving_dir', GetRepoServingDirForProject(desc)])
120  # If it is not found there, fall back on the dummy one (in this directory.)
121  args.extend(['--serving_dir', SCRIPT_DIR])
122
123  if toolchain == platform:
124    exe_dir = GetExecutableDirForProject(desc, toolchain, config)
125    ppapi_plugin = os.path.join(exe_dir, desc['NAME'])
126    if platform == 'win':
127      ppapi_plugin += '.dll'
128    else:
129      ppapi_plugin += '.so'
130    args.extend(['--ppapi_plugin', ppapi_plugin])
131
132    ppapi_plugin_mimetype = 'application/x-ppapi-%s' % config.lower()
133    args.extend(['--ppapi_plugin_mimetype', ppapi_plugin_mimetype])
134
135  if toolchain == 'pnacl':
136    args.extend(['--browser_flag', '--enable-pnacl'])
137
138  url = 'index.html'
139  url += '?tc=%s&config=%s&test=true' % (toolchain, config)
140  args.extend(['--url', url])
141  return args
142
143
144def GetBrowserTesterEnv():
145  # browser_tester imports tools/valgrind/memcheck_analyze, which imports
146  # tools/valgrind/common. Well, it tries to, anyway, but instead imports
147  # common from PYTHONPATH first (which on the buildbots, is a
148  # common/__init__.py file...).
149  #
150  # Clear the PYTHONPATH so it imports the correct file.
151  env = dict(os.environ)
152  env['PYTHONPATH'] = ''
153  return env
154
155
156def RunTestOnce(desc, toolchain, config):
157  args = GetBrowserTesterCommand(desc, toolchain, config)
158  env = GetBrowserTesterEnv()
159  start_time = time.time()
160  try:
161    subprocess.check_call(args, env=env)
162    result = True
163  except subprocess.CalledProcessError:
164    result = False
165  elapsed = (time.time() - start_time) * 1000
166  return result, elapsed
167
168
169def RunTestNTimes(desc, toolchain, config, times):
170  total_elapsed = 0
171  for _ in xrange(times):
172    result, elapsed = RunTestOnce(desc, toolchain, config)
173    total_elapsed += elapsed
174    if result:
175      # Success, stop retrying.
176      break
177  return result, total_elapsed
178
179
180def RunTestWithGtestOutput(desc, toolchain, config, retry_on_failure_times):
181  test_name = GetTestName(desc, toolchain, config)
182  WriteGtestHeader(test_name)
183  result, elapsed = RunTestNTimes(desc, toolchain, config,
184                                  retry_on_failure_times)
185  WriteGtestFooter(result, test_name, elapsed)
186  return result
187
188
189def WriteGtestHeader(test_name):
190  print '\n[ RUN      ] %s' % test_name
191  sys.stdout.flush()
192  sys.stderr.flush()
193
194
195def WriteGtestFooter(success, test_name, elapsed):
196  sys.stdout.flush()
197  sys.stderr.flush()
198  if success:
199    message = '[       OK ]'
200  else:
201    message = '[  FAILED  ]'
202  print '%s %s (%d ms)' % (message, test_name, elapsed)
203
204
205def GetTestName(desc, toolchain, config):
206  return '%s.%s_%s_test' % (desc['NAME'], toolchain, config.lower())
207
208
209def IsTestDisabled(desc, toolchain, config):
210  def AsList(value):
211    if type(value) not in (list, tuple):
212      return [value]
213    return value
214
215  def TestMatchesDisabled(test_values, disabled_test):
216    for key in test_values:
217      if key in disabled_test:
218        if test_values[key] not in AsList(disabled_test[key]):
219          return False
220    return True
221
222  test_values = {
223      'name': desc['NAME'],
224      'toolchain': toolchain,
225      'config': config,
226      'platform': platform
227  }
228
229  for disabled_test in DISABLED_TESTS:
230    if TestMatchesDisabled(test_values, disabled_test):
231      return True
232  return False
233
234
235def WriteHorizontalBar():
236  print '-' * 80
237
238
239def WriteBanner(message):
240  WriteHorizontalBar()
241  print message
242  WriteHorizontalBar()
243
244
245def RunAllTestsInTree(tree, toolchains, configs, retry_on_failure_times):
246  tests_run = 0
247  total_tests = 0
248  failed = []
249  disabled = []
250
251  for _, desc in parse_dsc.GenerateProjects(tree):
252    desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
253    valid_toolchains = set(toolchains) & set(desc['TOOLS'])
254    valid_configs = set(configs) & set(desc_configs)
255    for toolchain in sorted(valid_toolchains):
256      for config in sorted(valid_configs):
257        test_name = GetTestName(desc, toolchain, config)
258        total_tests += 1
259        if IsTestDisabled(desc, toolchain, config):
260          disabled.append(test_name)
261          continue
262
263        tests_run += 1
264        success = RunTestWithGtestOutput(desc, toolchain, config,
265                                         retry_on_failure_times)
266        if not success:
267          failed.append(test_name)
268
269  if failed:
270    WriteBanner('FAILED TESTS')
271    for test in failed:
272      print '  %s failed.' % test
273
274  if disabled:
275    WriteBanner('DISABLED TESTS')
276    for test in disabled:
277      print '  %s disabled.' % test
278
279  WriteHorizontalBar()
280  print 'Tests run: %d/%d (%d disabled).' % (
281      tests_run, total_tests, len(disabled))
282  print 'Tests succeeded: %d/%d.' % (tests_run - len(failed), tests_run)
283
284  success = len(failed) != 0
285  return success
286
287
288def BuildAllTestsInTree(tree, toolchains, configs):
289  for branch, desc in parse_dsc.GenerateProjects(tree):
290    desc_configs = desc.get('CONFIGS', ALL_CONFIGS)
291    valid_toolchains = set(toolchains) & set(desc['TOOLS'])
292    valid_configs = set(configs) & set(desc_configs)
293    for toolchain in sorted(valid_toolchains):
294      for config in sorted(valid_configs):
295        name = '%s/%s' % (branch, desc['NAME'])
296        build_projects.BuildProjectsBranch(pepperdir, name, deps=False,
297                                           clean=False, config=config,
298                                           args=['TOOLCHAIN=%s' % toolchain])
299
300
301def GetProjectTree(include):
302  # Everything in src is a library, and cannot be run.
303  exclude = {'DEST': 'src'}
304  try:
305    return parse_dsc.LoadProjectTree(SDK_SRC_DIR, include=include,
306                                     exclude=exclude)
307  except parse_dsc.ValidationError as e:
308    buildbot_common.ErrorExit(str(e))
309
310
311def main(args):
312  parser = optparse.OptionParser()
313  parser.add_option('-c', '--config',
314      help='Choose configuration to run (Debug or Release).  Runs both '
315           'by default', action='append')
316  parser.add_option('-x', '--experimental',
317      help='Run experimental projects', action='store_true')
318  parser.add_option('-t', '--toolchain',
319      help='Run using toolchain. Can be passed more than once.',
320      action='append', default=[])
321  parser.add_option('-d', '--dest',
322      help='Select which destinations (project types) are valid.',
323      action='append')
324  parser.add_option('-b', '--build',
325      help='Build each project before testing.', action='store_true')
326  parser.add_option('--retry-times',
327      help='Number of types to retry on failure (Default: %default)',
328          type='int', default=1)
329
330  options, args = parser.parse_args(args[1:])
331
332  if not options.toolchain:
333    options.toolchain = ['newlib', 'glibc', 'pnacl', 'host']
334
335  if 'host' in options.toolchain:
336    options.toolchain.remove('host')
337    options.toolchain.append(platform)
338    print 'Adding platform: ' + platform
339
340  ValidateToolchains(options.toolchain)
341
342  include = {}
343  if options.toolchain:
344    include['TOOLS'] = options.toolchain
345    print 'Filter by toolchain: ' + str(options.toolchain)
346  if not options.experimental:
347    include['EXPERIMENTAL'] = False
348  if options.dest:
349    include['DEST'] = options.dest
350    print 'Filter by type: ' + str(options.dest)
351  if args:
352    include['NAME'] = args
353    print 'Filter by name: ' + str(args)
354  if not options.config:
355    options.config = ALL_CONFIGS
356
357  project_tree = GetProjectTree(include)
358  if options.build:
359    BuildAllTestsInTree(project_tree, options.toolchain, options.config)
360
361  return RunAllTestsInTree(project_tree, options.toolchain, options.config,
362                           options.retry_times)
363
364
365if __name__ == '__main__':
366  script_name = os.path.basename(sys.argv[0])
367  try:
368    sys.exit(main(sys.argv))
369  except parse_dsc.ValidationError as e:
370    buildbot_common.ErrorExit('%s: %s' % (script_name, e))
371  except KeyboardInterrupt:
372    buildbot_common.ErrorExit('%s: interrupted' % script_name)
373