1#!/usr/bin/env python
2# Copyright 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Runs the WebDriver Java acceptance tests.
7
8This script is called from chrome/test/chromedriver/run_all_tests.py and reports
9results using the buildbot annotation scheme.
10
11For ChromeDriver documentation, refer to http://code.google.com/p/chromedriver.
12"""
13
14import optparse
15import os
16import shutil
17import sys
18import xml.dom.minidom as minidom
19
20_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
21sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir))
22
23import chrome_paths
24import test_environment
25import util
26
27if util.IsLinux():
28  sys.path.insert(0, os.path.join(chrome_paths.GetSrc(), 'build', 'android'))
29  from pylib import constants
30
31
32class TestResult(object):
33  """A result for an attempted single test case."""
34
35  def __init__(self, name, time, failure):
36    """Initializes a test result.
37
38    Args:
39      name: the full name of the test.
40      time: the amount of time the test ran, in seconds.
41      failure: the test error or failure message, or None if the test passed.
42    """
43    self._name = name
44    self._time = time
45    self._failure = failure
46
47  def GetName(self):
48    """Returns the test name."""
49    return self._name
50
51  def GetTime(self):
52    """Returns the time it took to run the test."""
53    return self._time
54
55  def IsPass(self):
56    """Returns whether the test passed."""
57    return self._failure is None
58
59  def GetFailureMessage(self):
60    """Returns the test failure message, or None if the test passed."""
61    return self._failure
62
63
64def _Run(java_tests_src_dir, test_filter,
65         chromedriver_path, chrome_path, log_path, android_package_key,
66         verbose, debug):
67  """Run the WebDriver Java tests and return the test results.
68
69  Args:
70    java_tests_src_dir: the java test source code directory.
71    test_filter: the filter to use when choosing tests to run. Format is same
72        as Google C++ Test format.
73    chromedriver_path: path to ChromeDriver exe.
74    chrome_path: path to Chrome exe.
75    log_path: path to server log.
76    android_package_key: name of Chrome's Android package.
77    verbose: whether the output should be verbose.
78    debug: whether the tests should wait until attached by a debugger.
79
80  Returns:
81    A list of |TestResult|s.
82  """
83  test_dir = util.MakeTempDir()
84  keystore_path = ('java', 'client', 'test', 'keystore')
85  required_dirs = [keystore_path[:-1],
86                   ('javascript',),
87                   ('third_party', 'closure', 'goog'),
88                   ('third_party', 'js')]
89  for required_dir in required_dirs:
90    os.makedirs(os.path.join(test_dir, *required_dir))
91
92  test_jar = 'test-standalone.jar'
93  class_path = test_jar
94  shutil.copyfile(os.path.join(java_tests_src_dir, 'keystore'),
95                  os.path.join(test_dir, *keystore_path))
96  util.Unzip(os.path.join(java_tests_src_dir, 'common.zip'), test_dir)
97  shutil.copyfile(os.path.join(java_tests_src_dir, test_jar),
98                  os.path.join(test_dir, test_jar))
99
100  sys_props = ['selenium.browser=chrome',
101               'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)]
102  if chrome_path:
103    sys_props += ['webdriver.chrome.binary=' + os.path.abspath(chrome_path)]
104  if log_path:
105    sys_props += ['webdriver.chrome.logfile=' + log_path]
106  if android_package_key:
107    android_package = constants.PACKAGE_INFO[android_package_key].package
108    sys_props += ['webdriver.chrome.android_package=' + android_package]
109    if android_package_key == 'chromedriver_webview_shell':
110      android_activity = constants.PACKAGE_INFO[android_package_key].activity
111      android_process = '%s:main' % android_package
112      sys_props += ['webdriver.chrome.android_activity=' + android_activity]
113      sys_props += ['webdriver.chrome.android_process=' + android_process]
114  if test_filter:
115    # Test jar actually takes a regex. Convert from glob.
116    test_filter = test_filter.replace('*', '.*')
117    sys_props += ['filter=' + test_filter]
118
119  jvm_args = []
120  if debug:
121    transport = 'dt_socket'
122    if util.IsWindows():
123      transport = 'dt_shmem'
124    jvm_args += ['-agentlib:jdwp=transport=%s,server=y,suspend=y,'
125                 'address=33081' % transport]
126    # Unpack the sources into the test directory and add to the class path
127    # for ease of debugging, particularly with jdb.
128    util.Unzip(os.path.join(java_tests_src_dir, 'test-nodeps-srcs.jar'),
129               test_dir)
130    class_path += ':' + test_dir
131
132  return _RunAntTest(
133      test_dir, 'org.openqa.selenium.chrome.ChromeDriverTests',
134      class_path, sys_props, jvm_args, verbose)
135
136
137def _RunAntTest(test_dir, test_class, class_path, sys_props, jvm_args, verbose):
138  """Runs a single Ant JUnit test suite and returns the |TestResult|s.
139
140  Args:
141    test_dir: the directory to run the tests in.
142    test_class: the name of the JUnit test suite class to run.
143    class_path: the Java class path used when running the tests, colon delimited
144    sys_props: Java system properties to set when running the tests.
145    jvm_args: Java VM command line args to use.
146    verbose: whether the output should be verbose.
147
148  Returns:
149    A list of |TestResult|s.
150  """
151  def _CreateBuildConfig(test_name, results_file, class_path, junit_props,
152                         sys_props, jvm_args):
153    def _SystemPropToXml(prop):
154      key, value = prop.split('=')
155      return '<sysproperty key="%s" value="%s"/>' % (key, value)
156    def _JvmArgToXml(arg):
157      return '<jvmarg value="%s"/>' % arg
158    return '\n'.join([
159        '<project>',
160        '  <target name="test">',
161        '    <junit %s>' % ' '.join(junit_props),
162        '      <formatter type="xml"/>',
163        '      <classpath>',
164        '        <pathelement path="%s"/>' % class_path,
165        '      </classpath>',
166        '      ' + '\n      '.join(map(_SystemPropToXml, sys_props)),
167        '      ' + '\n      '.join(map(_JvmArgToXml, jvm_args)),
168        '      <test name="%s" outfile="%s"/>' % (test_name, results_file),
169        '    </junit>',
170        '  </target>',
171        '</project>'])
172
173  def _ProcessResults(results_path):
174    doc = minidom.parse(results_path)
175    tests = []
176    for test in doc.getElementsByTagName('testcase'):
177      name = test.getAttribute('classname') + '.' + test.getAttribute('name')
178      time = test.getAttribute('time')
179      failure = None
180      error_nodes = test.getElementsByTagName('error')
181      failure_nodes = test.getElementsByTagName('failure')
182      if error_nodes:
183        failure = error_nodes[0].childNodes[0].nodeValue
184      elif failure_nodes:
185        failure = failure_nodes[0].childNodes[0].nodeValue
186      tests += [TestResult(name, time, failure)]
187    return tests
188
189  junit_props = ['printsummary="yes"',
190                 'fork="yes"',
191                 'haltonfailure="no"',
192                 'haltonerror="no"']
193  if verbose:
194    junit_props += ['showoutput="yes"']
195
196  ant_file = open(os.path.join(test_dir, 'build.xml'), 'w')
197  ant_file.write(_CreateBuildConfig(
198      test_class, 'results', class_path, junit_props, sys_props, jvm_args))
199  ant_file.close()
200
201  if util.IsWindows():
202    ant_name = 'ant.bat'
203  else:
204    ant_name = 'ant'
205  code = util.RunCommand([ant_name, 'test'], cwd=test_dir)
206  if code != 0:
207    print 'FAILED to run java tests of %s through ant' % test_class
208    return
209  return _ProcessResults(os.path.join(test_dir, 'results.xml'))
210
211
212def PrintTestResults(results):
213  """Prints the given results in a format recognized by the buildbot."""
214  failures = []
215  failure_names = []
216  for result in results:
217    if not result.IsPass():
218      failures += [result]
219      failure_names += ['.'.join(result.GetName().split('.')[-2:])]
220
221  print 'Ran %s tests' % len(results)
222  print 'Failed %s:' % len(failures)
223  util.AddBuildStepText('failed %s/%s' % (len(failures), len(results)))
224  for result in failures:
225    print '=' * 80
226    print '=' * 10, result.GetName(), '(%ss)' % result.GetTime()
227    print result.GetFailureMessage()
228    if len(failures) < 10:
229      util.AddBuildStepText('.'.join(result.GetName().split('.')[-2:]))
230  print 'Rerun failing tests with filter:', ':'.join(failure_names)
231  return len(failures)
232
233
234def main():
235  parser = optparse.OptionParser()
236  parser.add_option(
237      '', '--verbose', action='store_true', default=False,
238      help='Whether output should be verbose')
239  parser.add_option(
240      '', '--debug', action='store_true', default=False,
241      help='Whether to wait to be attached by a debugger')
242  parser.add_option(
243      '', '--chromedriver', type='string', default=None,
244      help='Path to a build of the chromedriver library(REQUIRED!)')
245  parser.add_option(
246      '', '--chrome', type='string', default=None,
247      help='Path to a build of the chrome binary')
248  parser.add_option(
249      '', '--log-path',
250      help='Output verbose server logs to this file')
251  parser.add_option(
252      '', '--chrome-version', default='HEAD',
253      help='Version of chrome. Default is \'HEAD\'')
254  parser.add_option(
255      '', '--android-package', help='Android package key')
256  parser.add_option(
257      '', '--filter', type='string', default=None,
258      help='Filter for specifying what tests to run, "*" will run all. E.g., '
259           '*testShouldReturnTitleOfPageIfSet')
260  parser.add_option(
261      '', '--also-run-disabled-tests', action='store_true', default=False,
262      help='Include disabled tests while running the tests')
263  parser.add_option(
264      '', '--isolate-tests', action='store_true', default=False,
265      help='Relaunch the jar test harness after each test')
266  options, _ = parser.parse_args()
267
268  options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
269  if options.chromedriver is None or not os.path.exists(options.chromedriver):
270    parser.error('chromedriver is required or the given path is invalid.' +
271                 'Please run "%s --help" for help' % __file__)
272
273  if options.android_package:
274    if options.android_package not in constants.PACKAGE_INFO:
275      parser.error('Invalid --android-package')
276    if options.chrome_version != 'HEAD':
277      parser.error('Android does not support the --chrome-version argument.')
278    environment = test_environment.AndroidTestEnvironment(
279        options.android_package)
280  else:
281    environment = test_environment.DesktopTestEnvironment(
282        options.chrome_version)
283
284  try:
285    environment.GlobalSetUp()
286    # Run passed tests when filter is not provided.
287    if options.isolate_tests:
288      test_filters = environment.GetPassedJavaTests()
289    else:
290      if options.filter:
291        test_filter = options.filter
292      else:
293        test_filter = '*'
294      if not options.also_run_disabled_tests:
295        if '-' in test_filter:
296          test_filter += ':'
297        else:
298          test_filter += '-'
299        test_filter += ':'.join(environment.GetDisabledJavaTestMatchers())
300      test_filters = [test_filter]
301
302    java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test',
303                                      'chromedriver', 'third_party',
304                                      'java_tests')
305    if (not os.path.exists(java_tests_src_dir) or
306        not os.listdir(java_tests_src_dir)):
307      java_tests_url = ('http://src.chromium.org/svn/trunk/deps/third_party'
308                        '/webdriver')
309      print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir +
310             'Need to map <chrome-svn>/trunk/deps/third_party/webdriver to '
311             'chrome/test/chromedriver/third_party/java_tests in .gclient.\n'
312             'Alternatively, do:\n'
313             '  $ cd chrome/test/chromedriver/third_party\n'
314             '  $ svn co %s java_tests' % java_tests_url)
315      return 1
316
317    results = []
318    for filter in test_filters:
319      results += _Run(
320          java_tests_src_dir=java_tests_src_dir,
321          test_filter=filter,
322          chromedriver_path=options.chromedriver,
323          chrome_path=util.GetAbsolutePathOfUserPath(options.chrome),
324          log_path=options.log_path,
325          android_package_key=options.android_package,
326          verbose=options.verbose,
327          debug=options.debug)
328    return PrintTestResults(results)
329  finally:
330    environment.GlobalTearDown()
331
332
333if __name__ == '__main__':
334  sys.exit(main())
335