1#!/usr/bin/env python
2# Copyright 2013 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Runs the WebDriver Java acceptance tests.
7
8This script is called from chrome/test/chromedriver/run_all_tests.py and reports
9results using the buildbot annotation scheme.
10
11For ChromeDriver documentation, refer to http://code.google.com/p/chromedriver.
12"""
13
14import optparse
15import os
16import shutil
17import sys
18import xml.dom.minidom as minidom
19
20_THIS_DIR = os.path.abspath(os.path.dirname(__file__))
21sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir))
22
23import chrome_paths
24import test_environment
25import util
26
27
28class TestResult(object):
29  """A result for an attempted single test case."""
30
31  def __init__(self, name, time, failure):
32    """Initializes a test result.
33
34    Args:
35      name: the full name of the test.
36      time: the amount of time the test ran, in seconds.
37      failure: the test error or failure message, or None if the test passed.
38    """
39    self._name = name
40    self._time = time
41    self._failure = failure
42
43  def GetName(self):
44    """Returns the test name."""
45    return self._name
46
47  def GetTime(self):
48    """Returns the time it took to run the test."""
49    return self._time
50
51  def IsPass(self):
52    """Returns whether the test passed."""
53    return self._failure is None
54
55  def GetFailureMessage(self):
56    """Returns the test failure message, or None if the test passed."""
57    return self._failure
58
59
60def _Run(java_tests_src_dir, test_filter,
61         chromedriver_path, chrome_path, android_package,
62         verbose, debug):
63  """Run the WebDriver Java tests and return the test results.
64
65  Args:
66    java_tests_src_dir: the java test source code directory.
67    test_filter: the filter to use when choosing tests to run. Format is same
68        as Google C++ Test format.
69    chromedriver_path: path to ChromeDriver exe.
70    chrome_path: path to Chrome exe.
71    android_package: name of Chrome's Android package.
72    verbose: whether the output should be verbose.
73    debug: whether the tests should wait until attached by a debugger.
74
75  Returns:
76    A list of |TestResult|s.
77  """
78  test_dir = util.MakeTempDir()
79  keystore_path = ('java', 'client', 'test', 'keystore')
80  required_dirs = [keystore_path[:-1],
81                   ('javascript',),
82                   ('third_party', 'closure', 'goog'),
83                   ('third_party', 'js')]
84  for required_dir in required_dirs:
85    os.makedirs(os.path.join(test_dir, *required_dir))
86
87  test_jar = 'test-standalone.jar'
88  class_path = test_jar
89  shutil.copyfile(os.path.join(java_tests_src_dir, 'keystore'),
90                  os.path.join(test_dir, *keystore_path))
91  util.Unzip(os.path.join(java_tests_src_dir, 'common.zip'), test_dir)
92  shutil.copyfile(os.path.join(java_tests_src_dir, test_jar),
93                  os.path.join(test_dir, test_jar))
94
95  sys_props = ['selenium.browser=chrome',
96               'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)]
97  if chrome_path is not None:
98    sys_props += ['webdriver.chrome.binary=' + os.path.abspath(chrome_path)]
99  if android_package is not None:
100    sys_props += ['webdriver.chrome.android_package=' + android_package]
101  if test_filter:
102    # Test jar actually takes a regex. Convert from glob.
103    test_filter = test_filter.replace('*', '.*')
104    sys_props += ['filter=' + test_filter]
105
106  jvm_args = []
107  if debug:
108    jvm_args += ['-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,'
109                 'address=33081']
110    # Unpack the sources into the test directory and add to the class path
111    # for ease of debugging, particularly with jdb.
112    util.Unzip(os.path.join(java_tests_src_dir, 'test-nodeps-srcs.jar'),
113               test_dir)
114    class_path += ':' + test_dir
115
116  return _RunAntTest(
117      test_dir, 'org.openqa.selenium.chrome.ChromeDriverTests',
118      class_path, sys_props, jvm_args, verbose)
119
120
121def _RunAntTest(test_dir, test_class, class_path, sys_props, jvm_args, verbose):
122  """Runs a single Ant JUnit test suite and returns the |TestResult|s.
123
124  Args:
125    test_dir: the directory to run the tests in.
126    test_class: the name of the JUnit test suite class to run.
127    class_path: the Java class path used when running the tests, colon delimited
128    sys_props: Java system properties to set when running the tests.
129    jvm_args: Java VM command line args to use.
130    verbose: whether the output should be verbose.
131
132  Returns:
133    A list of |TestResult|s.
134  """
135  def _CreateBuildConfig(test_name, results_file, class_path, junit_props,
136                         sys_props, jvm_args):
137    def _SystemPropToXml(prop):
138      key, value = prop.split('=')
139      return '<sysproperty key="%s" value="%s"/>' % (key, value)
140    def _JvmArgToXml(arg):
141      return '<jvmarg value="%s"/>' % arg
142    return '\n'.join([
143        '<project>',
144        '  <target name="test">',
145        '    <junit %s>' % ' '.join(junit_props),
146        '      <formatter type="xml"/>',
147        '      <classpath>',
148        '        <pathelement path="%s"/>' % class_path,
149        '      </classpath>',
150        '      ' + '\n      '.join(map(_SystemPropToXml, sys_props)),
151        '      ' + '\n      '.join(map(_JvmArgToXml, jvm_args)),
152        '      <test name="%s" outfile="%s"/>' % (test_name, results_file),
153        '    </junit>',
154        '  </target>',
155        '</project>'])
156
157  def _ProcessResults(results_path):
158    doc = minidom.parse(results_path)
159    tests = []
160    for test in doc.getElementsByTagName('testcase'):
161      name = test.getAttribute('classname') + '.' + test.getAttribute('name')
162      time = test.getAttribute('time')
163      failure = None
164      error_nodes = test.getElementsByTagName('error')
165      failure_nodes = test.getElementsByTagName('failure')
166      if error_nodes:
167        failure = error_nodes[0].childNodes[0].nodeValue
168      elif failure_nodes:
169        failure = failure_nodes[0].childNodes[0].nodeValue
170      tests += [TestResult(name, time, failure)]
171    return tests
172
173  junit_props = ['printsummary="yes"',
174                 'fork="yes"',
175                 'haltonfailure="no"',
176                 'haltonerror="no"']
177  if verbose:
178    junit_props += ['showoutput="yes"']
179
180  ant_file = open(os.path.join(test_dir, 'build.xml'), 'w')
181  ant_file.write(_CreateBuildConfig(
182      test_class, 'results', class_path, junit_props, sys_props, jvm_args))
183  ant_file.close()
184
185  if util.IsWindows():
186    ant_name = 'ant.bat'
187  else:
188    ant_name = 'ant'
189  code = util.RunCommand([ant_name, 'test'], cwd=test_dir)
190  if code != 0:
191    print 'FAILED to run java tests of %s through ant' % test_class
192    return
193  return _ProcessResults(os.path.join(test_dir, 'results.xml'))
194
195
196def PrintTestResults(results):
197  """Prints the given results in a format recognized by the buildbot."""
198  failures = []
199  failure_names = []
200  for result in results:
201    if not result.IsPass():
202      failures += [result]
203      failure_names += ['.'.join(result.GetName().split('.')[-2:])]
204
205  print 'Ran %s tests' % len(results)
206  print 'Failed %s:' % len(failures)
207  util.AddBuildStepText('failed %s/%s' % (len(failures), len(results)))
208  for result in failures:
209    print '=' * 80
210    print '=' * 10, result.GetName(), '(%ss)' % result.GetTime()
211    print result.GetFailureMessage()
212    if len(failures) < 10:
213      util.AddBuildStepText('.'.join(result.GetName().split('.')[-2:]))
214  print 'Rerun failing tests with filter:', ':'.join(failure_names)
215  return len(failures)
216
217
218def main():
219  parser = optparse.OptionParser()
220  parser.add_option(
221      '', '--verbose', action='store_true', default=False,
222      help='Whether output should be verbose')
223  parser.add_option(
224      '', '--debug', action='store_true', default=False,
225      help='Whether to wait to be attached by a debugger')
226  parser.add_option(
227      '', '--chromedriver', type='string', default=None,
228      help='Path to a build of the chromedriver library(REQUIRED!)')
229  parser.add_option(
230      '', '--chrome', type='string', default=None,
231      help='Path to a build of the chrome binary')
232  parser.add_option(
233      '', '--chrome-version', default='HEAD',
234      help='Version of chrome. Default is \'HEAD\'')
235  parser.add_option(
236      '', '--android-package', type='string', default=None,
237      help='Name of Chrome\'s Android package')
238  parser.add_option(
239      '', '--filter', type='string', default=None,
240      help='Filter for specifying what tests to run, "*" will run all. E.g., '
241           '*testShouldReturnTitleOfPageIfSet')
242  parser.add_option(
243      '', '--also-run-disabled-tests', action='store_true', default=False,
244      help='Include disabled tests while running the tests')
245  parser.add_option(
246      '', '--isolate-tests', action='store_true', default=False,
247      help='Relaunch the jar test harness after each test')
248  options, _ = parser.parse_args()
249
250  if options.chromedriver is None or not os.path.exists(options.chromedriver):
251    parser.error('chromedriver is required or the given path is invalid.' +
252                 'Please run "%s --help" for help' % __file__)
253
254  if options.android_package is not None:
255    if options.chrome_version != 'HEAD':
256      parser.error('Android does not support the --chrome-version argument.')
257    environment = test_environment.AndroidTestEnvironment()
258  else:
259    environment = test_environment.DesktopTestEnvironment(
260        options.chrome_version)
261
262  try:
263    environment.GlobalSetUp()
264    # Run passed tests when filter is not provided.
265    if options.isolate_tests:
266      test_filters = environment.GetPassedJavaTests()
267    else:
268      if options.filter:
269        test_filter = options.filter
270      else:
271        test_filter = '*'
272      if not options.also_run_disabled_tests:
273        if '-' in test_filter:
274          test_filter += ':'
275        else:
276          test_filter += '-'
277        test_filter += ':'.join(environment.GetDisabledJavaTestMatchers())
278      test_filters = [test_filter]
279
280    java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test',
281                                      'chromedriver', 'third_party',
282                                      'java_tests')
283    if (not os.path.exists(java_tests_src_dir) or
284        not os.listdir(java_tests_src_dir)):
285      java_tests_url = ('http://src.chromium.org/svn/trunk/deps/third_party'
286                        '/webdriver')
287      print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir +
288             'Need to map <chrome-svn>/trunk/deps/third_party/webdriver to '
289             'chrome/test/chromedriver/third_party/java_tests in .gclient.\n'
290             'Alternatively, do:\n'
291             '  $ cd chrome/test/chromedriver/third_party\n'
292             '  $ svn co %s java_tests' % java_tests_url)
293      return 1
294
295    results = []
296    for filter in test_filters:
297      results += _Run(
298          java_tests_src_dir=java_tests_src_dir,
299          test_filter=filter,
300          chromedriver_path=options.chromedriver,
301          chrome_path=options.chrome,
302          android_package=options.android_package,
303          verbose=options.verbose,
304          debug=options.debug)
305    return PrintTestResults(results)
306  finally:
307    environment.GlobalTearDown()
308
309
310if __name__ == '__main__':
311  sys.exit(main())
312