1#!/usr/bin/env python 2# Copyright 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Runs the WebDriver Java acceptance tests. 7 8This script is called from chrome/test/chromedriver/run_all_tests.py and reports 9results using the buildbot annotation scheme. 10 11For ChromeDriver documentation, refer to http://code.google.com/p/chromedriver. 12""" 13 14import optparse 15import os 16import shutil 17import sys 18import xml.dom.minidom as minidom 19 20_THIS_DIR = os.path.abspath(os.path.dirname(__file__)) 21sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir)) 22 23import chrome_paths 24import test_environment 25import util 26 27 28class TestResult(object): 29 """A result for an attempted single test case.""" 30 31 def __init__(self, name, time, failure): 32 """Initializes a test result. 33 34 Args: 35 name: the full name of the test. 36 time: the amount of time the test ran, in seconds. 37 failure: the test error or failure message, or None if the test passed. 38 """ 39 self._name = name 40 self._time = time 41 self._failure = failure 42 43 def GetName(self): 44 """Returns the test name.""" 45 return self._name 46 47 def GetTime(self): 48 """Returns the time it took to run the test.""" 49 return self._time 50 51 def IsPass(self): 52 """Returns whether the test passed.""" 53 return self._failure is None 54 55 def GetFailureMessage(self): 56 """Returns the test failure message, or None if the test passed.""" 57 return self._failure 58 59 60def _Run(java_tests_src_dir, test_filter, 61 chromedriver_path, chrome_path, android_package, 62 verbose, debug): 63 """Run the WebDriver Java tests and return the test results. 64 65 Args: 66 java_tests_src_dir: the java test source code directory. 67 test_filter: the filter to use when choosing tests to run. Format is same 68 as Google C++ Test format. 69 chromedriver_path: path to ChromeDriver exe. 70 chrome_path: path to Chrome exe. 71 android_package: name of Chrome's Android package. 72 verbose: whether the output should be verbose. 73 debug: whether the tests should wait until attached by a debugger. 74 75 Returns: 76 A list of |TestResult|s. 77 """ 78 test_dir = util.MakeTempDir() 79 keystore_path = ('java', 'client', 'test', 'keystore') 80 required_dirs = [keystore_path[:-1], 81 ('javascript',), 82 ('third_party', 'closure', 'goog'), 83 ('third_party', 'js')] 84 for required_dir in required_dirs: 85 os.makedirs(os.path.join(test_dir, *required_dir)) 86 87 test_jar = 'test-standalone.jar' 88 class_path = test_jar 89 shutil.copyfile(os.path.join(java_tests_src_dir, 'keystore'), 90 os.path.join(test_dir, *keystore_path)) 91 util.Unzip(os.path.join(java_tests_src_dir, 'common.zip'), test_dir) 92 shutil.copyfile(os.path.join(java_tests_src_dir, test_jar), 93 os.path.join(test_dir, test_jar)) 94 95 sys_props = ['selenium.browser=chrome', 96 'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)] 97 if chrome_path is not None: 98 sys_props += ['webdriver.chrome.binary=' + os.path.abspath(chrome_path)] 99 if android_package is not None: 100 sys_props += ['webdriver.chrome.android_package=' + android_package] 101 if test_filter: 102 # Test jar actually takes a regex. Convert from glob. 103 test_filter = test_filter.replace('*', '.*') 104 sys_props += ['filter=' + test_filter] 105 106 jvm_args = [] 107 if debug: 108 jvm_args += ['-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,' 109 'address=33081'] 110 # Unpack the sources into the test directory and add to the class path 111 # for ease of debugging, particularly with jdb. 112 util.Unzip(os.path.join(java_tests_src_dir, 'test-nodeps-srcs.jar'), 113 test_dir) 114 class_path += ':' + test_dir 115 116 return _RunAntTest( 117 test_dir, 'org.openqa.selenium.chrome.ChromeDriverTests', 118 class_path, sys_props, jvm_args, verbose) 119 120 121def _RunAntTest(test_dir, test_class, class_path, sys_props, jvm_args, verbose): 122 """Runs a single Ant JUnit test suite and returns the |TestResult|s. 123 124 Args: 125 test_dir: the directory to run the tests in. 126 test_class: the name of the JUnit test suite class to run. 127 class_path: the Java class path used when running the tests, colon delimited 128 sys_props: Java system properties to set when running the tests. 129 jvm_args: Java VM command line args to use. 130 verbose: whether the output should be verbose. 131 132 Returns: 133 A list of |TestResult|s. 134 """ 135 def _CreateBuildConfig(test_name, results_file, class_path, junit_props, 136 sys_props, jvm_args): 137 def _SystemPropToXml(prop): 138 key, value = prop.split('=') 139 return '<sysproperty key="%s" value="%s"/>' % (key, value) 140 def _JvmArgToXml(arg): 141 return '<jvmarg value="%s"/>' % arg 142 return '\n'.join([ 143 '<project>', 144 ' <target name="test">', 145 ' <junit %s>' % ' '.join(junit_props), 146 ' <formatter type="xml"/>', 147 ' <classpath>', 148 ' <pathelement path="%s"/>' % class_path, 149 ' </classpath>', 150 ' ' + '\n '.join(map(_SystemPropToXml, sys_props)), 151 ' ' + '\n '.join(map(_JvmArgToXml, jvm_args)), 152 ' <test name="%s" outfile="%s"/>' % (test_name, results_file), 153 ' </junit>', 154 ' </target>', 155 '</project>']) 156 157 def _ProcessResults(results_path): 158 doc = minidom.parse(results_path) 159 tests = [] 160 for test in doc.getElementsByTagName('testcase'): 161 name = test.getAttribute('classname') + '.' + test.getAttribute('name') 162 time = test.getAttribute('time') 163 failure = None 164 error_nodes = test.getElementsByTagName('error') 165 failure_nodes = test.getElementsByTagName('failure') 166 if error_nodes: 167 failure = error_nodes[0].childNodes[0].nodeValue 168 elif failure_nodes: 169 failure = failure_nodes[0].childNodes[0].nodeValue 170 tests += [TestResult(name, time, failure)] 171 return tests 172 173 junit_props = ['printsummary="yes"', 174 'fork="yes"', 175 'haltonfailure="no"', 176 'haltonerror="no"'] 177 if verbose: 178 junit_props += ['showoutput="yes"'] 179 180 ant_file = open(os.path.join(test_dir, 'build.xml'), 'w') 181 ant_file.write(_CreateBuildConfig( 182 test_class, 'results', class_path, junit_props, sys_props, jvm_args)) 183 ant_file.close() 184 185 if util.IsWindows(): 186 ant_name = 'ant.bat' 187 else: 188 ant_name = 'ant' 189 code = util.RunCommand([ant_name, 'test'], cwd=test_dir) 190 if code != 0: 191 print 'FAILED to run java tests of %s through ant' % test_class 192 return 193 return _ProcessResults(os.path.join(test_dir, 'results.xml')) 194 195 196def PrintTestResults(results): 197 """Prints the given results in a format recognized by the buildbot.""" 198 failures = [] 199 failure_names = [] 200 for result in results: 201 if not result.IsPass(): 202 failures += [result] 203 failure_names += ['.'.join(result.GetName().split('.')[-2:])] 204 205 print 'Ran %s tests' % len(results) 206 print 'Failed %s:' % len(failures) 207 util.AddBuildStepText('failed %s/%s' % (len(failures), len(results))) 208 for result in failures: 209 print '=' * 80 210 print '=' * 10, result.GetName(), '(%ss)' % result.GetTime() 211 print result.GetFailureMessage() 212 if len(failures) < 10: 213 util.AddBuildStepText('.'.join(result.GetName().split('.')[-2:])) 214 print 'Rerun failing tests with filter:', ':'.join(failure_names) 215 return len(failures) 216 217 218def main(): 219 parser = optparse.OptionParser() 220 parser.add_option( 221 '', '--verbose', action='store_true', default=False, 222 help='Whether output should be verbose') 223 parser.add_option( 224 '', '--debug', action='store_true', default=False, 225 help='Whether to wait to be attached by a debugger') 226 parser.add_option( 227 '', '--chromedriver', type='string', default=None, 228 help='Path to a build of the chromedriver library(REQUIRED!)') 229 parser.add_option( 230 '', '--chrome', type='string', default=None, 231 help='Path to a build of the chrome binary') 232 parser.add_option( 233 '', '--chrome-version', default='HEAD', 234 help='Version of chrome. Default is \'HEAD\'') 235 parser.add_option( 236 '', '--android-package', type='string', default=None, 237 help='Name of Chrome\'s Android package') 238 parser.add_option( 239 '', '--filter', type='string', default=None, 240 help='Filter for specifying what tests to run, "*" will run all. E.g., ' 241 '*testShouldReturnTitleOfPageIfSet') 242 parser.add_option( 243 '', '--also-run-disabled-tests', action='store_true', default=False, 244 help='Include disabled tests while running the tests') 245 parser.add_option( 246 '', '--isolate-tests', action='store_true', default=False, 247 help='Relaunch the jar test harness after each test') 248 options, _ = parser.parse_args() 249 250 if options.chromedriver is None or not os.path.exists(options.chromedriver): 251 parser.error('chromedriver is required or the given path is invalid.' + 252 'Please run "%s --help" for help' % __file__) 253 254 if options.android_package is not None: 255 if options.chrome_version != 'HEAD': 256 parser.error('Android does not support the --chrome-version argument.') 257 environment = test_environment.AndroidTestEnvironment() 258 else: 259 environment = test_environment.DesktopTestEnvironment( 260 options.chrome_version) 261 262 try: 263 environment.GlobalSetUp() 264 # Run passed tests when filter is not provided. 265 if options.isolate_tests: 266 test_filters = environment.GetPassedJavaTests() 267 else: 268 if options.filter: 269 test_filter = options.filter 270 else: 271 test_filter = '*' 272 if not options.also_run_disabled_tests: 273 if '-' in test_filter: 274 test_filter += ':' 275 else: 276 test_filter += '-' 277 test_filter += ':'.join(environment.GetDisabledJavaTestMatchers()) 278 test_filters = [test_filter] 279 280 java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test', 281 'chromedriver', 'third_party', 282 'java_tests') 283 if (not os.path.exists(java_tests_src_dir) or 284 not os.listdir(java_tests_src_dir)): 285 java_tests_url = ('http://src.chromium.org/svn/trunk/deps/third_party' 286 '/webdriver') 287 print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir + 288 'Need to map <chrome-svn>/trunk/deps/third_party/webdriver to ' 289 'chrome/test/chromedriver/third_party/java_tests in .gclient.\n' 290 'Alternatively, do:\n' 291 ' $ cd chrome/test/chromedriver/third_party\n' 292 ' $ svn co %s java_tests' % java_tests_url) 293 return 1 294 295 results = [] 296 for filter in test_filters: 297 results += _Run( 298 java_tests_src_dir=java_tests_src_dir, 299 test_filter=filter, 300 chromedriver_path=options.chromedriver, 301 chrome_path=options.chrome, 302 android_package=options.android_package, 303 verbose=options.verbose, 304 debug=options.debug) 305 return PrintTestResults(results) 306 finally: 307 environment.GlobalTearDown() 308 309 310if __name__ == '__main__': 311 sys.exit(main()) 312