1#!/usr/bin/env python 2# Copyright 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Runs the WebDriver Java acceptance tests. 7 8This script is called from chrome/test/chromedriver/run_all_tests.py and reports 9results using the buildbot annotation scheme. 10 11For ChromeDriver documentation, refer to http://code.google.com/p/chromedriver. 12""" 13 14import optparse 15import os 16import shutil 17import sys 18import xml.dom.minidom as minidom 19 20_THIS_DIR = os.path.abspath(os.path.dirname(__file__)) 21sys.path.insert(1, os.path.join(_THIS_DIR, os.pardir)) 22 23import chrome_paths 24import test_environment 25import util 26 27if util.IsLinux(): 28 sys.path.insert(0, os.path.join(chrome_paths.GetSrc(), 'build', 'android')) 29 from pylib import constants 30 31 32class TestResult(object): 33 """A result for an attempted single test case.""" 34 35 def __init__(self, name, time, failure): 36 """Initializes a test result. 37 38 Args: 39 name: the full name of the test. 40 time: the amount of time the test ran, in seconds. 41 failure: the test error or failure message, or None if the test passed. 42 """ 43 self._name = name 44 self._time = time 45 self._failure = failure 46 47 def GetName(self): 48 """Returns the test name.""" 49 return self._name 50 51 def GetTime(self): 52 """Returns the time it took to run the test.""" 53 return self._time 54 55 def IsPass(self): 56 """Returns whether the test passed.""" 57 return self._failure is None 58 59 def GetFailureMessage(self): 60 """Returns the test failure message, or None if the test passed.""" 61 return self._failure 62 63 64def _Run(java_tests_src_dir, test_filter, 65 chromedriver_path, chrome_path, log_path, android_package_key, 66 verbose, debug): 67 """Run the WebDriver Java tests and return the test results. 68 69 Args: 70 java_tests_src_dir: the java test source code directory. 71 test_filter: the filter to use when choosing tests to run. Format is same 72 as Google C++ Test format. 73 chromedriver_path: path to ChromeDriver exe. 74 chrome_path: path to Chrome exe. 75 log_path: path to server log. 76 android_package_key: name of Chrome's Android package. 77 verbose: whether the output should be verbose. 78 debug: whether the tests should wait until attached by a debugger. 79 80 Returns: 81 A list of |TestResult|s. 82 """ 83 test_dir = util.MakeTempDir() 84 keystore_path = ('java', 'client', 'test', 'keystore') 85 required_dirs = [keystore_path[:-1], 86 ('javascript',), 87 ('third_party', 'closure', 'goog'), 88 ('third_party', 'js')] 89 for required_dir in required_dirs: 90 os.makedirs(os.path.join(test_dir, *required_dir)) 91 92 test_jar = 'test-standalone.jar' 93 class_path = test_jar 94 shutil.copyfile(os.path.join(java_tests_src_dir, 'keystore'), 95 os.path.join(test_dir, *keystore_path)) 96 util.Unzip(os.path.join(java_tests_src_dir, 'common.zip'), test_dir) 97 shutil.copyfile(os.path.join(java_tests_src_dir, test_jar), 98 os.path.join(test_dir, test_jar)) 99 100 sys_props = ['selenium.browser=chrome', 101 'webdriver.chrome.driver=' + os.path.abspath(chromedriver_path)] 102 if chrome_path: 103 sys_props += ['webdriver.chrome.binary=' + os.path.abspath(chrome_path)] 104 if log_path: 105 sys_props += ['webdriver.chrome.logfile=' + log_path] 106 if android_package_key: 107 android_package = constants.PACKAGE_INFO[android_package_key].package 108 sys_props += ['webdriver.chrome.android_package=' + android_package] 109 if android_package_key == 'chromedriver_webview_shell': 110 android_activity = constants.PACKAGE_INFO[android_package_key].activity 111 android_process = '%s:main' % android_package 112 sys_props += ['webdriver.chrome.android_activity=' + android_activity] 113 sys_props += ['webdriver.chrome.android_process=' + android_process] 114 if test_filter: 115 # Test jar actually takes a regex. Convert from glob. 116 test_filter = test_filter.replace('*', '.*') 117 sys_props += ['filter=' + test_filter] 118 119 jvm_args = [] 120 if debug: 121 transport = 'dt_socket' 122 if util.IsWindows(): 123 transport = 'dt_shmem' 124 jvm_args += ['-agentlib:jdwp=transport=%s,server=y,suspend=y,' 125 'address=33081' % transport] 126 # Unpack the sources into the test directory and add to the class path 127 # for ease of debugging, particularly with jdb. 128 util.Unzip(os.path.join(java_tests_src_dir, 'test-nodeps-srcs.jar'), 129 test_dir) 130 class_path += ':' + test_dir 131 132 return _RunAntTest( 133 test_dir, 'org.openqa.selenium.chrome.ChromeDriverTests', 134 class_path, sys_props, jvm_args, verbose) 135 136 137def _RunAntTest(test_dir, test_class, class_path, sys_props, jvm_args, verbose): 138 """Runs a single Ant JUnit test suite and returns the |TestResult|s. 139 140 Args: 141 test_dir: the directory to run the tests in. 142 test_class: the name of the JUnit test suite class to run. 143 class_path: the Java class path used when running the tests, colon delimited 144 sys_props: Java system properties to set when running the tests. 145 jvm_args: Java VM command line args to use. 146 verbose: whether the output should be verbose. 147 148 Returns: 149 A list of |TestResult|s. 150 """ 151 def _CreateBuildConfig(test_name, results_file, class_path, junit_props, 152 sys_props, jvm_args): 153 def _SystemPropToXml(prop): 154 key, value = prop.split('=') 155 return '<sysproperty key="%s" value="%s"/>' % (key, value) 156 def _JvmArgToXml(arg): 157 return '<jvmarg value="%s"/>' % arg 158 return '\n'.join([ 159 '<project>', 160 ' <target name="test">', 161 ' <junit %s>' % ' '.join(junit_props), 162 ' <formatter type="xml"/>', 163 ' <classpath>', 164 ' <pathelement path="%s"/>' % class_path, 165 ' </classpath>', 166 ' ' + '\n '.join(map(_SystemPropToXml, sys_props)), 167 ' ' + '\n '.join(map(_JvmArgToXml, jvm_args)), 168 ' <test name="%s" outfile="%s"/>' % (test_name, results_file), 169 ' </junit>', 170 ' </target>', 171 '</project>']) 172 173 def _ProcessResults(results_path): 174 doc = minidom.parse(results_path) 175 tests = [] 176 for test in doc.getElementsByTagName('testcase'): 177 name = test.getAttribute('classname') + '.' + test.getAttribute('name') 178 time = test.getAttribute('time') 179 failure = None 180 error_nodes = test.getElementsByTagName('error') 181 failure_nodes = test.getElementsByTagName('failure') 182 if error_nodes: 183 failure = error_nodes[0].childNodes[0].nodeValue 184 elif failure_nodes: 185 failure = failure_nodes[0].childNodes[0].nodeValue 186 tests += [TestResult(name, time, failure)] 187 return tests 188 189 junit_props = ['printsummary="yes"', 190 'fork="yes"', 191 'haltonfailure="no"', 192 'haltonerror="no"'] 193 if verbose: 194 junit_props += ['showoutput="yes"'] 195 196 ant_file = open(os.path.join(test_dir, 'build.xml'), 'w') 197 ant_file.write(_CreateBuildConfig( 198 test_class, 'results', class_path, junit_props, sys_props, jvm_args)) 199 ant_file.close() 200 201 if util.IsWindows(): 202 ant_name = 'ant.bat' 203 else: 204 ant_name = 'ant' 205 code = util.RunCommand([ant_name, 'test'], cwd=test_dir) 206 if code != 0: 207 print 'FAILED to run java tests of %s through ant' % test_class 208 return 209 return _ProcessResults(os.path.join(test_dir, 'results.xml')) 210 211 212def PrintTestResults(results): 213 """Prints the given results in a format recognized by the buildbot.""" 214 failures = [] 215 failure_names = [] 216 for result in results: 217 if not result.IsPass(): 218 failures += [result] 219 failure_names += ['.'.join(result.GetName().split('.')[-2:])] 220 221 print 'Ran %s tests' % len(results) 222 print 'Failed %s:' % len(failures) 223 util.AddBuildStepText('failed %s/%s' % (len(failures), len(results))) 224 for result in failures: 225 print '=' * 80 226 print '=' * 10, result.GetName(), '(%ss)' % result.GetTime() 227 print result.GetFailureMessage() 228 if len(failures) < 10: 229 util.AddBuildStepText('.'.join(result.GetName().split('.')[-2:])) 230 print 'Rerun failing tests with filter:', ':'.join(failure_names) 231 return len(failures) 232 233 234def main(): 235 parser = optparse.OptionParser() 236 parser.add_option( 237 '', '--verbose', action='store_true', default=False, 238 help='Whether output should be verbose') 239 parser.add_option( 240 '', '--debug', action='store_true', default=False, 241 help='Whether to wait to be attached by a debugger') 242 parser.add_option( 243 '', '--chromedriver', type='string', default=None, 244 help='Path to a build of the chromedriver library(REQUIRED!)') 245 parser.add_option( 246 '', '--chrome', type='string', default=None, 247 help='Path to a build of the chrome binary') 248 parser.add_option( 249 '', '--log-path', 250 help='Output verbose server logs to this file') 251 parser.add_option( 252 '', '--chrome-version', default='HEAD', 253 help='Version of chrome. Default is \'HEAD\'') 254 parser.add_option( 255 '', '--android-package', help='Android package key') 256 parser.add_option( 257 '', '--filter', type='string', default=None, 258 help='Filter for specifying what tests to run, "*" will run all. E.g., ' 259 '*testShouldReturnTitleOfPageIfSet') 260 parser.add_option( 261 '', '--also-run-disabled-tests', action='store_true', default=False, 262 help='Include disabled tests while running the tests') 263 parser.add_option( 264 '', '--isolate-tests', action='store_true', default=False, 265 help='Relaunch the jar test harness after each test') 266 options, _ = parser.parse_args() 267 268 options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver) 269 if options.chromedriver is None or not os.path.exists(options.chromedriver): 270 parser.error('chromedriver is required or the given path is invalid.' + 271 'Please run "%s --help" for help' % __file__) 272 273 if options.android_package: 274 if options.android_package not in constants.PACKAGE_INFO: 275 parser.error('Invalid --android-package') 276 if options.chrome_version != 'HEAD': 277 parser.error('Android does not support the --chrome-version argument.') 278 environment = test_environment.AndroidTestEnvironment( 279 options.android_package) 280 else: 281 environment = test_environment.DesktopTestEnvironment( 282 options.chrome_version) 283 284 try: 285 environment.GlobalSetUp() 286 # Run passed tests when filter is not provided. 287 if options.isolate_tests: 288 test_filters = environment.GetPassedJavaTests() 289 else: 290 if options.filter: 291 test_filter = options.filter 292 else: 293 test_filter = '*' 294 if not options.also_run_disabled_tests: 295 if '-' in test_filter: 296 test_filter += ':' 297 else: 298 test_filter += '-' 299 test_filter += ':'.join(environment.GetDisabledJavaTestMatchers()) 300 test_filters = [test_filter] 301 302 java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test', 303 'chromedriver', 'third_party', 304 'java_tests') 305 if (not os.path.exists(java_tests_src_dir) or 306 not os.listdir(java_tests_src_dir)): 307 java_tests_url = ('http://src.chromium.org/svn/trunk/deps/third_party' 308 '/webdriver') 309 print ('"%s" is empty or it doesn\'t exist. ' % java_tests_src_dir + 310 'Need to map <chrome-svn>/trunk/deps/third_party/webdriver to ' 311 'chrome/test/chromedriver/third_party/java_tests in .gclient.\n' 312 'Alternatively, do:\n' 313 ' $ cd chrome/test/chromedriver/third_party\n' 314 ' $ svn co %s java_tests' % java_tests_url) 315 return 1 316 317 results = [] 318 for filter in test_filters: 319 results += _Run( 320 java_tests_src_dir=java_tests_src_dir, 321 test_filter=filter, 322 chromedriver_path=options.chromedriver, 323 chrome_path=util.GetAbsolutePathOfUserPath(options.chrome), 324 log_path=options.log_path, 325 android_package_key=options.android_package, 326 verbose=options.verbose, 327 debug=options.debug) 328 return PrintTestResults(results) 329 finally: 330 environment.GlobalTearDown() 331 332 333if __name__ == '__main__': 334 sys.exit(main()) 335