chromium.py revision 81bc750723a18f21cd17d1b173cd2a4dda9cea6e
1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Chromium implementations of the Port interface.""" 31 32import errno 33import logging 34import re 35import signal 36import subprocess 37import sys 38import time 39import webbrowser 40 41from webkitpy.common.system import executive 42from webkitpy.common.system.path import cygpath 43from webkitpy.layout_tests.layout_package import test_expectations 44from webkitpy.layout_tests.port import base 45from webkitpy.layout_tests.port import http_server 46from webkitpy.layout_tests.port import websocket_server 47 48_log = logging.getLogger("webkitpy.layout_tests.port.chromium") 49 50 51# FIXME: This function doesn't belong in this package. 52class ChromiumPort(base.Port): 53 """Abstract base class for Chromium implementations of the Port class.""" 54 55 def __init__(self, **kwargs): 56 base.Port.__init__(self, **kwargs) 57 self._chromium_base_dir = None 58 59 def _check_file_exists(self, path_to_file, file_description, 60 override_step=None, logging=True): 61 """Verify the file is present where expected or log an error. 62 63 Args: 64 file_name: The (human friendly) name or description of the file 65 you're looking for (e.g., "HTTP Server"). Used for error logging. 66 override_step: An optional string to be logged if the check fails. 67 logging: Whether or not log the error messages.""" 68 if not self._filesystem.exists(path_to_file): 69 if logging: 70 _log.error('Unable to find %s' % file_description) 71 _log.error(' at %s' % path_to_file) 72 if override_step: 73 _log.error(' %s' % override_step) 74 _log.error('') 75 return False 76 return True 77 78 79 def baseline_path(self): 80 return self._webkit_baseline_path(self._name) 81 82 def check_build(self, needs_http): 83 result = True 84 85 dump_render_tree_binary_path = self._path_to_driver() 86 result = self._check_file_exists(dump_render_tree_binary_path, 87 'test driver') and result 88 if result and self.get_option('build'): 89 result = self._check_driver_build_up_to_date( 90 self.get_option('configuration')) 91 else: 92 _log.error('') 93 94 helper_path = self._path_to_helper() 95 if helper_path: 96 result = self._check_file_exists(helper_path, 97 'layout test helper') and result 98 99 if self.get_option('pixel_tests'): 100 result = self.check_image_diff( 101 'To override, invoke with --no-pixel-tests') and result 102 103 # It's okay if pretty patch isn't available, but we will at 104 # least log a message. 105 self.check_pretty_patch() 106 107 return result 108 109 def check_sys_deps(self, needs_http): 110 cmd = [self._path_to_driver(), '--check-layout-test-sys-deps'] 111 112 local_error = executive.ScriptError() 113 114 def error_handler(script_error): 115 local_error.exit_code = script_error.exit_code 116 117 output = self._executive.run_command(cmd, error_handler=error_handler) 118 if local_error.exit_code: 119 _log.error('System dependencies check failed.') 120 _log.error('To override, invoke with --nocheck-sys-deps') 121 _log.error('') 122 _log.error(output) 123 return False 124 return True 125 126 def check_image_diff(self, override_step=None, logging=True): 127 image_diff_path = self._path_to_image_diff() 128 return self._check_file_exists(image_diff_path, 'image diff exe', 129 override_step, logging) 130 131 def diff_image(self, expected_contents, actual_contents, 132 diff_filename=None): 133 executable = self._path_to_image_diff() 134 135 tempdir = self._filesystem.mkdtemp() 136 expected_filename = self._filesystem.join(str(tempdir), "expected.png") 137 self._filesystem.write_binary_file(expected_filename, expected_contents) 138 actual_filename = self._filesystem.join(str(tempdir), "actual.png") 139 self._filesystem.write_binary_file(actual_filename, actual_contents) 140 141 if diff_filename: 142 cmd = [executable, '--diff', expected_filename, 143 actual_filename, diff_filename] 144 else: 145 cmd = [executable, expected_filename, actual_filename] 146 147 result = True 148 try: 149 exit_code = self._executive.run_command(cmd, return_exit_code=True) 150 if exit_code == 0: 151 # The images are the same. 152 result = False 153 elif exit_code != 1: 154 _log.error("image diff returned an exit code of " 155 + str(exit_code)) 156 # Returning False here causes the script to think that we 157 # successfully created the diff even though we didn't. If 158 # we return True, we think that the images match but the hashes 159 # don't match. 160 # FIXME: Figure out why image_diff returns other values. 161 result = False 162 except OSError, e: 163 if e.errno == errno.ENOENT or e.errno == errno.EACCES: 164 _compare_available = False 165 else: 166 raise e 167 finally: 168 self._filesystem.rmtree(str(tempdir)) 169 return result 170 171 def driver_name(self): 172 return "DumpRenderTree" 173 174 def path_from_chromium_base(self, *comps): 175 """Returns the full path to path made by joining the top of the 176 Chromium source tree and the list of path components in |*comps|.""" 177 if not self._chromium_base_dir: 178 abspath = self._filesystem.abspath(__file__) 179 offset = abspath.find('third_party') 180 if offset == -1: 181 self._chromium_base_dir = self._filesystem.join( 182 abspath[0:abspath.find('Tools')], 183 'Source', 'WebKit', 'chromium') 184 else: 185 self._chromium_base_dir = abspath[0:offset] 186 return self._filesystem.join(self._chromium_base_dir, *comps) 187 188 def path_to_test_expectations_file(self): 189 return self.path_from_webkit_base('LayoutTests', 'platform', 190 'chromium', 'test_expectations.txt') 191 192 def results_directory(self): 193 try: 194 return self.path_from_chromium_base('webkit', 195 self.get_option('configuration'), 196 self.get_option('results_directory')) 197 except AssertionError: 198 return self._build_path(self.get_option('configuration'), 199 self.get_option('results_directory')) 200 201 def setup_test_run(self): 202 # Delete the disk cache if any to ensure a clean test run. 203 dump_render_tree_binary_path = self._path_to_driver() 204 cachedir = self._filesystem.dirname(dump_render_tree_binary_path) 205 cachedir = self._filesystem.join(cachedir, "cache") 206 if self._filesystem.exists(cachedir): 207 self._filesystem.rmtree(cachedir) 208 209 def create_driver(self, worker_number): 210 """Starts a new Driver and returns a handle to it.""" 211 return ChromiumDriver(self, worker_number) 212 213 def start_helper(self): 214 helper_path = self._path_to_helper() 215 if helper_path: 216 _log.debug("Starting layout helper %s" % helper_path) 217 # Note: Not thread safe: http://bugs.python.org/issue2320 218 self._helper = subprocess.Popen([helper_path], 219 stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None) 220 is_ready = self._helper.stdout.readline() 221 if not is_ready.startswith('ready'): 222 _log.error("layout_test_helper failed to be ready") 223 224 def stop_helper(self): 225 if self._helper: 226 _log.debug("Stopping layout test helper") 227 self._helper.stdin.write("x\n") 228 self._helper.stdin.close() 229 # wait() is not threadsafe and can throw OSError due to: 230 # http://bugs.python.org/issue1731717 231 self._helper.wait() 232 233 def test_expectations(self): 234 """Returns the test expectations for this port. 235 236 Basically this string should contain the equivalent of a 237 test_expectations file. See test_expectations.py for more details.""" 238 expectations_path = self.path_to_test_expectations_file() 239 return self._filesystem.read_text_file(expectations_path) 240 241 def test_expectations_overrides(self): 242 try: 243 overrides_path = self.path_from_chromium_base('webkit', 'tools', 244 'layout_tests', 'test_expectations.txt') 245 except AssertionError: 246 return None 247 if not self._filesystem.exists(overrides_path): 248 return None 249 return self._filesystem.read_text_file(overrides_path) 250 251 def skipped_layout_tests(self, extra_test_files=None): 252 expectations_str = self.test_expectations() 253 overrides_str = self.test_expectations_overrides() 254 test_platform_name = self.test_platform_name() 255 is_debug_mode = False 256 257 all_test_files = self.tests([]) 258 if extra_test_files: 259 all_test_files.update(extra_test_files) 260 261 expectations = test_expectations.TestExpectations( 262 self, all_test_files, expectations_str, self.test_configuration(), 263 is_lint_mode=False, overrides=overrides_str) 264 tests_dir = self.layout_tests_dir() 265 return [self.relative_test_filename(test) 266 for test in expectations.get_tests_with_result_type(test_expectations.SKIP)] 267 268 def test_platform_names(self): 269 return ('mac', 'win', 'linux', 'win-xp', 'win-vista', 'win-7') 270 271 def test_platform_name_to_name(self, test_platform_name): 272 if test_platform_name in self.test_platform_names(): 273 return 'chromium-' + test_platform_name 274 raise ValueError('Unsupported test_platform_name: %s' % 275 test_platform_name) 276 277 def test_repository_paths(self): 278 # Note: for JSON file's backward-compatibility we use 'chrome' rather 279 # than 'chromium' here. 280 repos = super(ChromiumPort, self).test_repository_paths() 281 repos.append(('chrome', self.path_from_chromium_base())) 282 return repos 283 284 # 285 # PROTECTED METHODS 286 # 287 # These routines should only be called by other methods in this file 288 # or any subclasses. 289 # 290 291 def _check_driver_build_up_to_date(self, configuration): 292 if configuration in ('Debug', 'Release'): 293 try: 294 debug_path = self._path_to_driver('Debug') 295 release_path = self._path_to_driver('Release') 296 297 debug_mtime = self._filesystem.mtime(debug_path) 298 release_mtime = self._filesystem.mtime(release_path) 299 300 if (debug_mtime > release_mtime and configuration == 'Release' or 301 release_mtime > debug_mtime and configuration == 'Debug'): 302 _log.warning('You are not running the most ' 303 'recent DumpRenderTree binary. You need to ' 304 'pass --debug or not to select between ' 305 'Debug and Release.') 306 _log.warning('') 307 # This will fail if we don't have both a debug and release binary. 308 # That's fine because, in this case, we must already be running the 309 # most up-to-date one. 310 except OSError: 311 pass 312 return True 313 314 def _chromium_baseline_path(self, platform): 315 if platform is None: 316 platform = self.name() 317 return self.path_from_webkit_base('LayoutTests', 'platform', platform) 318 319 def _convert_path(self, path): 320 """Handles filename conversion for subprocess command line args.""" 321 # See note above in diff_image() for why we need this. 322 if sys.platform == 'cygwin': 323 return cygpath(path) 324 return path 325 326 def _path_to_image_diff(self): 327 binary_name = 'ImageDiff' 328 return self._build_path(self.get_option('configuration'), binary_name) 329 330 331class ChromiumDriver(base.Driver): 332 """Abstract interface for DRT.""" 333 334 def __init__(self, port, worker_number): 335 self._port = port 336 self._worker_number = worker_number 337 self._image_path = None 338 if self._port.get_option('pixel_tests'): 339 self._image_path = self._port._filesystem.join( 340 self._port.get_option('results_directory'), 341 'png_result%s.png' % self._worker_number) 342 343 def cmd_line(self): 344 cmd = self._command_wrapper(self._port.get_option('wrapper')) 345 cmd.append(self._port._path_to_driver()) 346 if self._port.get_option('pixel_tests'): 347 # See note above in diff_image() for why we need _convert_path(). 348 cmd.append("--pixel-tests=" + 349 self._port._convert_path(self._image_path)) 350 351 cmd.append('--test-shell') 352 353 if self._port.get_option('startup_dialog'): 354 cmd.append('--testshell-startup-dialog') 355 356 if self._port.get_option('gp_fault_error_box'): 357 cmd.append('--gp-fault-error-box') 358 359 if self._port.get_option('js_flags') is not None: 360 cmd.append('--js-flags="' + self._port.get_option('js_flags') + '"') 361 362 if self._port.get_option('stress_opt'): 363 cmd.append('--stress-opt') 364 365 if self._port.get_option('stress_deopt'): 366 cmd.append('--stress-deopt') 367 368 if self._port.get_option('accelerated_compositing'): 369 cmd.append('--enable-accelerated-compositing') 370 if self._port.get_option('accelerated_2d_canvas'): 371 cmd.append('--enable-accelerated-2d-canvas') 372 if self._port.get_option('enable_hardware_gpu'): 373 cmd.append('--enable-hardware-gpu') 374 return cmd 375 376 def start(self): 377 # FIXME: Should be an error to call this method twice. 378 cmd = self.cmd_line() 379 380 # We need to pass close_fds=True to work around Python bug #2320 381 # (otherwise we can hang when we kill DumpRenderTree when we are running 382 # multiple threads). See http://bugs.python.org/issue2320 . 383 # Note that close_fds isn't supported on Windows, but this bug only 384 # shows up on Mac and Linux. 385 close_flag = sys.platform not in ('win32', 'cygwin') 386 self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, 387 stdout=subprocess.PIPE, 388 stderr=subprocess.STDOUT, 389 close_fds=close_flag) 390 391 def poll(self): 392 # poll() is not threadsafe and can throw OSError due to: 393 # http://bugs.python.org/issue1731717 394 return self._proc.poll() 395 396 def _write_command_and_read_line(self, input=None): 397 """Returns a tuple: (line, did_crash)""" 398 try: 399 if input: 400 if isinstance(input, unicode): 401 # DRT expects utf-8 402 input = input.encode("utf-8") 403 self._proc.stdin.write(input) 404 # DumpRenderTree text output is always UTF-8. However some tests 405 # (e.g. webarchive) may spit out binary data instead of text so we 406 # don't bother to decode the output. 407 line = self._proc.stdout.readline() 408 # We could assert() here that line correctly decodes as UTF-8. 409 return (line, False) 410 except IOError, e: 411 _log.error("IOError communicating w/ DRT: " + str(e)) 412 return (None, True) 413 414 def _test_shell_command(self, uri, timeoutms, checksum): 415 cmd = uri 416 if timeoutms: 417 cmd += ' ' + str(timeoutms) 418 if checksum: 419 cmd += ' ' + checksum 420 cmd += "\n" 421 return cmd 422 423 def _output_image(self): 424 """Returns the image output which driver generated.""" 425 png_path = self._image_path 426 if png_path and self._port._filesystem.exists(png_path): 427 return self._port._filesystem.read_binary_file(png_path) 428 else: 429 return '' 430 431 def _output_image_with_retry(self): 432 # Retry a few more times because open() sometimes fails on Windows, 433 # raising "IOError: [Errno 13] Permission denied:" 434 retry_num = 50 435 timeout_seconds = 5.0 436 for i in range(retry_num): 437 try: 438 return self._output_image() 439 except IOError, e: 440 if e.errno == errno.EACCES: 441 time.sleep(timeout_seconds / retry_num) 442 else: 443 raise e 444 return self._output_image() 445 446 def _clear_output_image(self): 447 png_path = self._image_path 448 if png_path and self._port._filesystem.exists(png_path): 449 self._port._filesystem.remove(png_path) 450 451 def run_test(self, driver_input): 452 output = [] 453 error = [] 454 crash = False 455 timeout = False 456 actual_uri = None 457 actual_checksum = None 458 self._clear_output_image() 459 start_time = time.time() 460 461 uri = self._port.filename_to_uri(driver_input.filename) 462 cmd = self._test_shell_command(uri, driver_input.timeout, 463 driver_input.image_hash) 464 (line, crash) = self._write_command_and_read_line(input=cmd) 465 466 while not crash and line.rstrip() != "#EOF": 467 # Make sure we haven't crashed. 468 if line == '' and self.poll() is not None: 469 # This is hex code 0xc000001d, which is used for abrupt 470 # termination. This happens if we hit ctrl+c from the prompt 471 # and we happen to be waiting on DRT. 472 # sdoyon: Not sure for which OS and in what circumstances the 473 # above code is valid. What works for me under Linux to detect 474 # ctrl+c is for the subprocess returncode to be negative 475 # SIGINT. And that agrees with the subprocess documentation. 476 if (-1073741510 == self._proc.returncode or 477 - signal.SIGINT == self._proc.returncode): 478 raise KeyboardInterrupt 479 crash = True 480 break 481 482 # Don't include #URL lines in our output 483 if line.startswith("#URL:"): 484 actual_uri = line.rstrip()[5:] 485 if uri != actual_uri: 486 # GURL capitalizes the drive letter of a file URL. 487 if (not re.search("^file:///[a-z]:", uri) or 488 uri.lower() != actual_uri.lower()): 489 _log.fatal("Test got out of sync:\n|%s|\n|%s|" % 490 (uri, actual_uri)) 491 raise AssertionError("test out of sync") 492 elif line.startswith("#MD5:"): 493 actual_checksum = line.rstrip()[5:] 494 elif line.startswith("#TEST_TIMED_OUT"): 495 timeout = True 496 # Test timed out, but we still need to read until #EOF. 497 elif actual_uri: 498 output.append(line) 499 else: 500 error.append(line) 501 502 (line, crash) = self._write_command_and_read_line(input=None) 503 504 run_time = time.time() - start_time 505 output_image = self._output_image_with_retry() 506 assert output_image is not None 507 return base.DriverOutput(''.join(output), output_image, actual_checksum, 508 crash, run_time, timeout, ''.join(error)) 509 510 def stop(self): 511 if self._proc: 512 self._proc.stdin.close() 513 self._proc.stdout.close() 514 if self._proc.stderr: 515 self._proc.stderr.close() 516 if sys.platform not in ('win32', 'cygwin'): 517 # Closing stdin/stdout/stderr hangs sometimes on OS X, 518 # (see __init__(), above), and anyway we don't want to hang 519 # the harness if DRT is buggy, so we wait a couple 520 # seconds to give DRT a chance to clean up, but then 521 # force-kill the process if necessary. 522 KILL_TIMEOUT = 3.0 523 timeout = time.time() + KILL_TIMEOUT 524 # poll() is not threadsafe and can throw OSError due to: 525 # http://bugs.python.org/issue1731717 526 while self._proc.poll() is None and time.time() < timeout: 527 time.sleep(0.1) 528 # poll() is not threadsafe and can throw OSError due to: 529 # http://bugs.python.org/issue1731717 530 if self._proc.poll() is None: 531 _log.warning('stopping test driver timed out, ' 532 'killing it') 533 self._port._executive.kill_process(self._proc.pid) 534