1# Copyright (C) 2010 Google Inc. All rights reserved. 2# 3# Redistribution and use in source and binary forms, with or without 4# modification, are permitted provided that the following conditions are 5# met: 6# 7# * Redistributions of source code must retain the above copyright 8# notice, this list of conditions and the following disclaimer. 9# * Redistributions in binary form must reproduce the above 10# copyright notice, this list of conditions and the following disclaimer 11# in the documentation and/or other materials provided with the 12# distribution. 13# * Neither the name of Google Inc. nor the names of its 14# contributors may be used to endorse or promote products derived from 15# this software without specific prior written permission. 16# 17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 29"""Unit testing base class for Port implementations.""" 30 31import collections 32import errno 33import logging 34import os 35import socket 36import sys 37import time 38import unittest 39 40from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2 41from webkitpy.common.system.filesystem_mock import MockFileSystem 42from webkitpy.common.system.outputcapture import OutputCapture 43from webkitpy.common.system.systemhost import SystemHost 44from webkitpy.common.system.systemhost_mock import MockSystemHost 45from webkitpy.layout_tests.models import test_run_results 46from webkitpy.layout_tests.port.base import Port, TestConfiguration 47from webkitpy.layout_tests.port.server_process_mock import MockServerProcess 48from webkitpy.tool.mocktool import MockOptions 49 50 51# FIXME: get rid of this fixture 52class TestWebKitPort(Port): 53 port_name = "testwebkitport" 54 55 def __init__(self, port_name=None, symbols_string=None, 56 expectations_file=None, skips_file=None, host=None, config=None, 57 **kwargs): 58 port_name = port_name or TestWebKitPort.port_name 59 self.symbols_string = symbols_string # Passing "" disables all staticly-detectable features. 60 host = host or MockSystemHost() 61 super(TestWebKitPort, self).__init__(host, port_name=port_name, **kwargs) 62 63 def all_test_configurations(self): 64 return [self.test_configuration()] 65 66 def _symbols_string(self): 67 return self.symbols_string 68 69 def _tests_for_disabled_features(self): 70 return ["accessibility", ] 71 72 73class FakePrinter(object): 74 def write_update(self, msg): 75 pass 76 77 def write_throttled_update(self, msg): 78 pass 79 80 81 82class PortTestCase(unittest.TestCase): 83 """Tests that all Port implementations must pass.""" 84 HTTP_PORTS = (8000, 8080, 8443) 85 WEBSOCKET_PORTS = (8880,) 86 87 # Subclasses override this to point to their Port subclass. 88 os_name = None 89 os_version = None 90 port_maker = TestWebKitPort 91 port_name = None 92 full_port_name = None 93 94 def make_port(self, host=None, port_name=None, options=None, os_name=None, os_version=None, **kwargs): 95 host = host or MockSystemHost(os_name=(os_name or self.os_name), os_version=(os_version or self.os_version)) 96 options = options or MockOptions(configuration='Release') 97 port_name = port_name or self.port_name 98 port_name = self.port_maker.determine_full_port_name(host, options, port_name) 99 port = self.port_maker(host, port_name, options=options, **kwargs) 100 port._config.build_directory = lambda configuration: '/mock-build' 101 return port 102 103 def make_wdiff_available(self, port): 104 port._wdiff_available = True 105 106 def test_check_build(self): 107 port = self.make_port() 108 port._check_file_exists = lambda path, desc: True 109 if port._dump_reader: 110 port._dump_reader.check_is_functional = lambda: True 111 port._options.build = True 112 port._check_driver_build_up_to_date = lambda config: True 113 port.check_httpd = lambda: True 114 oc = OutputCapture() 115 try: 116 oc.capture_output() 117 self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()), 118 test_run_results.OK_EXIT_STATUS) 119 finally: 120 out, err, logs = oc.restore_output() 121 self.assertIn('pretty patches', logs) # We should get a warning about PrettyPatch being missing, 122 self.assertNotIn('build requirements', logs) # but not the driver itself. 123 124 port._check_file_exists = lambda path, desc: False 125 port._check_driver_build_up_to_date = lambda config: False 126 try: 127 oc.capture_output() 128 self.assertEqual(port.check_build(needs_http=True, printer=FakePrinter()), 129 test_run_results.UNEXPECTED_ERROR_EXIT_STATUS) 130 finally: 131 out, err, logs = oc.restore_output() 132 self.assertIn('pretty patches', logs) # And, hereere we should get warnings about both. 133 self.assertIn('build requirements', logs) 134 135 def test_default_max_locked_shards(self): 136 port = self.make_port() 137 port.default_child_processes = lambda: 16 138 self.assertEqual(port.default_max_locked_shards(), 4) 139 port.default_child_processes = lambda: 2 140 self.assertEqual(port.default_max_locked_shards(), 1) 141 142 def test_default_timeout_ms(self): 143 self.assertEqual(self.make_port(options=MockOptions(configuration='Release')).default_timeout_ms(), 6000) 144 self.assertEqual(self.make_port(options=MockOptions(configuration='Debug')).default_timeout_ms(), 18000) 145 146 def test_default_pixel_tests(self): 147 self.assertEqual(self.make_port().default_pixel_tests(), True) 148 149 def test_driver_cmd_line(self): 150 port = self.make_port() 151 self.assertTrue(len(port.driver_cmd_line())) 152 153 options = MockOptions(additional_drt_flag=['--foo=bar', '--foo=baz']) 154 port = self.make_port(options=options) 155 cmd_line = port.driver_cmd_line() 156 self.assertTrue('--foo=bar' in cmd_line) 157 self.assertTrue('--foo=baz' in cmd_line) 158 159 def assert_servers_are_down(self, host, ports): 160 for port in ports: 161 try: 162 test_socket = socket.socket() 163 test_socket.connect((host, port)) 164 self.fail() 165 except IOError, e: 166 self.assertTrue(e.errno in (errno.ECONNREFUSED, errno.ECONNRESET)) 167 finally: 168 test_socket.close() 169 170 def assert_servers_are_up(self, host, ports): 171 for port in ports: 172 try: 173 test_socket = socket.socket() 174 test_socket.connect((host, port)) 175 except IOError, e: 176 self.fail('failed to connect to %s:%d' % (host, port)) 177 finally: 178 test_socket.close() 179 180 def test_diff_image__missing_both(self): 181 port = self.make_port() 182 self.assertEqual(port.diff_image(None, None), (None, None)) 183 self.assertEqual(port.diff_image(None, ''), (None, None)) 184 self.assertEqual(port.diff_image('', None), (None, None)) 185 186 self.assertEqual(port.diff_image('', ''), (None, None)) 187 188 def test_diff_image__missing_actual(self): 189 port = self.make_port() 190 self.assertEqual(port.diff_image(None, 'foo'), ('foo', None)) 191 self.assertEqual(port.diff_image('', 'foo'), ('foo', None)) 192 193 def test_diff_image__missing_expected(self): 194 port = self.make_port() 195 self.assertEqual(port.diff_image('foo', None), ('foo', None)) 196 self.assertEqual(port.diff_image('foo', ''), ('foo', None)) 197 198 def test_diff_image(self): 199 def _path_to_image_diff(): 200 return "/path/to/image_diff" 201 202 port = self.make_port() 203 port._path_to_image_diff = _path_to_image_diff 204 205 mock_image_diff = "MOCK Image Diff" 206 207 def mock_run_command(args): 208 port._filesystem.write_binary_file(args[4], mock_image_diff) 209 return 1 210 211 # Images are different. 212 port._executive = MockExecutive2(run_command_fn=mock_run_command) 213 self.assertEqual(mock_image_diff, port.diff_image("EXPECTED", "ACTUAL")[0]) 214 215 # Images are the same. 216 port._executive = MockExecutive2(exit_code=0) 217 self.assertEqual(None, port.diff_image("EXPECTED", "ACTUAL")[0]) 218 219 # There was some error running image_diff. 220 port._executive = MockExecutive2(exit_code=2) 221 exception_raised = False 222 try: 223 port.diff_image("EXPECTED", "ACTUAL") 224 except ValueError, e: 225 exception_raised = True 226 self.assertFalse(exception_raised) 227 228 def test_diff_image_crashed(self): 229 port = self.make_port() 230 port._executive = MockExecutive2(exit_code=2) 231 self.assertEqual(port.diff_image("EXPECTED", "ACTUAL"), (None, 'Image diff returned an exit code of 2. See http://crbug.com/278596')) 232 233 def test_check_wdiff(self): 234 port = self.make_port() 235 port.check_wdiff() 236 237 def test_wdiff_text_fails(self): 238 host = MockSystemHost(os_name=self.os_name, os_version=self.os_version) 239 host.executive = MockExecutive(should_throw=True) 240 port = self.make_port(host=host) 241 port._executive = host.executive # AndroidPortTest.make_port sets its own executive, so reset that as well. 242 243 # This should raise a ScriptError that gets caught and turned into the 244 # error text, and also mark wdiff as not available. 245 self.make_wdiff_available(port) 246 self.assertTrue(port.wdiff_available()) 247 diff_txt = port.wdiff_text("/tmp/foo.html", "/tmp/bar.html") 248 self.assertEqual(diff_txt, port._wdiff_error_html) 249 self.assertFalse(port.wdiff_available()) 250 251 def test_missing_symbol_to_skipped_tests(self): 252 # Test that we get the chromium skips and not the webkit default skips 253 port = self.make_port() 254 skip_dict = port._missing_symbol_to_skipped_tests() 255 if port.PORT_HAS_AUDIO_CODECS_BUILT_IN: 256 self.assertEqual(skip_dict, {}) 257 else: 258 self.assertTrue('ff_mp3_decoder' in skip_dict) 259 self.assertFalse('WebGLShader' in skip_dict) 260 261 def test_test_configuration(self): 262 port = self.make_port() 263 self.assertTrue(port.test_configuration()) 264 265 def test_all_test_configurations(self): 266 """Validate the complete set of configurations this port knows about.""" 267 port = self.make_port() 268 self.assertEqual(set(port.all_test_configurations()), set([ 269 TestConfiguration('snowleopard', 'x86', 'debug'), 270 TestConfiguration('snowleopard', 'x86', 'release'), 271 TestConfiguration('lion', 'x86', 'debug'), 272 TestConfiguration('lion', 'x86', 'release'), 273 TestConfiguration('retina', 'x86', 'debug'), 274 TestConfiguration('retina', 'x86', 'release'), 275 TestConfiguration('mountainlion', 'x86', 'debug'), 276 TestConfiguration('mountainlion', 'x86', 'release'), 277 TestConfiguration('mavericks', 'x86', 'debug'), 278 TestConfiguration('mavericks', 'x86', 'release'), 279 TestConfiguration('xp', 'x86', 'debug'), 280 TestConfiguration('xp', 'x86', 'release'), 281 TestConfiguration('win7', 'x86', 'debug'), 282 TestConfiguration('win7', 'x86', 'release'), 283 TestConfiguration('lucid', 'x86', 'debug'), 284 TestConfiguration('lucid', 'x86', 'release'), 285 TestConfiguration('lucid', 'x86_64', 'debug'), 286 TestConfiguration('lucid', 'x86_64', 'release'), 287 TestConfiguration('icecreamsandwich', 'x86', 'debug'), 288 TestConfiguration('icecreamsandwich', 'x86', 'release'), 289 ])) 290 def test_get_crash_log(self): 291 port = self.make_port() 292 self.assertEqual(port._get_crash_log(None, None, None, None, newer_than=None), 293 (None, 294 'crash log for <unknown process name> (pid <unknown>):\n' 295 'STDOUT: <empty>\n' 296 'STDERR: <empty>\n')) 297 298 self.assertEqual(port._get_crash_log('foo', 1234, 'out bar\nout baz', 'err bar\nerr baz\n', newer_than=None), 299 ('err bar\nerr baz\n', 300 'crash log for foo (pid 1234):\n' 301 'STDOUT: out bar\n' 302 'STDOUT: out baz\n' 303 'STDERR: err bar\n' 304 'STDERR: err baz\n')) 305 306 self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=None), 307 ('foo\xa6bar', 308 u'crash log for foo (pid 1234):\n' 309 u'STDOUT: foo\ufffdbar\n' 310 u'STDERR: foo\ufffdbar\n')) 311 312 self.assertEqual(port._get_crash_log('foo', 1234, 'foo\xa6bar', 'foo\xa6bar', newer_than=1.0), 313 ('foo\xa6bar', 314 u'crash log for foo (pid 1234):\n' 315 u'STDOUT: foo\ufffdbar\n' 316 u'STDERR: foo\ufffdbar\n')) 317 318 def assert_build_path(self, options, dirs, expected_path): 319 port = self.make_port(options=options) 320 for directory in dirs: 321 port.host.filesystem.maybe_make_directory(directory) 322 self.assertEqual(port._build_path(), expected_path) 323 324 def test_expectations_files(self): 325 port = self.make_port() 326 327 generic_path = port.path_to_generic_test_expectations_file() 328 never_fix_tests_path = port._filesystem.join(port.layout_tests_dir(), 'NeverFixTests') 329 stale_tests_path = port._filesystem.join(port.layout_tests_dir(), 'StaleTestExpectations') 330 slow_tests_path = port._filesystem.join(port.layout_tests_dir(), 'SlowTests') 331 flaky_tests_path = port._filesystem.join(port.layout_tests_dir(), 'FlakyTests') 332 skia_overrides_path = port.path_from_chromium_base( 333 'skia', 'skia_test_expectations.txt') 334 335 port._filesystem.write_text_file(skia_overrides_path, 'dummy text') 336 337 port._options.builder_name = 'DUMMY_BUILDER_NAME' 338 self.assertEqual(port.expectations_files(), 339 [generic_path, skia_overrides_path, 340 never_fix_tests_path, stale_tests_path, slow_tests_path, 341 flaky_tests_path]) 342 343 port._options.builder_name = 'builder (deps)' 344 self.assertEqual(port.expectations_files(), 345 [generic_path, skia_overrides_path, 346 never_fix_tests_path, stale_tests_path, slow_tests_path, 347 flaky_tests_path]) 348 349 # A builder which does NOT observe the Chromium test_expectations, 350 # but still observes the Skia test_expectations... 351 port._options.builder_name = 'builder' 352 self.assertEqual(port.expectations_files(), 353 [generic_path, skia_overrides_path, 354 never_fix_tests_path, stale_tests_path, slow_tests_path, 355 flaky_tests_path]) 356 357 def test_check_sys_deps(self): 358 port = self.make_port() 359 port._executive = MockExecutive2(exit_code=0) 360 self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.OK_EXIT_STATUS) 361 port._executive = MockExecutive2(exit_code=1, output='testing output failure') 362 self.assertEqual(port.check_sys_deps(needs_http=False), test_run_results.SYS_DEPS_EXIT_STATUS) 363 364 def test_expectations_ordering(self): 365 port = self.make_port() 366 for path in port.expectations_files(): 367 port._filesystem.write_text_file(path, '') 368 ordered_dict = port.expectations_dict() 369 self.assertEqual(port.path_to_generic_test_expectations_file(), ordered_dict.keys()[0]) 370 371 options = MockOptions(additional_expectations=['/tmp/foo', '/tmp/bar']) 372 port = self.make_port(options=options) 373 for path in port.expectations_files(): 374 port._filesystem.write_text_file(path, '') 375 port._filesystem.write_text_file('/tmp/foo', 'foo') 376 port._filesystem.write_text_file('/tmp/bar', 'bar') 377 ordered_dict = port.expectations_dict() 378 self.assertEqual(ordered_dict.keys()[-2:], options.additional_expectations) # pylint: disable=E1101 379 self.assertEqual(ordered_dict.values()[-2:], ['foo', 'bar']) 380 381 def test_skipped_directories_for_symbols(self): 382 # This first test confirms that the commonly found symbols result in the expected skipped directories. 383 symbols_string = " ".join(["fooSymbol"]) 384 expected_directories = set([ 385 "webaudio/codec-tests/mp3", 386 "webaudio/codec-tests/aac", 387 ]) 388 389 result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html'])) 390 self.assertEqual(result_directories, expected_directories) 391 392 # Test that the nm string parsing actually works: 393 symbols_string = """ 394000000000124f498 s __ZZN7WebCore13ff_mp3_decoder12replaceChildEPS0_S1_E19__PRETTY_FUNCTION__ 395000000000124f500 s __ZZN7WebCore13ff_mp3_decoder13addChildAboveEPS0_S1_E19__PRETTY_FUNCTION__ 396000000000124f670 s __ZZN7WebCore13ff_mp3_decoder13addChildBelowEPS0_S1_E19__PRETTY_FUNCTION__ 397""" 398 # Note 'compositing' is not in the list of skipped directories (hence the parsing of GraphicsLayer worked): 399 expected_directories = set([ 400 "webaudio/codec-tests/aac", 401 ]) 402 result_directories = set(TestWebKitPort(symbols_string=symbols_string)._skipped_tests_for_unsupported_features(test_list=['webaudio/codec-tests/mp3/foo.html'])) 403 self.assertEqual(result_directories, expected_directories) 404 405 def _assert_config_file_for_platform(self, port, platform, config_file): 406 self.assertEqual(port._apache_config_file_name_for_platform(platform), config_file) 407 408 def test_linux_distro_detection(self): 409 port = TestWebKitPort() 410 self.assertFalse(port._is_redhat_based()) 411 self.assertFalse(port._is_debian_based()) 412 413 port._filesystem = MockFileSystem({'/etc/redhat-release': ''}) 414 self.assertTrue(port._is_redhat_based()) 415 self.assertFalse(port._is_debian_based()) 416 417 port._filesystem = MockFileSystem({'/etc/debian_version': ''}) 418 self.assertFalse(port._is_redhat_based()) 419 self.assertTrue(port._is_debian_based()) 420 421 def test_apache_config_file_name_for_platform(self): 422 port = TestWebKitPort() 423 self._assert_config_file_for_platform(port, 'cygwin', 'cygwin-httpd.conf') 424 425 self._assert_config_file_for_platform(port, 'linux2', 'apache2-httpd.conf') 426 self._assert_config_file_for_platform(port, 'linux3', 'apache2-httpd.conf') 427 428 port._is_redhat_based = lambda: True 429 port._apache_version = lambda: '2.2' 430 self._assert_config_file_for_platform(port, 'linux2', 'fedora-httpd-2.2.conf') 431 432 port = TestWebKitPort() 433 port._is_debian_based = lambda: True 434 port._apache_version = lambda: '2.2' 435 self._assert_config_file_for_platform(port, 'linux2', 'debian-httpd-2.2.conf') 436 437 self._assert_config_file_for_platform(port, 'mac', 'apache2-httpd.conf') 438 self._assert_config_file_for_platform(port, 'win32', 'apache2-httpd.conf') # win32 isn't a supported sys.platform. AppleWin/WinCairo/WinCE ports all use cygwin. 439 self._assert_config_file_for_platform(port, 'barf', 'apache2-httpd.conf') 440 441 def test_path_to_apache_config_file(self): 442 port = TestWebKitPort() 443 444 saved_environ = os.environ.copy() 445 try: 446 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/path/to/httpd.conf' 447 self.assertRaises(IOError, port.path_to_apache_config_file) 448 port._filesystem.write_text_file('/existing/httpd.conf', 'Hello, world!') 449 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' 450 self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf') 451 finally: 452 os.environ = saved_environ.copy() 453 454 # Mock out _apache_config_file_name_for_platform to ignore the passed sys.platform value. 455 port._apache_config_file_name_for_platform = lambda platform: 'httpd.conf' 456 self.assertEqual(port.path_to_apache_config_file(), '/mock-checkout/third_party/WebKit/LayoutTests/http/conf/httpd.conf') 457 458 # Check that even if we mock out _apache_config_file_name, the environment variable takes precedence. 459 saved_environ = os.environ.copy() 460 try: 461 os.environ['WEBKIT_HTTP_SERVER_CONF_PATH'] = '/existing/httpd.conf' 462 self.assertEqual(port.path_to_apache_config_file(), '/existing/httpd.conf') 463 finally: 464 os.environ = saved_environ.copy() 465 466 def test_additional_platform_directory(self): 467 port = self.make_port(options=MockOptions(additional_platform_directory=['/tmp/foo'])) 468 self.assertEqual(port.baseline_search_path()[0], '/tmp/foo') 469 470 def test_virtual_test_suites(self): 471 # We test that we can load the real LayoutTests/VirtualTestSuites file properly, so we 472 # use a real SystemHost(). We don't care what virtual_test_suites() returns as long 473 # as it is iterable. 474 port = self.make_port(host=SystemHost(), port_name=self.full_port_name) 475 self.assertTrue(isinstance(port.virtual_test_suites(), collections.Iterable)) 476