port_testcase.py revision 2bde8e466a4451c7319e3a072d118917957d6554
15abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# Copyright (C) 2010 Google Inc. All rights reserved. 25abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# 35abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# Redistribution and use in source and binary forms, with or without 45abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# modification, are permitted provided that the following conditions are 55abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# met: 65abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# 75abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# * Redistributions of source code must retain the above copyright 85abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# notice, this list of conditions and the following disclaimer. 95abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# * Redistributions in binary form must reproduce the above 105abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# copyright notice, this list of conditions and the following disclaimer 115abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# in the documentation and/or other materials provided with the 125abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# distribution. 135abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# * Neither the name of Google Inc. nor the names of its 145abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# contributors may be used to endorse or promote products derived from 155abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# this software without specific prior written permission. 165abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# 175abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 295abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick"""Unit testing base class for Port implementations.""" 305abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 3181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochimport sys 325abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrickimport unittest 335abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 3481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch# Handle Python < 2.6 where multiprocessing isn't available. 3581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochtry: 3681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch import multiprocessing 3781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochexcept ImportError: 3881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch multiprocessing = None 3981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 40a94275402997c11dd2e778633dacf4b7e630a35dBen Murdochfrom webkitpy.tool import mocktool 41a94275402997c11dd2e778633dacf4b7e630a35dBen Murdochmock_options = mocktool.MockOptions(results_directory='layout-test-results', 42a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch use_apache=True, 43a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch configuration='Release') 445abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 454576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang# FIXME: This should be used for all ports, not just WebKit Mac. See 464576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang# https://bugs.webkit.org/show_bug.cgi?id=50043 . 475abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 485abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrickclass PortTestCase(unittest.TestCase): 495abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick """Tests the WebKit port implementation.""" 5081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def port_maker(self, platform): 5181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch """Override to return the class object of the port to be tested, 5281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch or None if a valid port object cannot be constructed on the specified 5381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch platform.""" 545abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick raise NotImplementedError() 555abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 5681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def make_port(self, options=mock_options): 5781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch """This routine should be used for tests that should only be run 5881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch when we can create a full, valid port object.""" 5981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch maker = self.port_maker(sys.platform) 6081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if not maker: 6181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return None 6281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 6381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch port = maker(options=options) 6481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if hasattr(options, "results_directory"): 6581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch port._options.results_directory = port.results_directory() 6681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return port 6781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 6881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_default_worker_model(self): 6981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch port = self.make_port() 7081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if not port: 7181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return 7281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 7381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if multiprocessing: 7481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertEqual(port.default_worker_model(), 'processes') 7581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch else: 762bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEqual(port.default_worker_model(), 'threads') 7781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 784576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def test_driver_cmd_line(self): 794576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang port = self.make_port() 804576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang if not port: 814576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang return 824576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang self.assertTrue(len(port.driver_cmd_line())) 834576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang 84ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def disabled_test_http_server(self): 855abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port = self.make_port() 865abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick if not port: 875abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick return 885abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port.start_http_server() 895abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port.stop_http_server() 905abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 91ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def disabled_test_image_diff(self): 925abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port = self.make_port() 935abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick if not port: 945abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick return 955abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 965abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick if not port.check_image_diff(): 97ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch # The port hasn't been built - don't run the tests. 985abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick return 995abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 1005abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick dir = port.layout_tests_dir() 101ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch file1 = port._filesystem.join(dir, 'fast', 'css', 'button_center.png') 102ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch contents1 = port._filesystem.read_binary_file(file1) 103ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch file2 = port._filesystem.join(dir, 'fast', 'css', 104ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 'remove-shorthand-expected.png') 105ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch contents2 = port._filesystem.read_binary_file(file2) 106ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tmpfd, tmpfile = port._filesystem.open_binary_tempfile('') 107ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tmpfd.close() 1085abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 109bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertFalse(port.diff_image(contents1, contents1)) 110bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(port.diff_image(contents1, contents2)) 1115abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 112bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(port.diff_image(contents1, contents2, tmpfile)) 1135abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 114ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch port._filesystem.remove(tmpfile) 115ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 116ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def disabled_test_websocket_server(self): 1175abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port = self.make_port() 1185abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick if not port: 1195abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick return 1205abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port.start_websocket_server() 1215abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick port.stop_websocket_server() 1222fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 1232fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def test_test_configuration(self): 1242fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port = self.make_port() 1252fc2651226baac27029e38c9d6ef883fa32084dbSteve Block if not port: 1262fc2651226baac27029e38c9d6ef883fa32084dbSteve Block return 1272fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertTrue(port.test_configuration()) 1282fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 1292fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def test_all_test_configurations(self): 1302fc2651226baac27029e38c9d6ef883fa32084dbSteve Block port = self.make_port() 1312fc2651226baac27029e38c9d6ef883fa32084dbSteve Block if not port: 1322fc2651226baac27029e38c9d6ef883fa32084dbSteve Block return 1332fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertTrue(len(port.all_test_configurations()) > 0) 13481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 13581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_baseline_search_path(self): 13681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch port = self.make_port() 13781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if not port: 13881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch return 13981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertTrue(port.baseline_path() in port.baseline_search_path()) 140