1dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block#!/usr/bin/python 2dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# Copyright (C) 2010 Google Inc. All rights reserved. 3a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged 4dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# 5dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# Redistribution and use in source and binary forms, with or without 6dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# modification, are permitted provided that the following conditions are 7dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# met: 8dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# 9dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# * Redistributions of source code must retain the above copyright 10dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# notice, this list of conditions and the following disclaimer. 11dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# * Redistributions in binary form must reproduce the above 12dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# copyright notice, this list of conditions and the following disclaimer 13dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# in the documentation and/or other materials provided with the 14dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# distribution. 15dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# * Neither the name of Google Inc. nor the names of its 16dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# contributors may be used to endorse or promote products derived from 17dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# this software without specific prior written permission. 18dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# 19dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 31dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block"""Unit tests for run_webkit_tests.""" 32dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 33ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdochfrom __future__ import with_statement 34ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 35e78cbe89e6f337f2f1fe40315be88f742b547151Steve Blockimport codecs 3628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhuimport itertools 37f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrickimport logging 382daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdochimport os 39f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrickimport Queue 40dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockimport sys 41f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrickimport thread 42f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrickimport time 43f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrickimport threading 44dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockimport unittest 45dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 4681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochtry: 4781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch import multiprocessing 4881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdochexcept ImportError: 4981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch multiprocessing = None 5081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 5121939df44de1705786c545cd1bf519d47250322dBen Murdochfrom webkitpy.common import array_stream 525ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsenfrom webkitpy.common.system import outputcapture 53ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdochfrom webkitpy.common.system import filesystem_mock 542fc2651226baac27029e38c9d6ef883fa32084dbSteve Blockfrom webkitpy.tool import mocktool 5521939df44de1705786c545cd1bf519d47250322dBen Murdochfrom webkitpy.layout_tests import port 5621939df44de1705786c545cd1bf519d47250322dBen Murdochfrom webkitpy.layout_tests import run_webkit_tests 5728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhufrom webkitpy.layout_tests.port.test import TestPort, TestDriver 582bde8e466a4451c7319e3a072d118917957d6554Steve Blockfrom webkitpy.layout_tests.port.test_files import is_reference_html_file 5928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhufrom webkitpy.python24.versioning import compare_version 6028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhufrom webkitpy.test.skip import skip_if 61dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 6221939df44de1705786c545cd1bf519d47250322dBen Murdochfrom webkitpy.thirdparty.mock import Mock 63dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 6421939df44de1705786c545cd1bf519d47250322dBen Murdoch 65f05b935882198ccf7d81675736e3aeb089c5113aBen Murdochdef parse_args(extra_args=None, record_results=False, tests_included=False, 66f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch print_nothing=True): 6728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu extra_args = extra_args or [] 68f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch if print_nothing: 69f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch args = ['--print', 'nothing'] 70f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch else: 71f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch args = [] 7228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if not '--platform' in extra_args: 7328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu args.extend(['--platform', 'test']) 745ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen if not record_results: 7528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu args.append('--no-record-results') 762bde8e466a4451c7319e3a072d118917957d6554Steve Block if not '--child-processes' in extra_args and not '--worker-model' in extra_args: 772bde8e466a4451c7319e3a072d118917957d6554Steve Block args.extend(['--worker-model', 'inline']) 7828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu args.extend(extra_args) 79e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block if not tests_included: 80e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block # We use the glob to test that globbing works. 8128040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu args.extend(['passes', 8228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 'http/tests', 834576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang 'websocket/tests', 8428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 'failures/expected/*']) 85f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch return run_webkit_tests.parse_args(args) 86f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 87f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 88f05b935882198ccf7d81675736e3aeb089c5113aBen Murdochdef passing_run(extra_args=None, port_obj=None, record_results=False, 89ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=False, filesystem=None): 90f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch options, parsed_args = parse_args(extra_args, record_results, 91f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included) 9228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if not port_obj: 93bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen port_obj = port.get(port_name=options.platform, options=options, 942fc2651226baac27029e38c9d6ef883fa32084dbSteve Block user=mocktool.MockUser(), filesystem=filesystem) 955ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen res = run_webkit_tests.run(port_obj, options, parsed_args) 96dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block return res == 0 97dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 98e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block 99ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdochdef logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None): 100f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch options, parsed_args = parse_args(extra_args=extra_args, 101ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch record_results=record_results, 102f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=tests_included, 103f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch print_nothing=False) 1042fc2651226baac27029e38c9d6ef883fa32084dbSteve Block user = mocktool.MockUser() 105f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch if not port_obj: 106f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch port_obj = port.get(port_name=options.platform, options=options, 107ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch user=user, filesystem=filesystem) 108f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 109f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch res, buildbot_output, regular_output = run_and_capture(port_obj, options, 110f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch parsed_args) 111f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch return (res, buildbot_output, regular_output, user) 112f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 1134576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang 114f05b935882198ccf7d81675736e3aeb089c5113aBen Murdochdef run_and_capture(port_obj, options, parsed_args): 1154576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang oc = outputcapture.OutputCapture() 1164576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang try: 1174576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang oc.capture_output() 1184576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang buildbot_output = array_stream.ArrayStream() 1194576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang regular_output = array_stream.ArrayStream() 1204576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang res = run_webkit_tests.run(port_obj, options, parsed_args, 1214576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang buildbot_output=buildbot_output, 1224576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang regular_output=regular_output) 1234576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang finally: 1244576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang oc.restore_output() 125f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch return (res, buildbot_output, regular_output) 12621939df44de1705786c545cd1bf519d47250322dBen Murdoch 127dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 1282bde8e466a4451c7319e3a072d118917957d6554Steve Blockdef get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, 1292bde8e466a4451c7319e3a072d118917957d6554Steve Block filesystem=None, include_reference_html=False): 13028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu extra_args = extra_args or [] 13128040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if not tests_included: 13228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # Not including http tests since they get run out of order (that 13328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # behavior has its own test, see test_get_test_file_queue) 134f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch extra_args = ['passes', 'failures'] + extra_args 135f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch options, parsed_args = parse_args(extra_args, tests_included=True) 136f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 1372fc2651226baac27029e38c9d6ef883fa32084dbSteve Block user = mocktool.MockUser() 13828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 13928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu test_batches = [] 14028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 1412bde8e466a4451c7319e3a072d118917957d6554Steve Block 14228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu class RecordingTestDriver(TestDriver): 1434576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def __init__(self, port, worker_number): 1444576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang TestDriver.__init__(self, port, worker_number) 14528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self._current_test_batch = None 14628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 14728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu def poll(self): 14828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # So that we don't create a new driver for every test 14928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu return None 15028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 15128040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu def stop(self): 15228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self._current_test_batch = None 15328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 1544576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def run_test(self, test_input): 15528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if self._current_test_batch is None: 15628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self._current_test_batch = [] 15728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu test_batches.append(self._current_test_batch) 1584576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang test_name = self._port.relative_test_filename(test_input.filename) 1592bde8e466a4451c7319e3a072d118917957d6554Steve Block # In case of reftest, one test calls the driver's run_test() twice. 1602bde8e466a4451c7319e3a072d118917957d6554Steve Block # We should not add a reference html used by reftests to tests unless include_reference_html parameter 1612bde8e466a4451c7319e3a072d118917957d6554Steve Block # is explicitly given. 1622bde8e466a4451c7319e3a072d118917957d6554Steve Block if include_reference_html or not is_reference_html_file(test_input.filename): 1632bde8e466a4451c7319e3a072d118917957d6554Steve Block self._current_test_batch.append(test_name) 1644576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang return TestDriver.run_test(self, test_input) 16528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 16628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu class RecordingTestPort(TestPort): 1674576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def create_driver(self, worker_number): 1684576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang return RecordingTestDriver(self, worker_number) 16928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 170ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch recording_port = RecordingTestPort(options=options, user=user, filesystem=filesystem) 171f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch run_and_capture(recording_port, options, parsed_args) 17228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 17328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu if flatten_batches: 17428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu return list(itertools.chain(*test_batches)) 17528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 17628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu return test_batches 17728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 178f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 179dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockclass MainTest(unittest.TestCase): 180bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen def test_accelerated_compositing(self): 181bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen # This just tests that we recognize the command line args 182bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(passing_run(['--accelerated-compositing'])) 183bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(passing_run(['--no-accelerated-compositing'])) 184bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen 185bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen def test_accelerated_2d_canvas(self): 186bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen # This just tests that we recognize the command line args 187bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(passing_run(['--accelerated-2d-canvas'])) 188bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(passing_run(['--no-accelerated-2d-canvas'])) 189bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen 1905ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_basic(self): 1915ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(passing_run()) 192dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 1935ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_batch_size(self): 19428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu batch_tests_run = get_tests_run(['--batch-size', '2']) 19528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu for batch in batch_tests_run: 19628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch)) 197e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block 1985ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_child_process_1(self): 1992daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch _, _, regular_output, _ = logging_run( 2002bde8e466a4451c7319e3a072d118917957d6554Steve Block ['--print', 'config', '--worker-model', 'threads', '--child-processes', '1']) 2012daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()])) 20221939df44de1705786c545cd1bf519d47250322dBen Murdoch 2035ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_child_processes_2(self): 2042daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch _, _, regular_output, _ = logging_run( 2052bde8e466a4451c7319e3a072d118917957d6554Steve Block ['--print', 'config', '--worker-model', 'threads', '--child-processes', '2']) 2062daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(any(['Running 2 ' in line for line in regular_output.get()])) 2072daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 2082daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def test_child_processes_min(self): 2092daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch _, _, regular_output, _ = logging_run( 2102daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch ['--print', 'config', '--worker-model', 'threads', '--child-processes', '2', 'passes'], 2112daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests_included=True) 2122daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()])) 21321939df44de1705786c545cd1bf519d47250322dBen Murdoch 214f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def test_dryrun(self): 215f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch batch_tests_run = get_tests_run(['--dry-run']) 216f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEqual(batch_tests_run, []) 217f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 218f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch batch_tests_run = get_tests_run(['-n']) 219f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEqual(batch_tests_run, []) 220f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 2215ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_exception_raised(self): 2225ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertRaises(ValueError, logging_run, 2235ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ['failures/expected/exception.html'], tests_included=True) 2245ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2255ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_full_results_html(self): 2265ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # FIXME: verify html? 2272fc2651226baac27029e38c9d6ef883fa32084dbSteve Block res, out, err, user = logging_run(['--full-results-html']) 2282fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertEqual(res, 0) 2295ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2305ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_help_printing(self): 231bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(['--help-printing']) 2325ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertEqual(res, 0) 2335ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(out.empty()) 2345ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertFalse(err.empty()) 2355ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2365abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick def test_hung_thread(self): 237bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50', 238bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen 'failures/expected/hang.html'], 239bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen tests_included=True) 2405abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick self.assertEqual(res, 0) 2415abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick self.assertFalse(out.empty()) 2425abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick self.assertFalse(err.empty()) 2435abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 2445ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_keyboard_interrupt(self): 2455ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # Note that this also tests running a test marked as SKIP if 2465ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # you specify it explicitly. 2474576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang self.assertRaises(KeyboardInterrupt, logging_run, 2485ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ['failures/expected/keyboard.html'], tests_included=True) 2495ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 25081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_keyboard_interrupt_inline_worker_model(self): 25181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertRaises(KeyboardInterrupt, logging_run, 25281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch ['failures/expected/keyboard.html', '--worker-model', 'inline'], 25381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch tests_included=True) 25481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 2556c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen def test_last_results(self): 256ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 257ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch # We do a logging run here instead of a passing run in order to 258ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch # suppress the output from the json generator. 2592daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch res, buildbot_output, regular_output, user = logging_run(['--clobber-old-results'], record_results=True, filesystem=fs) 2602daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch res, buildbot_output, regular_output, user = logging_run( 261ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch ['--print-last-failures'], filesystem=fs) 2626c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen self.assertEqual(regular_output.get(), ['\n\n']) 2636c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen self.assertEqual(buildbot_output.get(), []) 26421939df44de1705786c545cd1bf519d47250322dBen Murdoch 2655ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_lint_test_files(self): 266f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch res, out, err, user = logging_run(['--lint-test-files']) 2675ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertEqual(res, 0) 2685ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(out.empty()) 269f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertTrue(any(['Lint succeeded' in msg for msg in err.get()])) 270f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 271f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def test_lint_test_files__errors(self): 272f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch options, parsed_args = parse_args(['--lint-test-files']) 2732fc2651226baac27029e38c9d6ef883fa32084dbSteve Block user = mocktool.MockUser() 274f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch port_obj = port.get(options.platform, options=options, user=user) 275f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch port_obj.test_expectations = lambda: "# syntax error" 276f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch res, out, err = run_and_capture(port_obj, options, parsed_args) 277f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 278f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEqual(res, -1) 279f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertTrue(out.empty()) 280f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertTrue(any(['Lint failed' in msg for msg in err.get()])) 2815ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 282e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block def test_no_tests_found(self): 283bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(['resources'], tests_included=True) 2845ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertEqual(res, -1) 2855ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(out.empty()) 2865ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue('No tests to run.\n' in err.get()) 2875ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2885ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_no_tests_found_2(self): 289bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(['foo'], tests_included=True) 2905ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertEqual(res, -1) 2915ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(out.empty()) 2925ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue('No tests to run.\n' in err.get()) 2935ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2945ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_randomize_order(self): 2955ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # FIXME: verify order was shuffled 2965ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertTrue(passing_run(['--randomize-order'])) 2975ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 2985ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_run_chunk(self): 29928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # Test that we actually select the right chunk 30028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu all_tests_run = get_tests_run(flatten_batches=True) 30128040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True) 30228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(all_tests_run[4:8], chunk_tests_run) 30328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 30428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # Test that we wrap around if the number of tests is not evenly divisible by the chunk size 30528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] 30628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True) 30728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run) 3085ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 3095ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_run_force(self): 3105ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # This raises an exception because we run 3115ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # failures/expected/exception.html, which is normally SKIPped. 3125ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertRaises(ValueError, logging_run, ['--force']) 3135ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 3145ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_run_part(self): 31528040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # Test that we actually select the right part 31628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html'] 31728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True) 31828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run) 31928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 32028040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # Test that we wrap around if the number of tests is not evenly divisible by the chunk size 32128040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the 32228040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu # last part repeats the first two tests). 32328040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True) 32428040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run) 3255ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 3265ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_run_singly(self): 32728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu batch_tests_run = get_tests_run(['--run-singly']) 32828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu for batch in batch_tests_run: 32928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch)) 3305ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 3312daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def test_run_singly_actually_runs_tests(self): 3322daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected']) 3332daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertEquals(res, 5) 3342daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 3355ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_single_file(self): 33628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True) 33728040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(['passes/text.html'], tests_run) 3385ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 339ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def test_single_file_with_prefix(self): 340ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_run = get_tests_run(['LayoutTests/passes/text.html'], tests_included=True, flatten_batches=True) 341ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertEquals(['passes/text.html'], tests_run) 342ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 343ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def test_single_skipped_file(self): 344ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_run = get_tests_run(['failures/expected/keybaord.html'], tests_included=True, flatten_batches=True) 345ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertEquals([], tests_run) 346ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 3472daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def test_stderr_is_saved(self): 3482daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch fs = port.unit_test_filesystem() 3492daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(passing_run(filesystem=fs)) 3502daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertEquals(fs.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'), 3512daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 'stuff going to stderr') 3522daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 3535abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick def test_test_list(self): 354ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 355ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch filename = '/tmp/foo.txt' 356ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs.write_text_file(filename, 'passes/text.html') 357ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs) 35828040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu self.assertEquals(['passes/text.html'], tests_run) 359ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs.remove(filename) 360bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(['--test-list=%s' % filename], 361ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True, filesystem=fs) 3625abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick self.assertEqual(res, -1) 3635abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick self.assertFalse(err.empty()) 3645abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 365ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def test_test_list_with_prefix(self): 366ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 367ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch filename = '/tmp/foo.txt' 368ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs.write_text_file(filename, 'LayoutTests/passes/text.html') 369ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs) 370ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertEquals(['passes/text.html'], tests_run) 371ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 3725ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen def test_unexpected_failures(self): 3735ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen # Run tests including the unexpected failures. 374bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self._url_opened = None 375bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen res, out, err, user = logging_run(tests_included=True) 3762bde8e466a4451c7319e3a072d118917957d6554Steve Block 3772bde8e466a4451c7319e3a072d118917957d6554Steve Block # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test 3782bde8e466a4451c7319e3a072d118917957d6554Steve Block # FIXME: It's nice to have a routine in port/test.py that returns this number. 3792bde8e466a4451c7319e3a072d118917957d6554Steve Block unexpected_tests_count = 5 3802bde8e466a4451c7319e3a072d118917957d6554Steve Block 3812bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEqual(res, unexpected_tests_count) 3825ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertFalse(out.empty()) 3835ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen self.assertFalse(err.empty()) 3842fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html']) 3855ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen 38681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_exit_after_n_failures_upload(self): 38781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch fs = port.unit_test_filesystem() 3882daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch res, buildbot_output, regular_output, user = logging_run([ 38981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 'failures/unexpected/text-image-checksum.html', 39081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 'passes/text.html', 39181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch '--exit-after-n-failures', '1', 39281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch ], 39381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch tests_included=True, 39481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch record_results=True, 39581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch filesystem=fs) 39681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertTrue('/tmp/layout-test-results/incremental_results.json' in fs.files) 39781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 398f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def test_exit_after_n_failures(self): 399f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch # Unexpected failures should result in tests stopping. 400f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_run = get_tests_run([ 401f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'failures/unexpected/text-image-checksum.html', 402f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'passes/text.html', 403f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch '--exit-after-n-failures', '1', 404f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch ], 405f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=True, 406f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch flatten_batches=True) 407f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run) 408f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 409f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch # But we'll keep going for expected ones. 410f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_run = get_tests_run([ 411f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'failures/expected/text.html', 412f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'passes/text.html', 413f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch '--exit-after-n-failures', '1', 414f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch ], 415f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=True, 416f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch flatten_batches=True) 417f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run) 418f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 419f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch def test_exit_after_n_crashes(self): 420f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch # Unexpected crashes should result in tests stopping. 421f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_run = get_tests_run([ 422f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'failures/unexpected/crash.html', 423f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'passes/text.html', 424f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch '--exit-after-n-crashes-or-timeouts', '1', 425f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch ], 426f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=True, 427f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch flatten_batches=True) 428f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEquals(['failures/unexpected/crash.html'], tests_run) 429f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 430f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch # Same with timeouts. 431f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_run = get_tests_run([ 432f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'failures/unexpected/timeout.html', 433f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'passes/text.html', 434f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch '--exit-after-n-crashes-or-timeouts', '1', 435f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch ], 436f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=True, 437f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch flatten_batches=True) 438f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEquals(['failures/unexpected/timeout.html'], tests_run) 439f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 440f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch # But we'll keep going for expected ones. 441f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_run = get_tests_run([ 442f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'failures/expected/crash.html', 443f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 'passes/text.html', 444f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch '--exit-after-n-crashes-or-timeouts', '1', 445f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch ], 446f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch tests_included=True, 447f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch flatten_batches=True) 448f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run) 449f05b935882198ccf7d81675736e3aeb089c5113aBen Murdoch 45081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_exit_after_n_crashes_inline_worker_model(self): 45181bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch tests_run = get_tests_run([ 45281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 'failures/unexpected/timeout.html', 45381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 'passes/text.html', 45481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch '--exit-after-n-crashes-or-timeouts', '1', 45581bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch '--worker-model', 'inline', 45681bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch ], 45781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch tests_included=True, 45881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch flatten_batches=True) 45981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertEquals(['failures/unexpected/timeout.html'], tests_run) 46081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 461a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch def test_results_directory_absolute(self): 462a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # We run a configuration that should fail, to generate output, then 463a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # look for what the output results url was. 464a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 465ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 466ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch with fs.mkdtemp() as tmpdir: 467ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)], 468ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True, filesystem=fs) 4692fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertEqual(user.opened_urls, [fs.join(tmpdir, 'results.html')]) 470a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 471a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch def test_results_directory_default(self): 472a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # We run a configuration that should fail, to generate output, then 473a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # look for what the output results url was. 474a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 475a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # This is the default location. 476a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch res, out, err, user = logging_run(tests_included=True) 4772fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html']) 478a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 479a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch def test_results_directory_relative(self): 480a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # We run a configuration that should fail, to generate output, then 481a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch # look for what the output results url was. 4822daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch fs = port.unit_test_filesystem() 4832daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch fs.maybe_make_directory('/tmp/cwd') 4842daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch fs.chdir('/tmp/cwd') 485a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch res, out, err, user = logging_run(['--results-directory=foo'], 4862daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch tests_included=True, filesystem=fs) 4872daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertEqual(user.opened_urls, ['/tmp/cwd/foo/results.html']) 488a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 4892bde8e466a4451c7319e3a072d118917957d6554Steve Block # These next tests test that we run the tests in ascending alphabetical 4902bde8e466a4451c7319e3a072d118917957d6554Steve Block # order per directory. HTTP tests are sharded separately from other tests, 4912bde8e466a4451c7319e3a072d118917957d6554Steve Block # so we have to test both. 4922bde8e466a4451c7319e3a072d118917957d6554Steve Block def assert_run_order(self, worker_model, child_processes='1'): 4932bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['--worker-model', worker_model, 4942bde8e466a4451c7319e3a072d118917957d6554Steve Block '--child-processes', child_processes, 'passes'], 4952bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_included=True, flatten_batches=True) 4962bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals(tests_run, sorted(tests_run)) 4972bde8e466a4451c7319e3a072d118917957d6554Steve Block 4982bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['--worker-model', worker_model, 4992bde8e466a4451c7319e3a072d118917957d6554Steve Block '--child-processes', child_processes, 'http/tests/passes'], 5002bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_included=True, flatten_batches=True) 5012bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals(tests_run, sorted(tests_run)) 5022bde8e466a4451c7319e3a072d118917957d6554Steve Block 5032bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_run_order__inline(self): 5042bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assert_run_order('inline') 5052bde8e466a4451c7319e3a072d118917957d6554Steve Block 506e14391e94c850b8bd03680c23b38978db68687a8John Reck def test_tolerance(self): 507e14391e94c850b8bd03680c23b38978db68687a8John Reck class ImageDiffTestPort(TestPort): 508e14391e94c850b8bd03680c23b38978db68687a8John Reck def diff_image(self, expected_contents, actual_contents, 509e14391e94c850b8bd03680c23b38978db68687a8John Reck diff_filename=None): 510e14391e94c850b8bd03680c23b38978db68687a8John Reck self.tolerance_used_for_diff_image = self._options.tolerance 511e14391e94c850b8bd03680c23b38978db68687a8John Reck return True 512e14391e94c850b8bd03680c23b38978db68687a8John Reck 513e14391e94c850b8bd03680c23b38978db68687a8John Reck def get_port_for_run(args): 514e14391e94c850b8bd03680c23b38978db68687a8John Reck options, parsed_args = run_webkit_tests.parse_args(args) 5152fc2651226baac27029e38c9d6ef883fa32084dbSteve Block test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser()) 51628040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu passing_run(args, port_obj=test_port, tests_included=True) 517e14391e94c850b8bd03680c23b38978db68687a8John Reck return test_port 518e14391e94c850b8bd03680c23b38978db68687a8John Reck 519e14391e94c850b8bd03680c23b38978db68687a8John Reck base_args = ['--pixel-tests', 'failures/expected/*'] 520e14391e94c850b8bd03680c23b38978db68687a8John Reck 521e14391e94c850b8bd03680c23b38978db68687a8John Reck # If we pass in an explicit tolerance argument, then that will be used. 522e14391e94c850b8bd03680c23b38978db68687a8John Reck test_port = get_port_for_run(base_args + ['--tolerance', '.1']) 523e14391e94c850b8bd03680c23b38978db68687a8John Reck self.assertEqual(0.1, test_port.tolerance_used_for_diff_image) 524e14391e94c850b8bd03680c23b38978db68687a8John Reck test_port = get_port_for_run(base_args + ['--tolerance', '0']) 525e14391e94c850b8bd03680c23b38978db68687a8John Reck self.assertEqual(0, test_port.tolerance_used_for_diff_image) 526e14391e94c850b8bd03680c23b38978db68687a8John Reck 527e14391e94c850b8bd03680c23b38978db68687a8John Reck # Otherwise the port's default tolerance behavior (including ignoring it) 528e14391e94c850b8bd03680c23b38978db68687a8John Reck # should be used. 529e14391e94c850b8bd03680c23b38978db68687a8John Reck test_port = get_port_for_run(base_args) 530e14391e94c850b8bd03680c23b38978db68687a8John Reck self.assertEqual(None, test_port.tolerance_used_for_diff_image) 531a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch 5324576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def test_worker_model__inline(self): 5332fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertTrue(passing_run(['--worker-model', 'inline'])) 5342fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 5352bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_worker_model__inline_with_child_processes(self): 5362bde8e466a4451c7319e3a072d118917957d6554Steve Block res, out, err, user = logging_run(['--worker-model', 'inline', 5372fc2651226baac27029e38c9d6ef883fa32084dbSteve Block '--child-processes', '2']) 5382fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertEqual(res, 0) 5392bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertTrue('--worker-model=inline overrides --child-processes\n' in err.get()) 5402fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 5412fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def test_worker_model__processes(self): 54281bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch # FIXME: remove this when we fix test-webkitpy to work properly 54381bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch # with the multiprocessing module (bug 54520). 54481bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if multiprocessing and sys.platform not in ('cygwin', 'win32'): 5452fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertTrue(passing_run(['--worker-model', 'processes'])) 5462fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 54781bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch def test_worker_model__processes_and_dry_run(self): 54881bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch if multiprocessing and sys.platform not in ('cygwin', 'win32'): 54981bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch self.assertTrue(passing_run(['--worker-model', 'processes', '--dry-run'])) 55081bc750723a18f21cd17d1b173cd2a4dda9cea6eBen Murdoch 5512fc2651226baac27029e38c9d6ef883fa32084dbSteve Block def test_worker_model__threads(self): 5522fc2651226baac27029e38c9d6ef883fa32084dbSteve Block self.assertTrue(passing_run(['--worker-model', 'threads'])) 5532fc2651226baac27029e38c9d6ef883fa32084dbSteve Block 5544576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang def test_worker_model__unknown(self): 5554576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang self.assertRaises(ValueError, logging_run, 5564576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang ['--worker-model', 'unknown']) 5574576aa36e9a9671459299c7963ac95aa94beaea9Shimeng (Simon) Wang 5582bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_reftest_run(self): 5592bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True) 5602bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals(['passes/reftest.html'], tests_run) 5612bde8e466a4451c7319e3a072d118917957d6554Steve Block 5622bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_reftest_expected_html_should_be_ignored(self): 5632bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True) 5642bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals([], tests_run) 5652bde8e466a4451c7319e3a072d118917957d6554Steve Block 5662bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_reftest_driver_should_run_expected_html(self): 5672bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True, 5682bde8e466a4451c7319e3a072d118917957d6554Steve Block include_reference_html=True) 5692bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run) 5702bde8e466a4451c7319e3a072d118917957d6554Steve Block 5712bde8e466a4451c7319e3a072d118917957d6554Steve Block def test_reftest_driver_should_run_expected_mismatch_html(self): 5722bde8e466a4451c7319e3a072d118917957d6554Steve Block tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True, 5732bde8e466a4451c7319e3a072d118917957d6554Steve Block include_reference_html=True) 5742bde8e466a4451c7319e3a072d118917957d6554Steve Block self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run) 5752bde8e466a4451c7319e3a072d118917957d6554Steve Block 5762daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch def test_additional_platform_directory(self): 5772daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo'])) 5782daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo'])) 5792daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', 5802daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch '--additional-platform-directory', '/tmp/bar'])) 5812daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 5822daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch res, buildbot_output, regular_output, user = logging_run( 5832daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch ['--additional-platform-directory', 'foo']) 5842daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch self.assertTrue('--additional-platform-directory=foo is ignored since it is not absolute\n' 5852daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch in regular_output.get()) 5862daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch 5872bde8e466a4451c7319e3a072d118917957d6554Steve Block 58828040489d744e0c5d475a88663056c9040ed5320Teng-Hui ZhuMainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2') 58928040489d744e0c5d475a88663056c9040ed5320Teng-Hui Zhu 590f486d19d62f1bc33246748b14b14a9dfa617b57fIain Merrick 591e78cbe89e6f337f2f1fe40315be88f742b547151Steve Blockclass RebaselineTest(unittest.TestCase): 592e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block def assertBaselines(self, file_list, file): 593e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block "assert that the file_list contains the baselines.""" 594e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block for ext in [".txt", ".png", ".checksum"]: 595e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block baseline = file + "-expected" + ext 596e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block self.assertTrue(any(f.find(baseline) != -1 for f in file_list)) 597e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block 5986b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # FIXME: Add tests to ensure that we're *not* writing baselines when we're not 5996b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # supposed to be. 6006b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner 601ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def test_reset_results(self): 6026b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # Test that we update expectations in place. If the expectation 6036b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # is missing, update the expected generic location. 604ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 6056b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner passing_run(['--pixel-tests', 6066b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner '--reset-results', 6076b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner 'passes/image.html', 6086b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner 'failures/expected/missing_image.html'], 609ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True, filesystem=fs) 610ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch file_list = fs.written_files.keys() 6112fc2651226baac27029e38c9d6ef883fa32084dbSteve Block file_list.remove('/tmp/layout-test-results/tests_run0.txt') 6126b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner self.assertEqual(len(file_list), 6) 6136b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner self.assertBaselines(file_list, 614ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch "/passes/image") 6156b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner self.assertBaselines(file_list, 616ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch "/failures/expected/missing_image") 6176b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner 618ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def test_new_baseline(self): 6196b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # Test that we update the platform expectations. If the expectation 6206b70adc33054f8aee8c54d0f460458a9df11b8a5Russell Brenner # is mssing, then create a new expectation in the platform dir. 621ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch fs = port.unit_test_filesystem() 622ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch passing_run(['--pixel-tests', 623ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch '--new-baseline', 624ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 'passes/image.html', 625ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 'failures/expected/missing_image.html'], 626ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True, filesystem=fs) 627ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch file_list = fs.written_files.keys() 6282fc2651226baac27029e38c9d6ef883fa32084dbSteve Block file_list.remove('/tmp/layout-test-results/tests_run0.txt') 629ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertEqual(len(file_list), 6) 630ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertBaselines(file_list, 6312daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch "/platform/test-mac-leopard/passes/image") 632ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertBaselines(file_list, 6332daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch "/platform/test-mac-leopard/failures/expected/missing_image") 634e78cbe89e6f337f2f1fe40315be88f742b547151Steve Block 635bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen 636dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockclass DryrunTest(unittest.TestCase): 637e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block # FIXME: it's hard to know which platforms are safe to test; the 638e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block # chromium platforms require a chromium checkout, and the mac platform 639e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block # requires fcntl, so it can't be tested on win32, etc. There is 640e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block # probably a better way of handling this. 641ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch def disabled_test_darwin(self): 642e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block if sys.platform != "darwin": 6436c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen return 644e8b154fd68f9b33be40a3590e58347f353835f5cSteve Block 645ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertTrue(passing_run(['--platform', 'dryrun', 'fast/html'], 646ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True)) 647ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch self.assertTrue(passing_run(['--platform', 'dryrun-mac', 'fast/html'], 648ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch tests_included=True)) 6496c2af9490927c3c5959b5cb07461b646f8b32f6cKristian Monsen 6505abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick def test_test(self): 651bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen self.assertTrue(passing_run(['--platform', 'dryrun-test', 652bec39347bb3bb5bf1187ccaf471d26247f28b585Kristian Monsen '--pixel-tests'])) 6535abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick 654dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block 655dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Blockif __name__ == '__main__': 656dcc8cf2e65d1aa555cce12431a16547e66b469eeSteve Block unittest.main() 657