15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the Google name nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import base64 3093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)import copy 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import sys 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import time 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3451b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles)from webkitpy.layout_tests.port import DeviceFailure, Driver, DriverOutput, Port 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.port.base import VirtualTestSuite 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models.test_configuration import TestConfiguration 37f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)from webkitpy.layout_tests.models import test_run_results 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.system.filesystem_mock import MockFileSystem 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.system.crashlogs import CrashLogs 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# This sets basic expectations for a test. Each individual expectation 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# can be overridden by a keyword argument in TestList.add(). 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestInstance(object): 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, name): 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.name = name 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.base = name[(name.rfind("/") + 1):name.rfind(".")] 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.crash = False 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.web_process_crash = False 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.exception = False 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.keyboard = False 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.error = '' 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.timeout = False 545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.is_reftest = False 5551b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) self.device_failure = False 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # The values of each field are treated as raw byte strings. They 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # will be converted to unicode strings where appropriate using 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FileSystem.read_text_file(). 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.actual_text = self.base + '-txt' 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.actual_checksum = self.base + '-checksum' 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # We add the '\x8a' for the image file to prevent the value from 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # being treated as UTF-8 (the character is invalid) 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.expected_text = self.actual_text 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.expected_image = self.actual_image 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.actual_audio = None 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.expected_audio = None 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# This is an in-memory list of tests, what we want them to produce, and 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# what we want to claim are the expected results. 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestList(object): 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self): 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tests = {} 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def add(self, name, **kwargs): 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test = TestInstance(name) 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for key, value in kwargs.items(): 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test.__dict__[key] = value 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tests[name] = test 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 86e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) def add_reftest(self, name, reference_name, same_image, crash=False): 87e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True, crash=crash) 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if same_image: 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True) 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True) 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def keys(self): 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.tests.keys() 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __contains__(self, item): 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return item in self.tests 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __getitem__(self, item): 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.tests[item] 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 102926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 1031e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles)# These numbers may need to be updated whenever we add or delete tests. This includes virtual tests. 104926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 10519cde67944066db31e633d9e386f2aa9bf9fadb3Torne (Richard Coles)TOTAL_TESTS = 110 10619cde67944066db31e633d9e386f2aa9bf9fadb3Torne (Richard Coles)TOTAL_SKIPS = 28 107926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1083c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben MurdochUNEXPECTED_PASSES = 1 109bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)UNEXPECTED_FAILURES = 25 1105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def unit_test_list(): 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests = TestList() 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/crash.html', crash=True) 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/exception.html', exception=True) 11551b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) tests.add('failures/expected/device_failure.html', device_failure=True) 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/timeout.html', timeout=True) 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/missing_text.html', expected_text=None) 118591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch tests.add('failures/expected/needsrebaseline.html', actual_text='needsrebaseline text') 119e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needsmanualrebaseline text') 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/image.html', 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image='image_fail-pngtEXtchecksum\x00checksum_fail', 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_image='image-pngtEXtchecksum\x00checksum-png') 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/image_checksum.html', 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum='image_checksum_fail-checksum', 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image='image_checksum_fail-png') 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/audio.html', 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav', 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text=None, expected_text=None, 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image=None, expected_image=None, 1305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum=None) 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/keyboard.html', keyboard=True) 1325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/missing_check.html', 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_image='missing_check-png') 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/missing_image.html', expected_image=None) 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/missing_audio.html', expected_audio=None, 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text=None, expected_text=None, 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image=None, expected_image=None, 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum=None) 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/missing_text.html', expected_text=None) 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/newlines_leading.html', 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_text="\nfoo\n", actual_text="foo\n") 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/newlines_trailing.html', 1435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_text="foo\n\n", actual_text="foo\n") 1445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/newlines_with_excess_CR.html', 1455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_text="foo\r\r\r\n", actual_text="foo\n") 1465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/text.html', actual_text='text_fail-png') 147bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) tests.add('failures/expected/crash_then_text.html') 1485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/expected/skip_text.html', actual_text='text diff') 1495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/flaky/text.html') 1505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/missing_text.html', expected_text=None) 1515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png') 1525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/missing_image.html', expected_image=None) 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600 1545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RenderView at (0,0) size 800x600 1555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)layer at (0,0) size 800x34 1565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RenderBlock {HTML} at (0,0) size 800x34 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RenderBody {BODY} at (8,8) size 784x18 1585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) RenderText {#text} at (0,0) size 133x18 1595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) text run at (0,0) width 133: "This is an image test!" 1605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)""", expected_text=None) 1615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/crash.html', crash=True) 1625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/crash-with-stderr.html', crash=True, 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) error="mock-std-error-output") 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True, 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) error="mock-std-error-output") 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/pass.html') 1675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/text-checksum.html', 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text='text-checksum_fail-txt', 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum='text-checksum_fail-checksum') 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/text-image-checksum.html', 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text='text-image-checksum_fail-txt', 1725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail', 1735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum='text-image-checksum_fail-checksum') 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/checksum-with-matching-image.html', 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum='text-image-checksum_fail-checksum') 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/skip_pass.html') 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/text.html', actual_text='text_fail-txt') 1781e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) tests.add('failures/unexpected/text_then_crash.html') 1795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/timeout.html', timeout=True) 1805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('http/tests/passes/text.html') 1815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('http/tests/passes/image.html') 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('http/tests/ssl/text.html') 1835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/args.html') 1845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/error.html', error='stuff going to stderr') 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/image.html') 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/audio.html', 1875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav', 1885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text=None, expected_text=None, 1895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image=None, expected_image=None, 1905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_checksum=None) 1915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/platform_image.html') 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/checksum_in_image.html', 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_image='tEXtchecksum\x00checksum_in_image-checksum') 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/skipped/skip.html') 1955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that here the checksums don't match but the images do, so this test passes "unexpectedly". 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # See https://bugs.webkit.org/show_bug.cgi?id=69444 . 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum') 1995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Text output files contain "\r\n" on Windows. This may be 2015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling. 2025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/text.html', 2035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n') 2045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # For reftests. 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True) 207e08f70592b3fc0d5e68b9b914c9196e813720070Torne (Richard Coles) 208e08f70592b3fc0d5e68b9b914c9196e813720070Torne (Richard Coles) # This adds a different virtual reference to ensure that that also works. 209e08f70592b3fc0d5e68b9b914c9196e813720070Torne (Richard Coles) tests.add('virtual/passes/reftest-expected.html', actual_checksum='xxx', actual_image='XXX', is_reftest=True) 210e08f70592b3fc0d5e68b9b914c9196e813720070Torne (Richard Coles) 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False) 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True) 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True) 2145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False) 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False) 2165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True) 217e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) tests.add_reftest('failures/unexpected/crash-reftest.html', 'failures/unexpected/crash-reftest-expected.html', same_image=True, crash=True) 2185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False) 2195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True) 2205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True) 2215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True) 2225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/test.html') 2235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/test-ref.html') 2245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc') 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc') 2275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc') 2285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc') 2295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc') 2305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc') 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc') 2335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def') 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi') 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # The following files shouldn't be treated as reftests 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True) 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/reference/bar/common.html') 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('reftests/foo/reftest/bar/shared.html') 2405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('websocket/tests/passes/text.html') 2425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 24393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) # For testing that we don't run tests under platform/. Note that these don't contribute to TOTAL_TESTS. 2445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('platform/test-mac-leopard/http/test.html') 2455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('platform/test-win-win7/http/test.html') 2465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # For testing if perf tests are running in a locked shard. 2485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('perf/foo/test.html') 2495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('perf/foo/test-ref.html') 2505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # For testing --pixel-test-directories. 2525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html', 2535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail', 2545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png') 2555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('failures/unexpected/image_not_in_pixeldir.html', 2565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail', 2575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png') 2585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # For testing that virtual test suites don't expand names containing themselves 2605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # See webkit.org/b/97925 and base_unittest.PortTest.test_tests(). 2615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/test-virtual-passes.html') 2625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) tests.add('passes/passes/test-virtual-passes.html') 2635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return tests 2655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Here we use a non-standard location for the layout tests, to ensure that 2685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this works. The path contains a '.' in the name because we've seen bugs 2695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# related to this before. 2705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)LAYOUT_TEST_DIR = '/test.checkout/LayoutTests' 2725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)PERF_TEST_DIR = '/test.checkout/PerformanceTests' 2735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Here we synthesize an in-memory filesystem from the test list 2765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in order to fully control the test output and to demonstrate that 2775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# we don't need a real filesystem to run the tests. 2785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def add_unit_tests_to_mock_filesystem(filesystem): 2795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Add the test_expectations file. 280c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) filesystem.maybe_make_directory('/mock-checkout/LayoutTests') 281c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) if not filesystem.exists('/mock-checkout/LayoutTests/TestExpectations'): 282c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) filesystem.write_text_file('/mock-checkout/LayoutTests/TestExpectations', """ 2835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/crash.html [ Crash ] 284bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles)Bug(test) failures/expected/crash_then_text.html [ Failure ] 2855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/image.html [ ImageOnlyFailure ] 286591b958dee2cf159d33a0b931e6231072eaf38d5Ben MurdochBug(test) failures/expected/needsrebaseline.html [ NeedsRebaseline ] 287e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben MurdochBug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ] 2885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/audio.html [ Failure ] 2895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ] 2905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ] 2915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/missing_check.html [ Missing Pass ] 2925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/missing_image.html [ Missing Pass ] 2935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/missing_audio.html [ Missing Pass ] 2945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/missing_text.html [ Missing Pass ] 2955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/newlines_leading.html [ Failure ] 2965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/newlines_trailing.html [ Failure ] 2975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ] 2985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ] 2995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/text.html [ Failure ] 3005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/timeout.html [ Timeout ] 3015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/keyboard.html [ WontFix ] 3025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/expected/exception.html [ WontFix ] 30351b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles)Bug(test) failures/expected/device_failure.html [ WontFix ] 3045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) failures/unexpected/pass.html [ Failure ] 3055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)Bug(test) passes/skipped/skip.html [ Skip ] 30681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles)Bug(test) passes/text.html [ Pass ] 3075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)""") 3085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo') 3105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """ 3115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== test.html test-ref.html 3125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-match-success.html mismatching-ref.html 3145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-match-success.html matching-ref.html 3155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-match-failure.html mismatching-ref.html 3165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-match-failure.html second-mismatching-ref.html 3175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-mismatch-success.html mismatching-ref.html 3185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-mismatch-success.html second-mismatching-ref.html 3195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-mismatch-failure.html mismatching-ref.html 3205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-mismatch-failure.html matching-ref.html 3215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-both-success.html matching-ref.html 3225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-both-success.html mismatching-ref.html 3235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-both-success.html second-mismatching-ref.html 3245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)== multiple-both-failure.html matching-ref.html 3255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-both-failure.html second-mismatching-ref.html 3265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)!= multiple-both-failure.html matching-ref.html 3275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)""") 3285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: This test was only being ignored because of missing a leading '/'. 3305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Fixing the typo causes several tests to assert, so disabling the test entirely. 3315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Add in a file should be ignored by port.find_test_files(). 3325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe' 3335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def add_file(test, suffix, contents): 3355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')]) 3365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) base = test.base 3375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.maybe_make_directory(dirname) 3385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents) 3395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Add each test and the expected output, if any. 3415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_list = unit_test_list() 3425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for test in test_list.tests.values(): 3435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) add_file(test, test.name[test.name.rfind('.'):], '') 3445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if test.is_reftest: 3455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue 3465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if test.actual_audio: 3475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) add_file(test, '-expected.wav', test.expected_audio) 3485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) continue 3495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) add_file(test, '-expected.txt', test.expected_text) 3505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) add_file(test, '-expected.png', test.expected_image) 3515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg') 3535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Clear the list of written files so that we can watch what happens during testing. 3545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) filesystem.clear_written_files() 3555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestPort(Port): 3585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) port_name = 'test' 359926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) default_port_name = 'test-mac-leopard' 3605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Test implementation of the Port interface.""" 3625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ALL_BASELINE_VARIANTS = ( 3635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-linux-x86_64', 3645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-mac-snowleopard', 'test-mac-leopard', 365521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) 'test-win-win7', 'test-win-xp', 3665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ) 3675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3681e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) FALLBACK_PATHS = { 3691e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'xp': ['test-win-win7', 'test-win-xp'], 3701e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'win7': ['test-win-win7'], 3711e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'leopard': ['test-mac-leopard', 'test-mac-snowleopard'], 3721e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'snowleopard': ['test-mac-snowleopard'], 3731e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 'lucid': ['test-linux-x86_64', 'test-win-win7'], 3741e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) } 3751e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 3765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @classmethod 3775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def determine_full_port_name(cls, host, options, port_name): 3785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if port_name == 'test': 379926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return TestPort.default_port_name 3805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return port_name 3815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, host, port_name=None, **kwargs): 383926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs) 3845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._tests = unit_test_list() 3855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._flakes = set() 386c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 387c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # FIXME: crbug.com/279494. This needs to be in the "real layout tests 388c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # dir" in a mock filesystem, rather than outside of the checkout, so 389c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # that tests that want to write to a TestExpectations file can share 390c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # this between "test" ports and "real" ports. This is the result of 391c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # rebaseline_unittest.py having tests that refer to "real" port names 392c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # and real builders instead of fake builders that point back to the 393c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # test ports. rebaseline_unittest.py needs to not mix both "real" ports 394c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # and "test" ports 395c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 396c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._generic_expectations_path = '/mock-checkout/LayoutTests/TestExpectations' 3975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._results_directory = None 3985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._operating_system = 'mac' 400926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if self._name.startswith('test-win'): 4015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._operating_system = 'win' 402926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) elif self._name.startswith('test-linux'): 4035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._operating_system = 'linux' 4045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) version_map = { 4065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-win-xp': 'xp', 4075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-win-win7': 'win7', 4085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-mac-leopard': 'leopard', 4095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-mac-snowleopard': 'snowleopard', 4105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'test-linux-x86_64': 'lucid', 4115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 412926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self._version = version_map[self._name] 4135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4141e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) def repository_paths(self): 4151e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) """Returns a list of (repository_name, repository_path) tuples of its depending code base.""" 4161e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) # FIXME: We override this just to keep the perf tests happy. 4171e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) return [('blink', self.layout_tests_dir())] 4181e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 41953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def buildbot_archives_baselines(self): 42053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return self._name != 'test-win-xp' 42153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def default_pixel_tests(self): 4235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 4245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_driver(self): 4265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # This routine shouldn't normally be called, but it is called by 4275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # the mock_drt Driver. We return something, but make sure it's useless. 4285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'MOCK _path_to_driver' 4295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def default_child_processes(self): 4315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 1 4325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4339bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) def check_build(self, needs_http, printer): 434f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) return test_run_results.OK_EXIT_STATUS 4355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def check_sys_deps(self, needs_http): 437f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) return test_run_results.OK_EXIT_STATUS 4385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def default_configuration(self): 4405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return 'Release' 4415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 442591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch def diff_image(self, expected_contents, actual_contents): 4435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) diffed = actual_contents != expected_contents 4445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not actual_contents and not expected_contents: 445591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch return (None, None) 4465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not actual_contents or not expected_contents: 447591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch return (True, None) 4485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if diffed: 449591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), None) 450591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch return (None, None) 4515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def layout_tests_dir(self): 4535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return LAYOUT_TEST_DIR 4545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def perf_tests_dir(self): 4565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return PERF_TEST_DIR 4575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def webkit_base(self): 4595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return '/test.checkout' 4605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 461926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def _skipped_tests_for_unsupported_features(self, test_list): 4625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return set(['failures/expected/skip_text.html', 4635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'failures/unexpected/skip_pass.html', 4645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'virtual/skipped']) 4655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def name(self): 4675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._name 4685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def operating_system(self): 4705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._operating_system 4715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_wdiff(self): 4735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 4745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def default_results_directory(self): 4765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return '/tmp/layout-test-results' 4775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setup_test_run(self): 4795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _driver_class(self): 4825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return TestDriver 4835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def start_http_server(self, additional_dirs=None, number_of_servers=None): 4855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def start_websocket_server(self): 4885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def acquire_http_lock(self): 4915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def stop_http_server(self): 4945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def stop_websocket_server(self): 4975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 4985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def release_http_lock(self): 5005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pass 5015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_lighttpd(self): 5035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "/usr/sbin/lighttpd" 5045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_lighttpd_modules(self): 5065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "/usr/lib/lighttpd" 5075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_lighttpd_php(self): 5095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "/usr/bin/php-cgi" 5105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_apache(self): 5125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "/usr/sbin/httpd" 5135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _path_to_apache_config_file(self): 5155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf') 5165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 51753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def path_to_generic_test_expectations_file(self): 51853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return self._generic_expectations_path 5195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5201e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) def _port_specific_expectations_files(self): 5211e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) return [self._filesystem.join(self._webkit_baseline_path(d), 'TestExpectations') for d in ['test', 'test-win-xp']] 5221e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 5235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def all_test_configurations(self): 5245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Returns a sequence of the TestConfigurations the port supports.""" 5255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # By default, we assume we want to test every graphics type in 5265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # every configuration on every system. 5275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_configurations = [] 5285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for version, architecture in self._all_systems(): 5295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for build_type in self._all_build_types(): 5305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test_configurations.append(TestConfiguration( 5315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) version=version, 5325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) architecture=architecture, 5335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) build_type=build_type)) 5345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_configurations 5355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _all_systems(self): 5375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return (('leopard', 'x86'), 5385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('snowleopard', 'x86'), 5395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('xp', 'x86'), 5405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('win7', 'x86'), 5415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('lucid', 'x86'), 5425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ('lucid', 'x86_64')) 5435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _all_build_types(self): 5455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return ('debug', 'release') 5465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def configuration_specifier_macros(self): 5485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """To avoid surprises when introducing new macros, these are intentionally fixed in time.""" 549521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'win7'], 'linux': ['lucid']} 5505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def all_baseline_variants(self): 5525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.ALL_BASELINE_VARIANTS 5535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def virtual_test_suites(self): 5555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return [ 5561e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) VirtualTestSuite('passes', 'passes', ['--virtual-arg'], use_legacy_naming=True), 5571e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) VirtualTestSuite('skipped', 'failures/expected', ['--virtual-arg2'], use_legacy_naming=True), 5585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ] 5595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestDriver(Driver): 562591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch """Test/Dummy implementation of the driver interface.""" 563926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) next_pid = 1 564926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 565926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def __init__(self, *args, **kwargs): 566926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) super(TestDriver, self).__init__(*args, **kwargs) 567926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.started = False 568926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.pid = 0 5695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def cmd_line(self, pixel_tests, per_test_args): 5715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) pixel_tests_flag = '-p' if pixel_tests else '' 5725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args 5735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 57493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def run_test(self, driver_input, stop_when_done): 575926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if not self.started: 576926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.started = True 577926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.pid = TestDriver.next_pid 578926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) TestDriver.next_pid += 1 579926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 5805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) start_time = time.time() 58193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_name = driver_input.test_name 58293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) test_args = driver_input.args or [] 5835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) test = self._port._tests[test_name] 5845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if test.keyboard: 5855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise KeyboardInterrupt 5865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if test.exception: 5875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise ValueError('exception from ' + test_name) 58851b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) if test.device_failure: 58951b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) raise DeviceFailure('device failure in ' + test_name) 5905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) audio = None 5925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text = test.actual_text 593bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crash = test.crash 594bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) web_process_crash = test.web_process_crash 5955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5961e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) if 'flaky/text.html' in test_name and not test_name in self._port._flakes: 5975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._port._flakes.add(test_name) 5985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text = 'flaky text failure' 5995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 600bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) if 'crash_then_text.html' in test_name: 601bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) if test_name in self._port._flakes: 602bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) actual_text = 'text failure' 603bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) else: 604bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) self._port._flakes.add(test_name) 605bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crashed_process_name = self._port.driver_name() 606bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crashed_pid = 1 607bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crash = True 608bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) 6091e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) if 'text_then_crash.html' in test_name: 6101e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) if test_name in self._port._flakes: 6111e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) crashed_process_name = self._port.driver_name() 6121e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) crashed_pid = 1 613bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crash = True 6141e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) else: 6151e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) self._port._flakes.add(test_name) 6161e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) actual_text = 'text failure' 6171e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) 6185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if actual_text and test_args and test_name == 'passes/args.html': 6195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) actual_text = actual_text + ' ' + ' '.join(test_args) 6205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if test.actual_audio: 6225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) audio = base64.b64decode(test.actual_audio) 6235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_process_name = None 6245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_pid = None 625bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) if crash: 6265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_process_name = self._port.driver_name() 6275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_pid = 1 628bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) elif web_process_crash: 6295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_process_name = 'WebProcess' 6305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_pid = 2 6315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crash_log = '' 6335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if crashed_process_name: 6345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crash_logs = CrashLogs(self._port.host) 6355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crash_log = crash_logs.find_newest_log(crashed_process_name, None) or '' 6365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if stop_when_done: 6385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.stop() 6395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 64093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if test.actual_checksum == driver_input.image_hash: 6415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) image = None 6425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 6435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) image = test.actual_image 6445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return DriverOutput(actual_text, image, test.actual_checksum, audio, 645bfe3590b1806e3ff18f46ee3af5d4b83078f305aTorne (Richard Coles) crash=(crash or web_process_crash), crashed_process_name=crashed_process_name, 6465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) crashed_pid=crashed_pid, crash_log=crash_log, 64751b2906e11752df6c18351cf520e30522d3b53a1Torne (Richard Coles) test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid) 6485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def stop(self): 650926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.started = False 651