15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import cPickle 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models import test_expectations 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def is_reftest_failure(failure_list): 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) failure_types = [type(f) for f in failure_list] 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types) 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# FIXME: This is backwards. Each TestFailure subclass should know what 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# test_expectation type it corresponds too. Then this method just 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# collects them all from the failure list and returns the worst one. 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)def determine_result_type(failure_list): 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Takes a set of test_failures and returns which result type best fits 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) the list of failures. "Best fits" means we use the worst type of failure. 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) Returns: 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) one of the test_expectations result types - PASS, FAIL, CRASH, etc.""" 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not failure_list or len(failure_list) == 0: 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.PASS 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 515c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) failure_types = [type(f) for f in failure_list] 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if FailureCrash in failure_types: 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.CRASH 5407a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch elif FailureLeak in failure_types: 5507a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch return test_expectations.LEAK 565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif FailureTimeout in failure_types: 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.TIMEOUT 585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif FailureEarlyExit in failure_types: 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.SKIP 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif (FailureMissingResult in failure_types or 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureMissingImage in failure_types or 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureMissingImageHash in failure_types or 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureMissingAudio in failure_types): 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.MISSING 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 6609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) is_text_failure = (FailureTextMismatch in failure_types or 6709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) FailureTestHarnessAssertion in failure_types) 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) is_image_failure = (FailureImageHashIncorrect in failure_types or 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureImageHashMismatch in failure_types) 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) is_audio_failure = (FailureAudioMismatch in failure_types) 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if is_text_failure and is_image_failure: 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.IMAGE_PLUS_TEXT 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif is_text_failure: 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.TEXT 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif is_image_failure or is_reftest_failure(failure_list): 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.IMAGE 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) elif is_audio_failure: 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return test_expectations.AUDIO 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise ValueError("unclassifiable set of failures: " 815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) + str(failure_types)) 825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestFailure(object): 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Abstract base class that defines the failure interface.""" 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def loads(s): 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Creates a TestFailure object from the specified string.""" 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return cPickle.loads(s) 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Returns a string describing the failure in more detail.""" 945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) raise NotImplementedError 955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __eq__(self, other): 975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.__class__.__name__ == other.__class__.__name__ 985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __ne__(self, other): 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.__class__.__name__ != other.__class__.__name__ 1015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __hash__(self): 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return hash(self.__class__.__name__) 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def dumps(self): 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) """Returns the string/JSON representation of a TestFailure.""" 1075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return cPickle.dumps(self) 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def driver_needs_restart(self): 110591b958dee2cf159d33a0b931e6231072eaf38d5Ben Murdoch """Returns True if we should kill the driver before the next test.""" 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return False 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureTimeout(TestFailure): 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, is_reftest=False): 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(FailureTimeout, self).__init__() 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.is_reftest = is_reftest 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "test timed out" 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def driver_needs_restart(self): 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureCrash(TestFailure): 1277242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci def __init__(self, is_reftest=False, process_name='content_shell', pid=None, has_log=False): 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(FailureCrash, self).__init__() 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.process_name = process_name 1305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.pid = pid 1315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.is_reftest = is_reftest 1327242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.has_log = has_log 1335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if self.pid: 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "%s crashed [pid=%d]" % (self.process_name, self.pid) 1375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.process_name + " crashed" 1385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def driver_needs_restart(self): 1405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return True 1415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 14307a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdochclass FailureLeak(TestFailure): 14407a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch def __init__(self, is_reftest=False, log=''): 14507a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch super(FailureLeak, self).__init__() 14607a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch self.is_reftest = is_reftest 14707a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch self.log = log 14807a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch 14907a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch def message(self): 15007a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch return "leak detected: %s" % (self.log) 15107a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch 15207a852d8c1953036774d8f3b65d18dcfea3bb4a2Ben Murdoch 1535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureMissingResult(TestFailure): 1545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "-expected.txt was missing" 1565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 15809380295ba73501a205346becac22c6978e4671dTorne (Richard Coles)class FailureTestHarnessAssertion(TestFailure): 15909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) def message(self): 16009380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) return "asserts failed" 16109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) 16209380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) 1635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureTextMismatch(TestFailure): 1645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "text diff" 1665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 16709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) 1685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureMissingImageHash(TestFailure): 1695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "-expected.png was missing an embedded checksum" 1715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureMissingImage(TestFailure): 1745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "-expected.png was missing" 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureImageHashMismatch(TestFailure): 1795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "image diff" 1815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureImageHashIncorrect(TestFailure): 1845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "-expected.png embedded checksum is incorrect" 1865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureReftestMismatch(TestFailure): 1895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, reference_filename=None): 1905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(FailureReftestMismatch, self).__init__() 1915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.reference_filename = reference_filename 1925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "reference mismatch" 1955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureReftestMismatchDidNotOccur(TestFailure): 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, reference_filename=None): 1995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(FailureReftestMismatchDidNotOccur, self).__init__() 2005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.reference_filename = reference_filename 2015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 2035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "reference mismatch didn't happen" 2045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureReftestNoImagesGenerated(TestFailure): 2075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, reference_filename=None): 2085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(FailureReftestNoImagesGenerated, self).__init__() 2095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.reference_filename = reference_filename 2105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 2125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "reference didn't generate pixel results." 2135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureMissingAudio(TestFailure): 2165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 2175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "expected audio result was missing" 2185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureAudioMismatch(TestFailure): 2215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 2225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "audio mismatch" 2235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class FailureEarlyExit(TestFailure): 2265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def message(self): 2275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return "skipped due to early exit" 2285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 2305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Convenient collection of all failure classes for anything that might 2315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# need to enumerate over them all. 2325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, 23309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) FailureTestHarnessAssertion, 2345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureTextMismatch, FailureMissingImageHash, 2355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureMissingImage, FailureImageHashMismatch, 2365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureImageHashIncorrect, FailureReftestMismatch, 2375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated, 2385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureMissingAudio, FailureAudioMismatch, 2395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) FailureEarlyExit) 240