1926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 2926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 3926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# Redistribution and use in source and binary forms, with or without 4926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# modification, are permitted provided that the following conditions are 5926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# met: 6926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 7926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Redistributions of source code must retain the above copyright 8926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# notice, this list of conditions and the following disclaimer. 9926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Redistributions in binary form must reproduce the above 10926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 11926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# in the documentation and/or other materials provided with the 12926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# distribution. 13926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 14926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# contributors may be used to endorse or promote products derived from 15926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# this software without specific prior written permission. 16926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# 17926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 29e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles)import unittest 30926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 31926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.common.host_mock import MockHost 32926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_expectations 33926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_failures 34926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_results 35926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.layout_tests.models import test_run_results 36926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 37926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 38926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)def get_result(test_name, result_type=test_expectations.PASS, run_time=0): 39926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) failures = [] 40926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if result_type == test_expectations.TIMEOUT: 41926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) failures = [test_failures.FailureTimeout()] 42926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) elif result_type == test_expectations.AUDIO: 43926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) failures = [test_failures.FailureAudioMismatch()] 44926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) elif result_type == test_expectations.CRASH: 45926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) failures = [test_failures.FailureCrash()] 466f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch elif result_type == test_expectations.LEAK: 476f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch failures = [test_failures.FailureLeak()] 48926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return test_results.TestResult(test_name, failures=failures, test_run_time=run_time) 49926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 50926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 5193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)def run_results(port, extra_skipped_tests=[]): 526f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html', 5381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 'failures/expected/audio.html', 'passes/skipped/skip.html'] 54926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) expectations = test_expectations.TestExpectations(port, tests) 5593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) if extra_skipped_tests: 5693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) expectations.add_extra_skipped_tests(extra_skipped_tests) 57926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) return test_run_results.TestRunResults(expectations, len(tests)) 58926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 59926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 6093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles)def summarized_results(port, expected, passing, flaky, only_include_failing=False, extra_skipped_tests=[]): 61926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_is_slow = False 62926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 6393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) initial_results = run_results(port, extra_skipped_tests) 64926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if expected: 65926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow) 66926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow) 67926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow) 68926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow) 696f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch initial_results.add(get_result('failures/expected/leak.html', test_expectations.LEAK), expected, test_is_slow) 70926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) elif passing: 7181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) skipped_result = get_result('passes/skipped/skip.html') 7281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) skipped_result.type = test_expectations.SKIP 7381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) initial_results.add(skipped_result, expected, test_is_slow) 74e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) 75e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) initial_results.add(get_result('passes/text.html', run_time=1), expected, test_is_slow) 76926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/audio.html'), expected, test_is_slow) 77926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/timeout.html'), expected, test_is_slow) 78926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/crash.html'), expected, test_is_slow) 796f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow) 80926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else: 81e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT, run_time=1), expected, test_is_slow) 82e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO, run_time=0.049), expected, test_is_slow) 83e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) initial_results.add(get_result('failures/expected/timeout.html', test_expectations.CRASH, run_time=0.05), expected, test_is_slow) 84926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow) 856f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch initial_results.add(get_result('failures/expected/leak.html', test_expectations.TIMEOUT), expected, test_is_slow) 86926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 87e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) # we only list keyboard.html here, since normally this is WontFix 88e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) initial_results.add(get_result('failures/expected/keyboard.html', test_expectations.SKIP), expected, test_is_slow) 89926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 90926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) if flaky: 9193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) retry_results = run_results(port, extra_skipped_tests) 92926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) retry_results.add(get_result('passes/text.html'), True, test_is_slow) 93926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) retry_results.add(get_result('failures/expected/timeout.html'), True, test_is_slow) 94926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) retry_results.add(get_result('failures/expected/crash.html'), True, test_is_slow) 956f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch retry_results.add(get_result('failures/expected/leak.html'), True, test_is_slow) 96926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) else: 97926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) retry_results = None 98926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 9993ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) return test_run_results.summarize_results(port, initial_results.expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_include_failing=only_include_failing) 100926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 101926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 102926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class InterpretTestFailuresTest(unittest.TestCase): 103926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def setUp(self): 104926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) host = MockHost() 105926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.port = host.port_factory.get(port_name='test') 106926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 107926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def test_interpret_test_failures(self): 108926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))]) 109926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(len(test_dict), 0) 110926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 111926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()]) 112926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIn('is_missing_audio', test_dict) 113926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 114926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingResult()]) 115926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIn('is_missing_text', test_dict) 116926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 117926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImage()]) 118926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIn('is_missing_image', test_dict) 119926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 120926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImageHash()]) 121926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIn('is_missing_image', test_dict) 122926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 123926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 124926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)class SummarizedResultsTest(unittest.TestCase): 125926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def setUp(self): 126926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) host = MockHost(initialize_scm_by_default=False) 127926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.port = host.port_factory.get(port_name='test') 128926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 129926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def test_no_svn_revision(self): 130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 131926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertNotIn('revision', summary) 132926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 13381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) def test_num_failures_by_type(self): 13481a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 1356f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1}) 13681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 13781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) summary = summarized_results(self.port, expected=True, passing=False, flaky=False) 1386f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0}) 13981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 14081a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 1416f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0}) 14281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 143926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def test_svn_revision(self): 144926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.port._options.builder_name = 'dummy builder' 145926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 14681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.assertNotEquals(summary['blink_revision'], '') 147926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 14893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def test_bug_entry(self): 14993ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.port._options.builder_name = 'dummy builder' 15093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 15193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bugs'], ['Bug(test)']) 15293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 15393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def test_extra_skipped_tests(self): 15493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.port._options.builder_name = 'dummy builder' 15593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False, extra_skipped_tests=['passes/text.html']) 15693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'NOTRUN') 15793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 158926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) def test_summarized_results_wontfix(self): 159926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.port._options.builder_name = 'dummy builder' 160926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 161e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'WONTFIX') 16281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected']) 16381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 16481a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) def test_summarized_results_expected_pass(self): 16581a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.port._options.builder_name = 'dummy builder' 16681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 16781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.assertTrue(summary['tests']['passes']['text.html']) 16881a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html']) 16981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 17093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def test_summarized_results_expected_only_include_failing(self): 17193ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.port._options.builder_name = 'dummy builder' 17293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) summary = summarized_results(self.port, expected=True, passing=False, flaky=False, only_include_failing=True) 17393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertNotIn('passes', summary['tests']) 17493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertTrue(summary['tests']['failures']['expected']['audio.html']) 17593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertTrue(summary['tests']['failures']['expected']['timeout.html']) 17693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertTrue(summary['tests']['failures']['expected']['crash.html']) 1776f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch self.assertTrue(summary['tests']['failures']['expected']['leak.html']) 17893ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 17981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) def test_summarized_results_skipped(self): 18081a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.port._options.builder_name = 'dummy builder' 18181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False) 18293ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['expected'], 'SKIP') 18393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 18493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def test_summarized_results_only_inlude_failing(self): 18593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.port._options.builder_name = 'dummy builder' 18693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=True, flaky=False, only_include_failing=True) 18793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertTrue('passes' not in summary['tests']) 188e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) 189e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) def test_rounded_run_times(self): 190e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) summary = summarized_results(self.port, expected=False, passing=False, flaky=False) 191e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) self.assertEquals(summary['tests']['passes']['text.html']['time'], 1) 192e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) self.assertTrue('time' not in summary['tests']['failures']['expected']['audio.html']) 193e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['time'], 0.1) 194e52495584422c5edb5b2944981473a2e208da323Torne (Richard Coles) self.assertTrue('time' not in summary['tests']['failures']['expected']['crash.html']) 1956f543c786fc42989f552b4daa774ca5ff32fa697Ben Murdoch self.assertTrue('time' not in summary['tests']['failures']['expected']['leak.html']) 19609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) 19709380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) def test_timeout_then_unexpected_pass(self): 19809380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) tests = ['failures/expected/image.html'] 19909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) expectations = test_expectations.TestExpectations(self.port, tests) 20009380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) initial_results = test_run_results.TestRunResults(expectations, len(tests)) 20109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) initial_results.add(get_result('failures/expected/image.html', test_expectations.TIMEOUT, run_time=1), False, False) 20209380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) retry_results = test_run_results.TestRunResults(expectations, len(tests)) 20309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) retry_results.add(get_result('failures/expected/image.html', test_expectations.PASS, run_time=0.1), False, False) 20409380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) summary = test_run_results.summarize_results(self.port, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_failing=True) 20509380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertEquals(summary['num_regressions'], 0) 20609380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertEquals(summary['num_passes'], 1) 207