1# Copyright (C) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import unittest
30
31from webkitpy.common.host_mock import MockHost
32from webkitpy.layout_tests.models import test_expectations
33from webkitpy.layout_tests.models import test_failures
34from webkitpy.layout_tests.models import test_results
35from webkitpy.layout_tests.models import test_run_results
36
37
38def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
39    failures = []
40    if result_type == test_expectations.TIMEOUT:
41        failures = [test_failures.FailureTimeout()]
42    elif result_type == test_expectations.AUDIO:
43        failures = [test_failures.FailureAudioMismatch()]
44    elif result_type == test_expectations.CRASH:
45        failures = [test_failures.FailureCrash()]
46    elif result_type == test_expectations.LEAK:
47        failures = [test_failures.FailureLeak()]
48    return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
49
50
51def run_results(port, extra_skipped_tests=[]):
52    tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/leak.html', 'failures/expected/keyboard.html',
53             'failures/expected/audio.html', 'passes/skipped/skip.html']
54    expectations = test_expectations.TestExpectations(port, tests)
55    if extra_skipped_tests:
56        expectations.add_extra_skipped_tests(extra_skipped_tests)
57    return test_run_results.TestRunResults(expectations, len(tests))
58
59
60def summarized_results(port, expected, passing, flaky, only_include_failing=False, extra_skipped_tests=[]):
61    test_is_slow = False
62
63    initial_results = run_results(port, extra_skipped_tests)
64    if expected:
65        initial_results.add(get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow)
66        initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow)
67        initial_results.add(get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow)
68        initial_results.add(get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow)
69        initial_results.add(get_result('failures/expected/leak.html', test_expectations.LEAK), expected, test_is_slow)
70    elif passing:
71        skipped_result = get_result('passes/skipped/skip.html')
72        skipped_result.type = test_expectations.SKIP
73        initial_results.add(skipped_result, expected, test_is_slow)
74
75        initial_results.add(get_result('passes/text.html', run_time=1), expected, test_is_slow)
76        initial_results.add(get_result('failures/expected/audio.html'), expected, test_is_slow)
77        initial_results.add(get_result('failures/expected/timeout.html'), expected, test_is_slow)
78        initial_results.add(get_result('failures/expected/crash.html'), expected, test_is_slow)
79        initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow)
80    else:
81        initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT, run_time=1), expected, test_is_slow)
82        initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO, run_time=0.049), expected, test_is_slow)
83        initial_results.add(get_result('failures/expected/timeout.html', test_expectations.CRASH, run_time=0.05), expected, test_is_slow)
84        initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
85        initial_results.add(get_result('failures/expected/leak.html', test_expectations.TIMEOUT), expected, test_is_slow)
86
87        # we only list keyboard.html here, since normally this is WontFix
88        initial_results.add(get_result('failures/expected/keyboard.html', test_expectations.SKIP), expected, test_is_slow)
89
90    if flaky:
91        retry_results = run_results(port, extra_skipped_tests)
92        retry_results.add(get_result('passes/text.html'), True, test_is_slow)
93        retry_results.add(get_result('failures/expected/timeout.html'), True, test_is_slow)
94        retry_results.add(get_result('failures/expected/crash.html'), True, test_is_slow)
95        retry_results.add(get_result('failures/expected/leak.html'), True, test_is_slow)
96    else:
97        retry_results = None
98
99    return test_run_results.summarize_results(port, initial_results.expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=False, only_include_failing=only_include_failing)
100
101
102class InterpretTestFailuresTest(unittest.TestCase):
103    def setUp(self):
104        host = MockHost()
105        self.port = host.port_factory.get(port_name='test')
106
107    def test_interpret_test_failures(self):
108        test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
109        self.assertEqual(len(test_dict), 0)
110
111        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()])
112        self.assertIn('is_missing_audio', test_dict)
113
114        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingResult()])
115        self.assertIn('is_missing_text', test_dict)
116
117        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImage()])
118        self.assertIn('is_missing_image', test_dict)
119
120        test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImageHash()])
121        self.assertIn('is_missing_image', test_dict)
122
123
124class SummarizedResultsTest(unittest.TestCase):
125    def setUp(self):
126        host = MockHost(initialize_scm_by_default=False)
127        self.port = host.port_factory.get(port_name='test')
128
129    def test_no_svn_revision(self):
130        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
131        self.assertNotIn('revision', summary)
132
133    def test_num_failures_by_type(self):
134        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
135        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 3, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1})
136
137        summary = summarized_results(self.port, expected=True, passing=False, flaky=False)
138        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 1, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0})
139
140        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
141        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 5, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0})
142
143    def test_svn_revision(self):
144        self.port._options.builder_name = 'dummy builder'
145        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
146        self.assertNotEquals(summary['blink_revision'], '')
147
148    def test_bug_entry(self):
149        self.port._options.builder_name = 'dummy builder'
150        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
151        self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['bugs'], ['Bug(test)'])
152
153    def test_extra_skipped_tests(self):
154        self.port._options.builder_name = 'dummy builder'
155        summary = summarized_results(self.port, expected=False, passing=True, flaky=False, extra_skipped_tests=['passes/text.html'])
156        self.assertEquals(summary['tests']['passes']['text.html']['expected'], 'NOTRUN')
157
158    def test_summarized_results_wontfix(self):
159        self.port._options.builder_name = 'dummy builder'
160        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
161        self.assertEquals(summary['tests']['failures']['expected']['keyboard.html']['expected'], 'WONTFIX')
162        self.assertTrue(summary['tests']['passes']['text.html']['is_unexpected'])
163
164    def test_summarized_results_expected_pass(self):
165        self.port._options.builder_name = 'dummy builder'
166        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
167        self.assertTrue(summary['tests']['passes']['text.html'])
168        self.assertTrue('is_unexpected' not in summary['tests']['passes']['text.html'])
169
170    def test_summarized_results_expected_only_include_failing(self):
171        self.port._options.builder_name = 'dummy builder'
172        summary = summarized_results(self.port, expected=True, passing=False, flaky=False, only_include_failing=True)
173        self.assertNotIn('passes', summary['tests'])
174        self.assertTrue(summary['tests']['failures']['expected']['audio.html'])
175        self.assertTrue(summary['tests']['failures']['expected']['timeout.html'])
176        self.assertTrue(summary['tests']['failures']['expected']['crash.html'])
177        self.assertTrue(summary['tests']['failures']['expected']['leak.html'])
178
179    def test_summarized_results_skipped(self):
180        self.port._options.builder_name = 'dummy builder'
181        summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
182        self.assertEquals(summary['tests']['passes']['skipped']['skip.html']['expected'], 'SKIP')
183
184    def test_summarized_results_only_inlude_failing(self):
185        self.port._options.builder_name = 'dummy builder'
186        summary = summarized_results(self.port, expected=False, passing=True, flaky=False, only_include_failing=True)
187        self.assertTrue('passes' not in summary['tests'])
188
189    def test_rounded_run_times(self):
190        summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
191        self.assertEquals(summary['tests']['passes']['text.html']['time'], 1)
192        self.assertTrue('time' not in summary['tests']['failures']['expected']['audio.html'])
193        self.assertEquals(summary['tests']['failures']['expected']['timeout.html']['time'], 0.1)
194        self.assertTrue('time' not in summary['tests']['failures']['expected']['crash.html'])
195        self.assertTrue('time' not in summary['tests']['failures']['expected']['leak.html'])
196
197    def test_timeout_then_unexpected_pass(self):
198        tests = ['failures/expected/image.html']
199        expectations = test_expectations.TestExpectations(self.port, tests)
200        initial_results = test_run_results.TestRunResults(expectations, len(tests))
201        initial_results.add(get_result('failures/expected/image.html', test_expectations.TIMEOUT, run_time=1), False, False)
202        retry_results = test_run_results.TestRunResults(expectations, len(tests))
203        retry_results.add(get_result('failures/expected/image.html', test_expectations.PASS, run_time=0.1), False, False)
204        summary = test_run_results.summarize_results(self.port, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry=True, only_include_failing=True)
205        self.assertEquals(summary['num_regressions'], 0)
206        self.assertEquals(summary['num_passes'], 1)
207