1#!/usr/bin/python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Unit tests for test_expectations.py."""
31
32import unittest
33
34from webkitpy.layout_tests import port
35from webkitpy.layout_tests.port import base
36from webkitpy.layout_tests.layout_package.test_expectations import *
37
38class FunctionsTest(unittest.TestCase):
39    def test_result_was_expected(self):
40        # test basics
41        self.assertEquals(result_was_expected(PASS, set([PASS]),
42                                              False, False), True)
43        self.assertEquals(result_was_expected(TEXT, set([PASS]),
44                                              False, False), False)
45
46        # test handling of FAIL expectations
47        self.assertEquals(result_was_expected(IMAGE_PLUS_TEXT, set([FAIL]),
48                                              False, False), True)
49        self.assertEquals(result_was_expected(IMAGE, set([FAIL]),
50                                              False, False), True)
51        self.assertEquals(result_was_expected(TEXT, set([FAIL]),
52                                              False, False), True)
53        self.assertEquals(result_was_expected(CRASH, set([FAIL]),
54                                              False, False), False)
55
56        # test handling of SKIPped tests and results
57        self.assertEquals(result_was_expected(SKIP, set([CRASH]),
58                                              False, True), True)
59        self.assertEquals(result_was_expected(SKIP, set([CRASH]),
60                                              False, False), False)
61
62        # test handling of MISSING results and the REBASELINE modifier
63        self.assertEquals(result_was_expected(MISSING, set([PASS]),
64                                              True, False), True)
65        self.assertEquals(result_was_expected(MISSING, set([PASS]),
66                                              False, False), False)
67
68    def test_remove_pixel_failures(self):
69        self.assertEquals(remove_pixel_failures(set([TEXT])),
70                          set([TEXT]))
71        self.assertEquals(remove_pixel_failures(set([PASS])),
72                          set([PASS]))
73        self.assertEquals(remove_pixel_failures(set([IMAGE])),
74                          set([PASS]))
75        self.assertEquals(remove_pixel_failures(set([IMAGE_PLUS_TEXT])),
76                          set([TEXT]))
77        self.assertEquals(remove_pixel_failures(set([PASS, IMAGE, CRASH])),
78                          set([PASS, CRASH]))
79
80
81class Base(unittest.TestCase):
82    # Note that all of these tests are written assuming the configuration
83    # being tested is Windows XP, Release build.
84
85    def __init__(self, testFunc, setUp=None, tearDown=None, description=None):
86        self._port = port.get('test-win-xp', None)
87        self._fs = self._port._filesystem
88        self._exp = None
89        unittest.TestCase.__init__(self, testFunc)
90
91    def get_test(self, test_name):
92        return self._fs.join(self._port.layout_tests_dir(), test_name)
93
94    def get_basic_tests(self):
95        return [self.get_test('failures/expected/text.html'),
96                self.get_test('failures/expected/image_checksum.html'),
97                self.get_test('failures/expected/crash.html'),
98                self.get_test('failures/expected/missing_text.html'),
99                self.get_test('failures/expected/image.html'),
100                self.get_test('passes/text.html')]
101
102    def get_basic_expectations(self):
103        return """
104BUG_TEST : failures/expected/text.html = TEXT
105BUG_TEST WONTFIX SKIP : failures/expected/crash.html = CRASH
106BUG_TEST REBASELINE : failures/expected/missing_image.html = MISSING
107BUG_TEST WONTFIX : failures/expected/image_checksum.html = IMAGE
108BUG_TEST WONTFIX MAC : failures/expected/image.html = IMAGE
109"""
110
111    def parse_exp(self, expectations, overrides=None, is_lint_mode=False):
112        test_config = self._port.test_configuration()
113        self._exp = TestExpectations(self._port,
114             tests=self.get_basic_tests(),
115             expectations=expectations,
116             test_config=test_config,
117             is_lint_mode=is_lint_mode,
118             overrides=overrides)
119
120    def assert_exp(self, test, result):
121        self.assertEquals(self._exp.get_expectations(self.get_test(test)),
122                          set([result]))
123
124
125class BasicTests(Base):
126    def test_basic(self):
127        self.parse_exp(self.get_basic_expectations())
128        self.assert_exp('failures/expected/text.html', TEXT)
129        self.assert_exp('failures/expected/image_checksum.html', IMAGE)
130        self.assert_exp('passes/text.html', PASS)
131        self.assert_exp('failures/expected/image.html', PASS)
132
133
134class MiscTests(Base):
135    def test_multiple_results(self):
136        self.parse_exp('BUGX : failures/expected/text.html = TEXT CRASH')
137        self.assertEqual(self._exp.get_expectations(
138            self.get_test('failures/expected/text.html')),
139            set([TEXT, CRASH]))
140
141    def test_category_expectations(self):
142        # This test checks unknown tests are not present in the
143        # expectations and that known test part of a test category is
144        # present in the expectations.
145        exp_str = """
146BUGX WONTFIX : failures/expected = IMAGE
147"""
148        self.parse_exp(exp_str)
149        test_name = 'failures/expected/unknown-test.html'
150        unknown_test = self.get_test(test_name)
151        self.assertRaises(KeyError, self._exp.get_expectations,
152                          unknown_test)
153        self.assert_exp('failures/expected/crash.html', IMAGE)
154
155    def test_get_options(self):
156        self.parse_exp(self.get_basic_expectations())
157        self.assertEqual(self._exp.get_options(
158                         self.get_test('passes/text.html')), [])
159
160    def test_expectations_json_for_all_platforms(self):
161        self.parse_exp(self.get_basic_expectations())
162        json_str = self._exp.get_expectations_json_for_all_platforms()
163        # FIXME: test actual content?
164        self.assertTrue(json_str)
165
166    def test_get_expectations_string(self):
167        self.parse_exp(self.get_basic_expectations())
168        self.assertEquals(self._exp.get_expectations_string(
169                          self.get_test('failures/expected/text.html')),
170                          'TEXT')
171
172    def test_expectation_to_string(self):
173        # Normal cases are handled by other tests.
174        self.parse_exp(self.get_basic_expectations())
175        self.assertRaises(ValueError, self._exp.expectation_to_string,
176                          -1)
177
178    def test_get_test_set(self):
179        # Handle some corner cases for this routine not covered by other tests.
180        self.parse_exp(self.get_basic_expectations())
181        s = self._exp._expected_failures.get_test_set(WONTFIX)
182        self.assertEqual(s,
183            set([self.get_test('failures/expected/crash.html'),
184                 self.get_test('failures/expected/image_checksum.html')]))
185        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH)
186        self.assertEqual(s,
187            set([self.get_test('failures/expected/crash.html')]))
188        s = self._exp._expected_failures.get_test_set(WONTFIX, CRASH,
189                                                      include_skips=False)
190        self.assertEqual(s, set([]))
191
192    def test_parse_error_fatal(self):
193        try:
194            self.parse_exp("""FOO : failures/expected/text.html = TEXT
195SKIP : failures/expected/image.html""")
196            self.assertFalse(True, "ParseError wasn't raised")
197        except ParseError, e:
198            self.assertTrue(e.fatal)
199            exp_errors = [u"Line:1 Unrecognized option 'foo' failures/expected/text.html",
200                          u"Line:2 Missing expectations. [' failures/expected/image.html']"]
201            self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
202            self.assertEqual(e.errors, exp_errors)
203
204    def test_parse_error_nonfatal(self):
205        try:
206            self.parse_exp('SKIP : failures/expected/text.html = TEXT',
207                           is_lint_mode=True)
208            self.assertFalse(True, "ParseError wasn't raised")
209        except ParseError, e:
210            self.assertFalse(e.fatal)
211            exp_errors = [u'Line:1 Test lacks BUG modifier. failures/expected/text.html']
212            self.assertEqual(str(e), '\n'.join(map(str, exp_errors)))
213            self.assertEqual(e.errors, exp_errors)
214
215    def test_overrides(self):
216        self.parse_exp("BUG_EXP: failures/expected/text.html = TEXT",
217                       "BUG_OVERRIDE : failures/expected/text.html = IMAGE")
218        self.assert_exp('failures/expected/text.html', IMAGE)
219
220    def test_overrides__duplicate(self):
221        self.assertRaises(ParseError, self.parse_exp,
222             "BUG_EXP: failures/expected/text.html = TEXT",
223             """
224BUG_OVERRIDE : failures/expected/text.html = IMAGE
225BUG_OVERRIDE : failures/expected/text.html = CRASH
226""")
227
228    def test_pixel_tests_flag(self):
229        def match(test, result, pixel_tests_enabled):
230            return self._exp.matches_an_expected_result(
231                self.get_test(test), result, pixel_tests_enabled)
232
233        self.parse_exp(self.get_basic_expectations())
234        self.assertTrue(match('failures/expected/text.html', TEXT, True))
235        self.assertTrue(match('failures/expected/text.html', TEXT, False))
236        self.assertFalse(match('failures/expected/text.html', CRASH, True))
237        self.assertFalse(match('failures/expected/text.html', CRASH, False))
238        self.assertTrue(match('failures/expected/image_checksum.html', IMAGE,
239                              True))
240        self.assertTrue(match('failures/expected/image_checksum.html', PASS,
241                              False))
242        self.assertTrue(match('failures/expected/crash.html', SKIP, False))
243        self.assertTrue(match('passes/text.html', PASS, False))
244
245    def test_more_specific_override_resets_skip(self):
246        self.parse_exp("BUGX SKIP : failures/expected = TEXT\n"
247                       "BUGX : failures/expected/text.html = IMAGE\n")
248        self.assert_exp('failures/expected/text.html', IMAGE)
249        self.assertFalse(self._port._filesystem.join(self._port.layout_tests_dir(),
250                                                     'failures/expected/text.html') in
251                         self._exp.get_tests_with_result_type(SKIP))
252
253class ExpectationSyntaxTests(Base):
254    def test_missing_expectation(self):
255        # This is missing the expectation.
256        self.assertRaises(ParseError, self.parse_exp,
257                          'BUG_TEST: failures/expected/text.html')
258
259    def test_missing_colon(self):
260        # This is missing the modifiers and the ':'
261        self.assertRaises(ParseError, self.parse_exp,
262                          'failures/expected/text.html = TEXT')
263
264    def disabled_test_too_many_colons(self):
265        # FIXME: Enable this test and fix the underlying bug.
266        self.assertRaises(ParseError, self.parse_exp,
267                          'BUG_TEST: failures/expected/text.html = PASS :')
268
269    def test_too_many_equals_signs(self):
270        self.assertRaises(ParseError, self.parse_exp,
271                          'BUG_TEST: failures/expected/text.html = TEXT = IMAGE')
272
273    def test_unrecognized_expectation(self):
274        self.assertRaises(ParseError, self.parse_exp,
275                          'BUG_TEST: failures/expected/text.html = UNKNOWN')
276
277    def test_macro(self):
278        exp_str = """
279BUG_TEST WIN-XP : failures/expected/text.html = TEXT
280"""
281        self.parse_exp(exp_str)
282        self.assert_exp('failures/expected/text.html', TEXT)
283
284
285class SemanticTests(Base):
286    def test_bug_format(self):
287        self.assertRaises(ParseError, self.parse_exp, 'BUG1234 : failures/expected/text.html = TEXT')
288
289    def test_missing_bugid(self):
290        # This should log a non-fatal error.
291        self.parse_exp('SLOW : failures/expected/text.html = TEXT')
292        self.assertEqual(
293            len(self._exp._expected_failures.get_non_fatal_errors()), 1)
294
295    def test_slow_and_timeout(self):
296        # A test cannot be SLOW and expected to TIMEOUT.
297        self.assertRaises(ParseError, self.parse_exp,
298            'BUG_TEST SLOW : failures/expected/timeout.html = TIMEOUT')
299
300    def test_rebaseline(self):
301        # Can't lint a file w/ 'REBASELINE' in it.
302        self.assertRaises(ParseError, self.parse_exp,
303            'BUG_TEST REBASELINE : failures/expected/text.html = TEXT',
304            is_lint_mode=True)
305
306    def test_duplicates(self):
307        self.assertRaises(ParseError, self.parse_exp, """
308BUG_EXP : failures/expected/text.html = TEXT
309BUG_EXP : failures/expected/text.html = IMAGE""")
310
311        self.assertRaises(ParseError, self.parse_exp,
312            self.get_basic_expectations(), overrides="""
313BUG_OVERRIDE : failures/expected/text.html = TEXT
314BUG_OVERRIDE : failures/expected/text.html = IMAGE""", )
315
316    def test_missing_file(self):
317        # This should log a non-fatal error.
318        self.parse_exp('BUG_TEST : missing_file.html = TEXT')
319        self.assertEqual(
320            len(self._exp._expected_failures.get_non_fatal_errors()), 1)
321
322
323class PrecedenceTests(Base):
324    def test_file_over_directory(self):
325        # This tests handling precedence of specific lines over directories
326        # and tests expectations covering entire directories.
327        exp_str = """
328BUGX : failures/expected/text.html = TEXT
329BUGX WONTFIX : failures/expected = IMAGE
330"""
331        self.parse_exp(exp_str)
332        self.assert_exp('failures/expected/text.html', TEXT)
333        self.assert_exp('failures/expected/crash.html', IMAGE)
334
335        exp_str = """
336BUGX WONTFIX : failures/expected = IMAGE
337BUGX : failures/expected/text.html = TEXT
338"""
339        self.parse_exp(exp_str)
340        self.assert_exp('failures/expected/text.html', TEXT)
341        self.assert_exp('failures/expected/crash.html', IMAGE)
342
343    def test_ambiguous(self):
344        self.assertRaises(ParseError, self.parse_exp, """
345BUG_TEST RELEASE : passes/text.html = PASS
346BUG_TEST WIN : passes/text.html = FAIL
347""")
348
349    def test_more_modifiers(self):
350        exp_str = """
351BUG_TEST RELEASE : passes/text.html = PASS
352BUG_TEST WIN RELEASE : passes/text.html = TEXT
353"""
354        self.assertRaises(ParseError, self.parse_exp, exp_str)
355
356    def test_order_in_file(self):
357        exp_str = """
358BUG_TEST WIN RELEASE : passes/text.html = TEXT
359BUG_TEST RELEASE : passes/text.html = PASS
360"""
361        self.assertRaises(ParseError, self.parse_exp, exp_str)
362
363    def test_version_overrides(self):
364        exp_str = """
365BUG_TEST WIN : passes/text.html = PASS
366BUG_TEST WIN XP : passes/text.html = TEXT
367"""
368        self.assertRaises(ParseError, self.parse_exp, exp_str)
369
370    def test_macro_overrides(self):
371        exp_str = """
372BUG_TEST WIN : passes/text.html = PASS
373BUG_TEST WIN-XP : passes/text.html = TEXT
374"""
375        self.assertRaises(ParseError, self.parse_exp, exp_str)
376
377
378class RebaseliningTest(Base):
379    """Test rebaselining-specific functionality."""
380    def assertRemove(self, input_expectations, tests, expected_expectations):
381        self.parse_exp(input_expectations)
382        actual_expectations = self._exp.remove_rebaselined_tests(tests)
383        self.assertEqual(expected_expectations, actual_expectations)
384
385    def test_remove(self):
386        self.assertRemove('BUGX REBASELINE : failures/expected/text.html = TEXT\n'
387                          'BUGY : failures/expected/image.html = IMAGE\n'
388                          'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n',
389                          ['failures/expected/text.html'],
390                          'BUGY : failures/expected/image.html = IMAGE\n'
391                          'BUGZ REBASELINE : failures/expected/crash.html = CRASH\n')
392
393    def test_no_get_rebaselining_failures(self):
394        self.parse_exp(self.get_basic_expectations())
395        self.assertEqual(len(self._exp.get_rebaselining_failures()), 0)
396
397
398class ModifierTests(unittest.TestCase):
399    def setUp(self):
400        port_obj = port.get('test-win-xp', None)
401        self.config = port_obj.test_configuration()
402        self.matcher = ModifierMatcher(self.config)
403
404    def match(self, modifiers, expected_num_matches=-1, values=None, num_errors=0):
405        matcher = self.matcher
406        if values:
407            matcher = ModifierMatcher(self.FakeTestConfiguration(values))
408        match_result = matcher.match(modifiers)
409        self.assertEqual(len(match_result.warnings), 0)
410        self.assertEqual(len(match_result.errors), num_errors)
411        self.assertEqual(match_result.num_matches, expected_num_matches,
412             'match(%s, %s) returned -> %d, expected %d' %
413             (modifiers, str(self.config.values()),
414              match_result.num_matches, expected_num_matches))
415
416    def test_bad_match_modifier(self):
417        self.match(['foo'], num_errors=1)
418
419    def test_none(self):
420        self.match([], 0)
421
422    def test_one(self):
423        self.match(['xp'], 1)
424        self.match(['win'], 1)
425        self.match(['release'], 1)
426        self.match(['cpu'], 1)
427        self.match(['x86'], 1)
428        self.match(['leopard'], -1)
429        self.match(['gpu'], -1)
430        self.match(['debug'], -1)
431
432    def test_two(self):
433        self.match(['xp', 'release'], 2)
434        self.match(['win7', 'release'], -1)
435        self.match(['win7', 'xp'], 1)
436
437    def test_three(self):
438        self.match(['win7', 'xp', 'release'], 2)
439        self.match(['xp', 'debug', 'x86'], -1)
440        self.match(['xp', 'release', 'x86'], 3)
441        self.match(['xp', 'cpu', 'release'], 3)
442
443    def test_four(self):
444        self.match(['xp', 'release', 'cpu', 'x86'], 4)
445        self.match(['win7', 'xp', 'release', 'cpu'], 3)
446        self.match(['win7', 'xp', 'debug', 'cpu'], -1)
447
448    def test_case_insensitivity(self):
449        self.match(['Win'], num_errors=1)
450        self.match(['WIN'], num_errors=1)
451        self.match(['win'], 1)
452
453    def test_duplicates(self):
454        self.match(['release', 'release'], num_errors=1)
455        self.match(['win-xp', 'xp'], num_errors=1)
456        self.match(['win-xp', 'win-xp'], num_errors=1)
457        self.match(['xp', 'release', 'xp', 'release'], num_errors=2)
458        self.match(['rebaseline', 'rebaseline'], num_errors=1)
459
460    def test_unknown_option(self):
461        self.match(['vms'], num_errors=1)
462
463    def test_duplicate_bugs(self):
464        # BUG* regexes can appear multiple times.
465        self.match(['bugfoo', 'bugbar'], 0)
466
467    def test_invalid_combinations(self):
468        # FIXME: This should probably raise an error instead of NO_MATCH.
469        self.match(['mac', 'xp'], num_errors=0)
470
471    def test_regexes_are_ignored(self):
472        self.match(['bug123xy', 'rebaseline', 'wontfix', 'slow', 'skip'], 0)
473
474    def test_none_is_invalid(self):
475        self.match(['none'], num_errors=1)
476
477
478if __name__ == '__main__':
479    unittest.main()
480