1#!/usr/bin/python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the name of Google Inc. nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Unit tests for rebaseline_chromium_webkit_tests.py."""
31
32import unittest
33
34from webkitpy.tool import mocktool
35from webkitpy.common.system import urlfetcher_mock
36from webkitpy.common.system import filesystem_mock
37from webkitpy.common.system import zipfileset_mock
38from webkitpy.common.system import outputcapture
39from webkitpy.common.system.executive import Executive, ScriptError
40
41from webkitpy.layout_tests import port
42from webkitpy.layout_tests import rebaseline_chromium_webkit_tests
43
44
45class MockPort(object):
46    def __init__(self, image_diff_exists):
47        self.image_diff_exists = image_diff_exists
48
49    def check_image_diff(self, override_step, logging):
50        return self.image_diff_exists
51
52
53def get_mock_get(config_expectations):
54    def mock_get(port_name, options):
55        return MockPort(config_expectations[options.configuration])
56    return mock_get
57
58
59ARCHIVE_URL = 'http://localhost/layout_test_results'
60
61
62def test_options():
63    return mocktool.MockOptions(configuration=None,
64                                backup=False,
65                                html_directory='/tmp',
66                                archive_url=ARCHIVE_URL,
67                                force_archive_url=None,
68                                verbose=False,
69                                quiet=False,
70                                platforms=None)
71
72
73def test_host_port_and_filesystem(options, expectations):
74    filesystem = port.unit_test_filesystem()
75    host_port_obj = port.get('test', options, filesystem=filesystem,
76                             user=mocktool.MockUser())
77
78    expectations_path = host_port_obj.path_to_test_expectations_file()
79    filesystem.write_text_file(expectations_path, expectations)
80    return (host_port_obj, filesystem)
81
82
83def test_url_fetcher(filesystem):
84    urls = {
85        ARCHIVE_URL + '/Webkit_Mac10_6/': '<a href="4/">',
86        ARCHIVE_URL + '/Webkit_Mac10_5/': '<a href="1/"><a href="2/">',
87        ARCHIVE_URL + '/Webkit_Win7/': '<a href="1/">',
88        ARCHIVE_URL + '/Webkit_Vista/': '<a href="1/">',
89        ARCHIVE_URL + '/Webkit_Win/': '<a href="1/">',
90        ARCHIVE_URL + '/Webkit_Linux/': '<a href="1/">',
91    }
92    return urlfetcher_mock.make_fetcher_cls(urls)(filesystem)
93
94
95def test_zip_factory():
96    ziphashes = {
97        ARCHIVE_URL + '/Webkit_Mac10_5/2/layout-test-results.zip': {
98            'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
99            'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
100            'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
101            'layout-test-results/failures/expected/image_checksum-actual.txt': 'png-comment-txt',
102            'layout-test-results/failures/expected/image_checksum-actual.checksum': '0123456789',
103            'layout-test-results/failures/expected/image_checksum-actual.png': 'tEXtchecksum\x000123456789',
104        },
105        ARCHIVE_URL + '/Webkit_Mac10_6/4/layout-test-results.zip': {
106            'layout-test-results/failures/expected/image-actual.txt': 'new-image-txt',
107            'layout-test-results/failures/expected/image-actual.checksum': 'new-image-checksum',
108            'layout-test-results/failures/expected/image-actual.png': 'new-image-png',
109        },
110         ARCHIVE_URL + '/Webkit_Vista/1/layout-test-results.zip': {
111            'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
112            'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
113            'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
114        },
115          ARCHIVE_URL + '/Webkit_Win7/1/layout-test-results.zip': {
116            'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
117            'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
118            'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
119        },
120          ARCHIVE_URL + '/Webkit_Win/1/layout-test-results.zip': {
121            'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
122            'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
123            'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
124        },
125          ARCHIVE_URL + '/Webkit_Linux/1/layout-test-results.zip': {
126            'layout-test-results/failures/expected/image-actual.txt': 'win-image-txt',
127            'layout-test-results/failures/expected/image-actual.checksum': 'win-image-checksum',
128            'layout-test-results/failures/expected/image-actual.png': 'win-image-png',
129        },
130    }
131    return zipfileset_mock.make_factory(ziphashes)
132
133
134def test_archive(orig_archive_dict):
135    new_archive_dict = {}
136    for platform, dirname in orig_archive_dict.iteritems():
137        platform = platform.replace('chromium', 'test')
138        new_archive_dict[platform] = dirname
139    return new_archive_dict
140
141
142class TestGetHostPortObject(unittest.TestCase):
143    def assert_result(self, release_present, debug_present, valid_port_obj):
144        # Tests whether we get a valid port object returned when we claim
145        # that Image diff is (or isn't) present in the two configs.
146        port.get = get_mock_get({'Release': release_present,
147                                 'Debug': debug_present})
148        options = mocktool.MockOptions(configuration=None,
149                                       html_directory='/tmp')
150        port_obj = rebaseline_chromium_webkit_tests.get_host_port_object(options)
151        if valid_port_obj:
152            self.assertNotEqual(port_obj, None)
153        else:
154            self.assertEqual(port_obj, None)
155
156    def test_get_host_port_object(self):
157        # Save the normal port.get() function for future testing.
158        old_get = port.get
159
160        # Test whether we get a valid port object back for the four
161        # possible cases of having ImageDiffs built. It should work when
162        # there is at least one binary present.
163        self.assert_result(False, False, False)
164        self.assert_result(True, False, True)
165        self.assert_result(False, True, True)
166        self.assert_result(True, True, True)
167
168        # Restore the normal port.get() function.
169        port.get = old_get
170
171
172class TestOptions(unittest.TestCase):
173    def test_parse_options(self):
174        (options, target_options) = rebaseline_chromium_webkit_tests.parse_options([])
175        self.assertTrue(target_options.chromium)
176        self.assertEqual(options.tolerance, 0)
177
178        (options, target_options) = rebaseline_chromium_webkit_tests.parse_options(['--target-platform', 'qt'])
179        self.assertFalse(hasattr(target_options, 'chromium'))
180        self.assertEqual(options.tolerance, 0)
181
182
183class TestRebaseliner(unittest.TestCase):
184    def setUp(self):
185        if not hasattr(self, '_orig_archive'):
186            self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
187            rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
188
189    def tearDown(self):
190        rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
191
192    def make_rebaseliner(self, expectations):
193        options = test_options()
194        host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
195
196        target_options = options
197        target_port_obj = port.get('test', target_options,
198                                   filesystem=filesystem)
199        target_port_obj._expectations = expectations
200        platform = target_port_obj.name()
201
202        url_fetcher = test_url_fetcher(filesystem)
203        zip_factory = test_zip_factory()
204        mock_scm = mocktool.MockSCM(filesystem)
205        rebaseliner = rebaseline_chromium_webkit_tests.Rebaseliner(host_port_obj,
206            target_port_obj, platform, options, url_fetcher, zip_factory, mock_scm)
207        return rebaseliner, filesystem
208
209    def test_noop(self):
210        # this method tests that was can at least instantiate an object, even
211        # if there is nothing to do.
212        rebaseliner, filesystem = self.make_rebaseliner("")
213        rebaseliner.run()
214        self.assertEqual(len(filesystem.written_files), 1)
215
216    def test_rebaselining_tests(self):
217        rebaseliner, filesystem = self.make_rebaseliner(
218            "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
219        compile_success = rebaseliner._compile_rebaselining_tests()
220        self.assertTrue(compile_success)
221        self.assertEqual(set(['failures/expected/image.html']), rebaseliner._rebaselining_tests)
222
223    def test_rebaselining_tests_should_ignore_reftests(self):
224        rebaseliner, filesystem = self.make_rebaseliner(
225            "BUGX REBASELINE : failures/expected/reftest.html = IMAGE")
226        compile_success = rebaseliner._compile_rebaselining_tests()
227        self.assertFalse(compile_success)
228        self.assertFalse(rebaseliner._rebaselining_tests)
229
230    def test_one_platform(self):
231        rebaseliner, filesystem = self.make_rebaseliner(
232            "BUGX REBASELINE MAC : failures/expected/image.html = IMAGE")
233        rebaseliner.run()
234        # We expect to have written 12 files over the course of this rebaseline:
235        # *) 3 files in /__im_tmp for the extracted archive members
236        # *) 3 new baselines under '/test.checkout/LayoutTests'
237        # *) 4 files in /tmp for the new and old baselines in the result file
238        #    (-{old,new}.{txt,png}
239        # *) 1 text diff in /tmp for the result file (-diff.txt). We don't
240        #    create image diffs (FIXME?) and don't display the checksums.
241        # *) 1 updated test_expectations file
242        self.assertEqual(len(filesystem.written_files), 12)
243        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
244        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
245        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
246
247    def test_all_platforms(self):
248        rebaseliner, filesystem = self.make_rebaseliner(
249            "BUGX REBASELINE : failures/expected/image.html = IMAGE")
250        rebaseliner.run()
251        # See comment in test_one_platform for an explanation of the 12 written tests.
252        # Note that even though the rebaseline is marked for all platforms, each
253        # rebaseliner only ever does one.
254        self.assertEqual(len(filesystem.written_files), 12)
255        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.checksum'], 'new-image-checksum')
256        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.png'], 'new-image-png')
257        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image-expected.txt'], 'new-image-txt')
258
259    def test_png_file_with_comment(self):
260        rebaseliner, filesystem = self.make_rebaseliner(
261            "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
262        compile_success = rebaseliner._compile_rebaselining_tests()
263        self.assertTrue(compile_success)
264        self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
265        rebaseliner.run()
266        # There is one less file written than |test_one_platform| because we only
267        # write 2 expectations (the png and the txt file).
268        self.assertEqual(len(filesystem.written_files), 11)
269        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
270        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
271        self.assertFalse(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None))
272
273    def test_png_file_with_comment_remove_old_checksum(self):
274        rebaseliner, filesystem = self.make_rebaseliner(
275            "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
276        filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'] = 'old'
277        filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum'] = 'old'
278        filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'] = 'old'
279
280        compile_success = rebaseliner._compile_rebaselining_tests()
281        self.assertTrue(compile_success)
282        self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
283        rebaseliner.run()
284        # There is one more file written than |test_png_file_with_comment_remove_old_checksum|
285        # because we also delete the old checksum.
286        self.assertEqual(len(filesystem.written_files), 12)
287        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png'], 'tEXtchecksum\x000123456789')
288        self.assertEqual(filesystem.files['/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt'], 'png-comment-txt')
289        self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
290
291    def test_png_file_with_comment_as_duplicate(self):
292        rebaseliner, filesystem = self.make_rebaseliner(
293            "BUGX REBASELINE MAC : failures/expected/image_checksum.html = IMAGE")
294        filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.png'] = 'tEXtchecksum\x000123456789'
295        filesystem.files['/test.checkout/LayoutTests/platform/test-mac-snowleopard/failures/expected/image_checksum-expected.txt'] = 'png-comment-txt'
296
297        compile_success = rebaseliner._compile_rebaselining_tests()
298        self.assertTrue(compile_success)
299        self.assertEqual(set(['failures/expected/image_checksum.html']), rebaseliner._rebaselining_tests)
300        rebaseliner.run()
301        self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.png', None), None)
302        self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.txt', None), None)
303        self.assertEqual(filesystem.files.get('/test.checkout/LayoutTests/platform/test-mac-leopard/failures/expected/image_checksum-expected.checksum', None), None)
304
305    def test_diff_baselines_txt(self):
306        rebaseliner, filesystem = self.make_rebaseliner("")
307        port = rebaseliner._port
308        output = port.expected_text(
309            port._filesystem.join(port.layout_tests_dir(), 'passes/text.html'))
310        self.assertFalse(rebaseliner._diff_baselines(output, output,
311                                                     is_image=False))
312
313    def test_diff_baselines_png(self):
314        rebaseliner, filesystem = self.make_rebaseliner('')
315        port = rebaseliner._port
316        image = port.expected_image(
317            port._filesystem.join(port.layout_tests_dir(), 'passes/image.html'))
318        self.assertFalse(rebaseliner._diff_baselines(image, image,
319                                                     is_image=True))
320
321
322class TestRealMain(unittest.TestCase):
323    def setUp(self):
324        if not hasattr(self, '_orig_archive'):
325            self._orig_archive = rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT
326            rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = test_archive(self._orig_archive)
327
328    def tearDown(self):
329        rebaseline_chromium_webkit_tests.ARCHIVE_DIR_NAME_DICT = self._orig_archive
330
331    def test_all_platforms(self):
332        expectations = "BUGX REBASELINE : failures/expected/image.html = IMAGE"
333
334        options = test_options()
335        host_port_obj, filesystem = test_host_port_and_filesystem(options, expectations)
336        url_fetcher = test_url_fetcher(filesystem)
337        zip_factory = test_zip_factory()
338        mock_scm = mocktool.MockSCM()
339        oc = outputcapture.OutputCapture()
340        oc.capture_output()
341        res = rebaseline_chromium_webkit_tests.real_main(options, options,
342            host_port_obj, host_port_obj, url_fetcher, zip_factory, mock_scm)
343        oc.restore_output()
344
345        # We expect to have written 36 files over the course of this rebaseline:
346        # *) 6*3 files in /__im_tmp/ for the archived members of the 6 ports
347        # *) 2*3 files in /test.checkout for actually differing baselines
348        # *) 1 file in /test.checkout for the updated test_expectations file
349        # *) 2*4 files in /tmp for the old/new baselines for the two actual ports
350        # *) 2 files in /tmp for the text diffs for the two ports
351        # *) 1 file in /tmp for the rebaseline results html file
352        self.assertEqual(res, 0)
353        self.assertEqual(len(filesystem.written_files), 36)
354
355
356class TestHtmlGenerator(unittest.TestCase):
357    def make_generator(self, files, tests):
358        options = mocktool.MockOptions(configuration=None, html_directory='/tmp')
359        host_port = port.get('test', options, filesystem=port.unit_test_filesystem(files))
360        generator = rebaseline_chromium_webkit_tests.HtmlGenerator(host_port,
361            target_port=None, options=options, platforms=['test-mac-leopard'], rebaselining_tests=tests)
362        return generator, host_port
363
364    def test_generate_baseline_links(self):
365        files = {
366            "/tmp/foo-expected-mac-old.txt": "",
367            "/tmp/foo-expected-mac-new.txt": "",
368            "/tmp/foo-expected-mac-diff.txt": "",
369        }
370        tests = ["foo.txt"]
371        generator, host_port = self.make_generator(files, tests)
372        links = generator._generate_baseline_links("foo", ".txt", "mac")
373        expected_links = '<td align=center><a href="file:///tmp/foo-expected-mac-old.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-new.txt">foo-expected.txt</a></td><td align=center><a href="file:///tmp/foo-expected-mac-diff.txt">Diff</a></td>'
374        self.assertEqual(links, expected_links)
375
376
377if __name__ == '__main__':
378    unittest.main()
379