1# Copyright (C) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#    * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#    * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#    * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29import logging
30import optparse
31import sys
32import tempfile
33import webkitpy.thirdparty.unittest2 as unittest
34
35from webkitpy.common.system.executive import Executive, ScriptError
36from webkitpy.common.system import executive_mock
37from webkitpy.common.system.filesystem_mock import MockFileSystem
38from webkitpy.common.system.outputcapture import OutputCapture
39from webkitpy.common.system.path import abspath_to_uri
40from webkitpy.thirdparty.mock import Mock
41from webkitpy.tool.mocktool import MockOptions
42from webkitpy.common.system.executive_mock import MockExecutive, MockExecutive2
43from webkitpy.common.system.systemhost_mock import MockSystemHost
44
45from webkitpy.layout_tests.port import Port, Driver, DriverOutput
46from webkitpy.layout_tests.port.test import add_unit_tests_to_mock_filesystem, TestPort
47
48class PortTest(unittest.TestCase):
49    def make_port(self, executive=None, with_tests=False, port_name=None, **kwargs):
50        host = MockSystemHost()
51        if executive:
52            host.executive = executive
53        if with_tests:
54            add_unit_tests_to_mock_filesystem(host.filesystem)
55            return TestPort(host, **kwargs)
56        return Port(host, port_name or 'baseport', **kwargs)
57
58    def test_default_child_processes(self):
59        port = self.make_port()
60        self.assertIsNotNone(port.default_child_processes())
61
62    def test_format_wdiff_output_as_html(self):
63        output = "OUTPUT %s %s %s" % (Port._WDIFF_DEL, Port._WDIFF_ADD, Port._WDIFF_END)
64        html = self.make_port()._format_wdiff_output_as_html(output)
65        expected_html = "<head><style>.del { background: #faa; } .add { background: #afa; }</style></head><pre>OUTPUT <span class=del> <span class=add> </span></pre>"
66        self.assertEqual(html, expected_html)
67
68    def test_wdiff_command(self):
69        port = self.make_port()
70        port._path_to_wdiff = lambda: "/path/to/wdiff"
71        command = port._wdiff_command("/actual/path", "/expected/path")
72        expected_command = [
73            "/path/to/wdiff",
74            "--start-delete=##WDIFF_DEL##",
75            "--end-delete=##WDIFF_END##",
76            "--start-insert=##WDIFF_ADD##",
77            "--end-insert=##WDIFF_END##",
78            "/actual/path",
79            "/expected/path",
80        ]
81        self.assertEqual(command, expected_command)
82
83    def _file_with_contents(self, contents, encoding="utf-8"):
84        new_file = tempfile.NamedTemporaryFile()
85        new_file.write(contents.encode(encoding))
86        new_file.flush()
87        return new_file
88
89    def test_pretty_patch_os_error(self):
90        port = self.make_port(executive=executive_mock.MockExecutive2(exception=OSError))
91        oc = OutputCapture()
92        oc.capture_output()
93        self.assertEqual(port.pretty_patch_text("patch.txt"),
94                         port._pretty_patch_error_html)
95
96        # This tests repeated calls to make sure we cache the result.
97        self.assertEqual(port.pretty_patch_text("patch.txt"),
98                         port._pretty_patch_error_html)
99        oc.restore_output()
100
101    def test_pretty_patch_script_error(self):
102        # FIXME: This is some ugly white-box test hacking ...
103        port = self.make_port(executive=executive_mock.MockExecutive2(exception=ScriptError))
104        port._pretty_patch_available = True
105        self.assertEqual(port.pretty_patch_text("patch.txt"),
106                         port._pretty_patch_error_html)
107
108        # This tests repeated calls to make sure we cache the result.
109        self.assertEqual(port.pretty_patch_text("patch.txt"),
110                         port._pretty_patch_error_html)
111
112    def test_wdiff_text(self):
113        port = self.make_port()
114        port.wdiff_available = lambda: True
115        port._run_wdiff = lambda a, b: 'PASS'
116        self.assertEqual('PASS', port.wdiff_text(None, None))
117
118    def test_diff_text(self):
119        port = self.make_port()
120        # Make sure that we don't run into decoding exceptions when the
121        # filenames are unicode, with regular or malformed input (expected or
122        # actual input is always raw bytes, not unicode).
123        port.diff_text('exp', 'act', 'exp.txt', 'act.txt')
124        port.diff_text('exp', 'act', u'exp.txt', 'act.txt')
125        port.diff_text('exp', 'act', u'a\xac\u1234\u20ac\U00008000', 'act.txt')
126
127        port.diff_text('exp' + chr(255), 'act', 'exp.txt', 'act.txt')
128        port.diff_text('exp' + chr(255), 'act', u'exp.txt', 'act.txt')
129
130        # Though expected and actual files should always be read in with no
131        # encoding (and be stored as str objects), test unicode inputs just to
132        # be safe.
133        port.diff_text(u'exp', 'act', 'exp.txt', 'act.txt')
134        port.diff_text(
135            u'a\xac\u1234\u20ac\U00008000', 'act', 'exp.txt', 'act.txt')
136
137        # And make sure we actually get diff output.
138        diff = port.diff_text('foo', 'bar', 'exp.txt', 'act.txt')
139        self.assertIn('foo', diff)
140        self.assertIn('bar', diff)
141        self.assertIn('exp.txt', diff)
142        self.assertIn('act.txt', diff)
143        self.assertNotIn('nosuchthing', diff)
144
145    def test_setup_test_run(self):
146        port = self.make_port()
147        # This routine is a no-op. We just test it for coverage.
148        port.setup_test_run()
149
150    def test_test_dirs(self):
151        port = self.make_port()
152        port.host.filesystem.write_text_file(port.layout_tests_dir() + '/canvas/test', '')
153        port.host.filesystem.write_text_file(port.layout_tests_dir() + '/css2.1/test', '')
154        dirs = port.test_dirs()
155        self.assertIn('canvas', dirs)
156        self.assertIn('css2.1', dirs)
157
158    def test_skipped_perf_tests(self):
159        port = self.make_port()
160
161        def add_text_file(dirname, filename, content='some content'):
162            dirname = port.host.filesystem.join(port.perf_tests_dir(), dirname)
163            port.host.filesystem.maybe_make_directory(dirname)
164            port.host.filesystem.write_text_file(port.host.filesystem.join(dirname, filename), content)
165
166        add_text_file('inspector', 'test1.html')
167        add_text_file('inspector', 'unsupported_test1.html')
168        add_text_file('inspector', 'test2.html')
169        add_text_file('inspector/resources', 'resource_file.html')
170        add_text_file('unsupported', 'unsupported_test2.html')
171        add_text_file('', 'Skipped', '\n'.join(['Layout', '', 'SunSpider', 'Supported/some-test.html']))
172        self.assertEqual(port.skipped_perf_tests(), ['Layout', 'SunSpider', 'Supported/some-test.html'])
173
174    def test_get_option__set(self):
175        options, args = optparse.OptionParser().parse_args([])
176        options.foo = 'bar'
177        port = self.make_port(options=options)
178        self.assertEqual(port.get_option('foo'), 'bar')
179
180    def test_get_option__unset(self):
181        port = self.make_port()
182        self.assertIsNone(port.get_option('foo'))
183
184    def test_get_option__default(self):
185        port = self.make_port()
186        self.assertEqual(port.get_option('foo', 'bar'), 'bar')
187
188    def test_additional_platform_directory(self):
189        port = self.make_port(port_name='foo')
190        port.default_baseline_search_path = lambda: ['LayoutTests/platform/foo']
191        layout_test_dir = port.layout_tests_dir()
192        test_file = 'fast/test.html'
193
194        # No additional platform directory
195        self.assertEqual(
196            port.expected_baselines(test_file, '.txt'),
197            [(None, 'fast/test-expected.txt')])
198        self.assertEqual(port.baseline_path(), 'LayoutTests/platform/foo')
199
200        # Simple additional platform directory
201        port._options.additional_platform_directory = ['/tmp/local-baselines']
202        port._filesystem.write_text_file('/tmp/local-baselines/fast/test-expected.txt', 'foo')
203        self.assertEqual(
204            port.expected_baselines(test_file, '.txt'),
205            [('/tmp/local-baselines', 'fast/test-expected.txt')])
206        self.assertEqual(port.baseline_path(), '/tmp/local-baselines')
207
208        # Multiple additional platform directories
209        port._options.additional_platform_directory = ['/foo', '/tmp/local-baselines']
210        self.assertEqual(
211            port.expected_baselines(test_file, '.txt'),
212            [('/tmp/local-baselines', 'fast/test-expected.txt')])
213        self.assertEqual(port.baseline_path(), '/foo')
214
215    def test_nonexistant_expectations(self):
216        port = self.make_port(port_name='foo')
217        port.expectations_files = lambda: ['/mock-checkout/LayoutTests/platform/exists/TestExpectations', '/mock-checkout/LayoutTests/platform/nonexistant/TestExpectations']
218        port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/exists/TestExpectations', '')
219        self.assertEqual('\n'.join(port.expectations_dict().keys()), '/mock-checkout/LayoutTests/platform/exists/TestExpectations')
220
221    def test_additional_expectations(self):
222        port = self.make_port(port_name='foo')
223        port.port_name = 'foo'
224        port._filesystem.write_text_file('/mock-checkout/LayoutTests/platform/foo/TestExpectations', '')
225        port._filesystem.write_text_file(
226            '/tmp/additional-expectations-1.txt', 'content1\n')
227        port._filesystem.write_text_file(
228            '/tmp/additional-expectations-2.txt', 'content2\n')
229
230        self.assertEqual('\n'.join(port.expectations_dict().values()), '')
231
232        port._options.additional_expectations = [
233            '/tmp/additional-expectations-1.txt']
234        self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n')
235
236        port._options.additional_expectations = [
237            '/tmp/nonexistent-file', '/tmp/additional-expectations-1.txt']
238        self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n')
239
240        port._options.additional_expectations = [
241            '/tmp/additional-expectations-1.txt', '/tmp/additional-expectations-2.txt']
242        self.assertEqual('\n'.join(port.expectations_dict().values()), '\ncontent1\n\ncontent2\n')
243
244    def test_additional_env_var(self):
245        port = self.make_port(options=optparse.Values({'additional_env_var': ['FOO=BAR', 'BAR=FOO']}))
246        self.assertEqual(port.get_option('additional_env_var'), ['FOO=BAR', 'BAR=FOO'])
247        environment = port.setup_environ_for_server()
248        self.assertTrue(('FOO' in environment) & ('BAR' in environment))
249        self.assertEqual(environment['FOO'], 'BAR')
250        self.assertEqual(environment['BAR'], 'FOO')
251
252    def test_find_no_paths_specified(self):
253        port = self.make_port(with_tests=True)
254        layout_tests_dir = port.layout_tests_dir()
255        tests = port.tests([])
256        self.assertNotEqual(len(tests), 0)
257
258    def test_find_one_test(self):
259        port = self.make_port(with_tests=True)
260        tests = port.tests(['failures/expected/image.html'])
261        self.assertEqual(len(tests), 1)
262
263    def test_find_glob(self):
264        port = self.make_port(with_tests=True)
265        tests = port.tests(['failures/expected/im*'])
266        self.assertEqual(len(tests), 2)
267
268    def test_find_with_skipped_directories(self):
269        port = self.make_port(with_tests=True)
270        tests = port.tests(['userscripts'])
271        self.assertNotIn('userscripts/resources/iframe.html', tests)
272
273    def test_find_with_skipped_directories_2(self):
274        port = self.make_port(with_tests=True)
275        tests = port.tests(['userscripts/resources'])
276        self.assertEqual(tests, [])
277
278    def test_is_test_file(self):
279        filesystem = MockFileSystem()
280        self.assertTrue(Port.is_test_file(filesystem, '', 'foo.html'))
281        self.assertTrue(Port.is_test_file(filesystem, '', 'foo.svg'))
282        self.assertTrue(Port.is_test_file(filesystem, '', 'test-ref-test.html'))
283        self.assertFalse(Port.is_test_file(filesystem, '', 'foo.png'))
284        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.html'))
285        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.svg'))
286        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected.xht'))
287        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.html'))
288        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.svg'))
289        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-expected-mismatch.xhtml'))
290        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.html'))
291        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.html'))
292        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-notref.xht'))
293        self.assertFalse(Port.is_test_file(filesystem, '', 'foo-ref.xhtml'))
294        self.assertFalse(Port.is_test_file(filesystem, '', 'ref-foo.html'))
295        self.assertFalse(Port.is_test_file(filesystem, '', 'notref-foo.xhr'))
296
297    def test_parse_reftest_list(self):
298        port = self.make_port(with_tests=True)
299        port.host.filesystem.files['bar/reftest.list'] = "\n".join(["== test.html test-ref.html",
300        "",
301        "# some comment",
302        "!= test-2.html test-notref.html # more comments",
303        "== test-3.html test-ref.html",
304        "== test-3.html test-ref2.html",
305        "!= test-3.html test-notref.html"])
306
307        reftest_list = Port._parse_reftest_list(port.host.filesystem, 'bar')
308        self.assertEqual(reftest_list, {'bar/test.html': [('==', 'bar/test-ref.html')],
309            'bar/test-2.html': [('!=', 'bar/test-notref.html')],
310            'bar/test-3.html': [('==', 'bar/test-ref.html'), ('==', 'bar/test-ref2.html'), ('!=', 'bar/test-notref.html')]})
311
312    def test_reference_files(self):
313        port = self.make_port(with_tests=True)
314        self.assertEqual(port.reference_files('passes/svgreftest.svg'), [('==', port.layout_tests_dir() + '/passes/svgreftest-expected.svg')])
315        self.assertEqual(port.reference_files('passes/xhtreftest.svg'), [('==', port.layout_tests_dir() + '/passes/xhtreftest-expected.html')])
316        self.assertEqual(port.reference_files('passes/phpreftest.php'), [('!=', port.layout_tests_dir() + '/passes/phpreftest-expected-mismatch.svg')])
317
318    def test_operating_system(self):
319        self.assertEqual('mac', self.make_port().operating_system())
320
321    def test_http_server_supports_ipv6(self):
322        port = self.make_port()
323        self.assertTrue(port.http_server_supports_ipv6())
324        port.host.platform.os_name = 'cygwin'
325        self.assertFalse(port.http_server_supports_ipv6())
326        port.host.platform.os_name = 'win'
327        self.assertTrue(port.http_server_supports_ipv6())
328
329    def test_check_httpd_success(self):
330        port = self.make_port(executive=MockExecutive2())
331        port._path_to_apache = lambda: '/usr/sbin/httpd'
332        capture = OutputCapture()
333        capture.capture_output()
334        self.assertTrue(port.check_httpd())
335        _, _, logs = capture.restore_output()
336        self.assertEqual('', logs)
337
338    def test_httpd_returns_error_code(self):
339        port = self.make_port(executive=MockExecutive2(exit_code=1))
340        port._path_to_apache = lambda: '/usr/sbin/httpd'
341        capture = OutputCapture()
342        capture.capture_output()
343        self.assertFalse(port.check_httpd())
344        _, _, logs = capture.restore_output()
345        self.assertEqual('httpd seems broken. Cannot run http tests.\n', logs)
346
347    def test_test_exists(self):
348        port = self.make_port(with_tests=True)
349        self.assertTrue(port.test_exists('passes'))
350        self.assertTrue(port.test_exists('passes/text.html'))
351        self.assertFalse(port.test_exists('passes/does_not_exist.html'))
352
353        self.assertTrue(port.test_exists('virtual'))
354        self.assertFalse(port.test_exists('virtual/does_not_exist.html'))
355        self.assertTrue(port.test_exists('virtual/passes/text.html'))
356
357    def test_test_isfile(self):
358        port = self.make_port(with_tests=True)
359        self.assertFalse(port.test_isfile('passes'))
360        self.assertTrue(port.test_isfile('passes/text.html'))
361        self.assertFalse(port.test_isfile('passes/does_not_exist.html'))
362
363        self.assertFalse(port.test_isfile('virtual'))
364        self.assertTrue(port.test_isfile('virtual/passes/text.html'))
365        self.assertFalse(port.test_isfile('virtual/does_not_exist.html'))
366
367    def test_test_isdir(self):
368        port = self.make_port(with_tests=True)
369        self.assertTrue(port.test_isdir('passes'))
370        self.assertFalse(port.test_isdir('passes/text.html'))
371        self.assertFalse(port.test_isdir('passes/does_not_exist.html'))
372        self.assertFalse(port.test_isdir('passes/does_not_exist/'))
373
374        self.assertTrue(port.test_isdir('virtual'))
375        self.assertFalse(port.test_isdir('virtual/does_not_exist.html'))
376        self.assertFalse(port.test_isdir('virtual/does_not_exist/'))
377        self.assertFalse(port.test_isdir('virtual/passes/text.html'))
378
379    def test_tests(self):
380        port = self.make_port(with_tests=True)
381        tests = port.tests([])
382        self.assertIn('passes/text.html', tests)
383        self.assertIn('virtual/passes/text.html', tests)
384
385        tests = port.tests(['passes'])
386        self.assertIn('passes/text.html', tests)
387        self.assertIn('passes/passes/test-virtual-passes.html', tests)
388        self.assertNotIn('virtual/passes/text.html', tests)
389
390        tests = port.tests(['virtual/passes'])
391        self.assertNotIn('passes/text.html', tests)
392        self.assertIn('virtual/passes/test-virtual-passes.html', tests)
393        self.assertIn('virtual/passes/passes/test-virtual-passes.html', tests)
394        self.assertNotIn('virtual/passes/test-virtual-virtual/passes.html', tests)
395        self.assertNotIn('virtual/passes/virtual/passes/test-virtual-passes.html', tests)
396
397    def test_build_path(self):
398        port = self.make_port(options=optparse.Values({'build_directory': '/my-build-directory/'}))
399        self.assertEqual(port._build_path(), '/my-build-directory/Release')
400
401    def test_dont_require_http_server(self):
402        port = self.make_port()
403        self.assertEqual(port.requires_http_server(), False)
404
405
406class NaturalCompareTest(unittest.TestCase):
407    def setUp(self):
408        self._port = TestPort(MockSystemHost())
409
410    def assert_cmp(self, x, y, result):
411        self.assertEqual(cmp(self._port._natural_sort_key(x), self._port._natural_sort_key(y)), result)
412
413    def test_natural_compare(self):
414        self.assert_cmp('a', 'a', 0)
415        self.assert_cmp('ab', 'a', 1)
416        self.assert_cmp('a', 'ab', -1)
417        self.assert_cmp('', '', 0)
418        self.assert_cmp('', 'ab', -1)
419        self.assert_cmp('1', '2', -1)
420        self.assert_cmp('2', '1', 1)
421        self.assert_cmp('1', '10', -1)
422        self.assert_cmp('2', '10', -1)
423        self.assert_cmp('foo_1.html', 'foo_2.html', -1)
424        self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
425        self.assert_cmp('foo_1.html', 'foo_10.html', -1)
426        self.assert_cmp('foo_2.html', 'foo_10.html', -1)
427        self.assert_cmp('foo_23.html', 'foo_10.html', 1)
428        self.assert_cmp('foo_23.html', 'foo_100.html', -1)
429
430
431class KeyCompareTest(unittest.TestCase):
432    def setUp(self):
433        self._port = TestPort(MockSystemHost())
434
435    def assert_cmp(self, x, y, result):
436        self.assertEqual(cmp(self._port.test_key(x), self._port.test_key(y)), result)
437
438    def test_test_key(self):
439        self.assert_cmp('/a', '/a', 0)
440        self.assert_cmp('/a', '/b', -1)
441        self.assert_cmp('/a2', '/a10', -1)
442        self.assert_cmp('/a2/foo', '/a10/foo', -1)
443        self.assert_cmp('/a/foo11', '/a/foo2', 1)
444        self.assert_cmp('/ab', '/a/a/b', -1)
445        self.assert_cmp('/a/a/b', '/ab', 1)
446        self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
447