1#!/usr/bin/env python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3#
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9# notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11# copyright notice, this list of conditions and the following disclaimer
12# in the documentation and/or other materials provided with the
13# distribution.
14#     * Neither the Google name nor the names of its
15# contributors may be used to endorse or promote products derived from
16# this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30"""Dummy Port implementation used for testing."""
31from __future__ import with_statement
32
33import base64
34import time
35
36from webkitpy.common.system import filesystem_mock
37from webkitpy.tool import mocktool
38
39import base
40
41
42# This sets basic expectations for a test. Each individual expectation
43# can be overridden by a keyword argument in TestList.add().
44class TestInstance:
45    def __init__(self, name):
46        self.name = name
47        self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
48        self.crash = False
49        self.exception = False
50        self.hang = False
51        self.keyboard = False
52        self.error = ''
53        self.timeout = False
54        self.is_reftest = False
55
56        # The values of each field are treated as raw byte strings. They
57        # will be converted to unicode strings where appropriate using
58        # MockFileSystem.read_text_file().
59        self.actual_text = self.base + '-txt'
60        self.actual_checksum = self.base + '-checksum'
61
62        # We add the '\x8a' for the image file to prevent the value from
63        # being treated as UTF-8 (the character is invalid)
64        self.actual_image = self.base + '\x8a' + '-png'
65
66        self.expected_text = self.actual_text
67        self.expected_checksum = self.actual_checksum
68        self.expected_image = self.actual_image
69
70        self.actual_audio = None
71        self.expected_audio = None
72
73# This is an in-memory list of tests, what we want them to produce, and
74# what we want to claim are the expected results.
75class TestList:
76    def __init__(self):
77        self.tests = {}
78
79    def add(self, name, **kwargs):
80        test = TestInstance(name)
81        for key, value in kwargs.items():
82            test.__dict__[key] = value
83        self.tests[name] = test
84
85    def add_reftest(self, name, reference_name, same_image):
86        self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
87        if same_image:
88            self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
89        else:
90            self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
91
92    def keys(self):
93        return self.tests.keys()
94
95    def __contains__(self, item):
96        return item in self.tests
97
98    def __getitem__(self, item):
99        return self.tests[item]
100
101
102def unit_test_list():
103    tests = TestList()
104    tests.add('failures/expected/checksum.html',
105              actual_checksum='checksum_fail-checksum')
106    tests.add('failures/expected/crash.html', crash=True)
107    tests.add('failures/expected/exception.html', exception=True)
108    tests.add('failures/expected/timeout.html', timeout=True)
109    tests.add('failures/expected/hang.html', hang=True)
110    tests.add('failures/expected/missing_text.html', expected_text=None)
111    tests.add('failures/expected/image.html',
112              actual_image='image_fail-png',
113              expected_image='image-png')
114    tests.add('failures/expected/image_checksum.html',
115              actual_checksum='image_checksum_fail-checksum',
116              actual_image='image_checksum_fail-png')
117    tests.add('failures/expected/audio.html',
118              actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
119              actual_text=None, expected_text=None,
120              actual_image=None, expected_image=None,
121              actual_checksum=None, expected_checksum=None)
122    tests.add('failures/expected/keyboard.html', keyboard=True)
123    tests.add('failures/expected/missing_check.html',
124              expected_checksum=None,
125              expected_image=None)
126    tests.add('failures/expected/missing_image.html', expected_image=None)
127    tests.add('failures/expected/missing_audio.html', expected_audio=None,
128              actual_text=None, expected_text=None,
129              actual_image=None, expected_image=None,
130              actual_checksum=None, expected_checksum=None)
131    tests.add('failures/expected/missing_text.html', expected_text=None)
132    tests.add('failures/expected/newlines_leading.html',
133              expected_text="\nfoo\n", actual_text="foo\n")
134    tests.add('failures/expected/newlines_trailing.html',
135              expected_text="foo\n\n", actual_text="foo\n")
136    tests.add('failures/expected/newlines_with_excess_CR.html',
137              expected_text="foo\r\r\r\n", actual_text="foo\n")
138    tests.add('failures/expected/text.html', actual_text='text_fail-png')
139    tests.add('failures/unexpected/crash.html', crash=True)
140    tests.add('failures/unexpected/text-image-checksum.html',
141              actual_text='text-image-checksum_fail-txt',
142              actual_checksum='text-image-checksum_fail-checksum')
143    tests.add('failures/unexpected/timeout.html', timeout=True)
144    tests.add('http/tests/passes/text.html')
145    tests.add('http/tests/passes/image.html')
146    tests.add('http/tests/ssl/text.html')
147    tests.add('passes/error.html', error='stuff going to stderr')
148    tests.add('passes/image.html')
149    tests.add('passes/audio.html',
150              actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
151              actual_text=None, expected_text=None,
152              actual_image=None, expected_image=None,
153              actual_checksum=None, expected_checksum=None)
154    tests.add('passes/platform_image.html')
155    tests.add('passes/checksum_in_image.html',
156              expected_checksum=None,
157              expected_image='tEXtchecksum\x00checksum_in_image-checksum')
158
159    # Text output files contain "\r\n" on Windows.  This may be
160    # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
161    tests.add('passes/text.html',
162              expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
163
164    # For reftests.
165    tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
166    tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
167    tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
168    tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
169    tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
170    tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
171    # FIXME: Add a reftest which crashes.
172
173    tests.add('websocket/tests/passes/text.html')
174    return tests
175
176
177# Here we use a non-standard location for the layout tests, to ensure that
178# this works. The path contains a '.' in the name because we've seen bugs
179# related to this before.
180
181LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
182
183
184# Here we synthesize an in-memory filesystem from the test list
185# in order to fully control the test output and to demonstrate that
186# we don't need a real filesystem to run the tests.
187
188def unit_test_filesystem(files=None):
189    """Return the FileSystem object used by the unit tests."""
190    test_list = unit_test_list()
191    files = files or {}
192
193    def add_file(files, test, suffix, contents):
194        dirname = test.name[0:test.name.rfind('/')]
195        base = test.base
196        path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
197        files[path] = contents
198
199    # Add each test and the expected output, if any.
200    for test in test_list.tests.values():
201        add_file(files, test, '.html', '')
202        if test.is_reftest:
203            continue
204        if test.actual_audio:
205            add_file(files, test, '-expected.wav', test.expected_audio)
206            continue
207
208        add_file(files, test, '-expected.txt', test.expected_text)
209        add_file(files, test, '-expected.checksum', test.expected_checksum)
210        add_file(files, test, '-expected.png', test.expected_image)
211
212
213    # Add the test_expectations file.
214    files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
215WONTFIX : failures/expected/checksum.html = IMAGE
216WONTFIX : failures/expected/crash.html = CRASH
217// This one actually passes because the checksums will match.
218WONTFIX : failures/expected/image.html = PASS
219WONTFIX : failures/expected/audio.html = AUDIO
220WONTFIX : failures/expected/image_checksum.html = IMAGE
221WONTFIX : failures/expected/mismatch.html = IMAGE
222WONTFIX : failures/expected/missing_check.html = MISSING PASS
223WONTFIX : failures/expected/missing_image.html = MISSING PASS
224WONTFIX : failures/expected/missing_audio.html = MISSING PASS
225WONTFIX : failures/expected/missing_text.html = MISSING PASS
226WONTFIX : failures/expected/newlines_leading.html = TEXT
227WONTFIX : failures/expected/newlines_trailing.html = TEXT
228WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
229WONTFIX : failures/expected/reftest.html = IMAGE
230WONTFIX : failures/expected/text.html = TEXT
231WONTFIX : failures/expected/timeout.html = TIMEOUT
232WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
233WONTFIX SKIP : failures/expected/keyboard.html = CRASH
234WONTFIX SKIP : failures/expected/exception.html = CRASH
235"""
236
237    # Add in a file should be ignored by test_files.find().
238    files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe'
239
240    fs = filesystem_mock.MockFileSystem(files)
241    fs._tests = test_list
242    return fs
243
244
245class TestPort(base.Port):
246    """Test implementation of the Port interface."""
247    ALL_BASELINE_VARIANTS = (
248        'test-mac-snowleopard', 'test-mac-leopard',
249        'test-win-win7', 'test-win-vista', 'test-win-xp',
250        'test-linux-x86',
251    )
252
253    def __init__(self, port_name=None, user=None, filesystem=None, **kwargs):
254        if not port_name or port_name == 'test':
255            port_name = 'test-mac-leopard'
256        user = user or mocktool.MockUser()
257        filesystem = filesystem or unit_test_filesystem()
258        base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user,
259                           **kwargs)
260        self._results_directory = None
261
262        assert filesystem._tests
263        self._tests = filesystem._tests
264
265        self._operating_system = 'mac'
266        if port_name.startswith('test-win'):
267            self._operating_system = 'win'
268        elif port_name.startswith('test-linux'):
269            self._operating_system = 'linux'
270
271        version_map = {
272            'test-win-xp': 'xp',
273            'test-win-win7': 'win7',
274            'test-win-vista': 'vista',
275            'test-mac-leopard': 'leopard',
276            'test-mac-snowleopard': 'snowleopard',
277            'test-linux-x86': '',
278        }
279        self._version = version_map[port_name]
280
281        self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
282
283    def _path_to_driver(self):
284        # This routine shouldn't normally be called, but it is called by
285        # the mock_drt Driver. We return something, but make sure it's useless.
286        return 'junk'
287
288    def baseline_path(self):
289        # We don't bother with a fallback path.
290        return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name())
291
292    def baseline_search_path(self):
293        search_paths = {
294            'test-mac-snowleopard': ['test-mac-snowleopard'],
295            'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
296            'test-win-win7': ['test-win-win7'],
297            'test-win-vista': ['test-win-vista', 'test-win-win7'],
298            'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
299            'test-linux-x86': ['test-linux', 'test-win-win7'],
300        }
301        return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
302
303    def default_child_processes(self):
304        return 1
305
306    def default_worker_model(self):
307        return 'inline'
308
309    def check_build(self, needs_http):
310        return True
311
312    def default_configuration(self):
313        return 'Release'
314
315    def diff_image(self, expected_contents, actual_contents,
316                   diff_filename=None):
317        diffed = actual_contents != expected_contents
318        if diffed and diff_filename:
319            self._filesystem.write_binary_file(diff_filename,
320                "< %s\n---\n> %s\n" % (expected_contents, actual_contents))
321        return diffed
322
323    def layout_tests_dir(self):
324        return LAYOUT_TEST_DIR
325
326    def name(self):
327        return self._name
328
329    def _path_to_wdiff(self):
330        return None
331
332    def default_results_directory(self):
333        return '/tmp/layout-test-results'
334
335    def setup_test_run(self):
336        pass
337
338    def create_driver(self, worker_number):
339        return TestDriver(self, worker_number)
340
341    def start_http_server(self):
342        pass
343
344    def start_websocket_server(self):
345        pass
346
347    def stop_http_server(self):
348        pass
349
350    def stop_websocket_server(self):
351        pass
352
353    def path_to_test_expectations_file(self):
354        return self._expectations_path
355
356    def all_baseline_variants(self):
357        return self.ALL_BASELINE_VARIANTS
358
359    # FIXME: These next two routines are copied from base.py with
360    # the calls to path.abspath_to_uri() removed. We shouldn't have
361    # to do this.
362    def filename_to_uri(self, filename):
363        """Convert a test file (which is an absolute path) to a URI."""
364        LAYOUTTEST_HTTP_DIR = "http/tests/"
365        LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
366
367        relative_path = self.relative_test_filename(filename)
368        port = None
369        use_ssl = False
370
371        if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
372            or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
373            relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
374            port = 8000
375
376        # Make http/tests/local run as local files. This is to mimic the
377        # logic in run-webkit-tests.
378        #
379        # TODO(dpranke): remove the media reference and the SSL reference?
380        if (port and not relative_path.startswith("local/") and
381            not relative_path.startswith("media/")):
382            if relative_path.startswith("ssl/"):
383                port += 443
384                protocol = "https"
385            else:
386                protocol = "http"
387            return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
388
389        return "file://" + self._filesystem.abspath(filename)
390
391    def uri_to_test_name(self, uri):
392        """Return the base layout test name for a given URI.
393
394        This returns the test name for a given URI, e.g., if you passed in
395        "file:///src/LayoutTests/fast/html/keygen.html" it would return
396        "fast/html/keygen.html".
397
398        """
399        test = uri
400        if uri.startswith("file:///"):
401            prefix = "file://" + self.layout_tests_dir() + "/"
402            return test[len(prefix):]
403
404        if uri.startswith("http://127.0.0.1:8880/"):
405            # websocket tests
406            return test.replace('http://127.0.0.1:8880/', '')
407
408        if uri.startswith("http://"):
409            # regular HTTP test
410            return test.replace('http://127.0.0.1:8000/', 'http/tests/')
411
412        if uri.startswith("https://"):
413            return test.replace('https://127.0.0.1:8443/', 'http/tests/')
414
415        raise NotImplementedError('unknown url type: %s' % uri)
416
417
418class TestDriver(base.Driver):
419    """Test/Dummy implementation of the DumpRenderTree interface."""
420
421    def __init__(self, port, worker_number):
422        self._port = port
423
424    def cmd_line(self):
425        return [self._port._path_to_driver()] + self._port.get_option('additional_drt_flag', [])
426
427    def poll(self):
428        return True
429
430    def run_test(self, test_input):
431        start_time = time.time()
432        test_name = self._port.relative_test_filename(test_input.filename)
433        test = self._port._tests[test_name]
434        if test.keyboard:
435            raise KeyboardInterrupt
436        if test.exception:
437            raise ValueError('exception from ' + test_name)
438        if test.hang:
439            time.sleep((float(test_input.timeout) * 4) / 1000.0)
440
441        audio = None
442        if test.actual_audio:
443            audio = base64.b64decode(test.actual_audio)
444        return base.DriverOutput(test.actual_text, test.actual_image,
445            test.actual_checksum, audio, crash=test.crash,
446            test_time=time.time() - start_time, timeout=test.timeout, error=test.error)
447
448    def start(self):
449        pass
450
451    def stop(self):
452        pass
453