1#!/usr/bin/python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Unit tests for run_webkit_tests."""
32
33from __future__ import with_statement
34
35import codecs
36import itertools
37import logging
38import os
39import Queue
40import sys
41import thread
42import time
43import threading
44import unittest
45
46try:
47    import multiprocessing
48except ImportError:
49    multiprocessing = None
50
51from webkitpy.common import array_stream
52from webkitpy.common.system import outputcapture
53from webkitpy.common.system import filesystem_mock
54from webkitpy.tool import mocktool
55from webkitpy.layout_tests import port
56from webkitpy.layout_tests import run_webkit_tests
57from webkitpy.layout_tests.port.test import TestPort, TestDriver
58from webkitpy.layout_tests.port.test_files import is_reference_html_file
59from webkitpy.python24.versioning import compare_version
60from webkitpy.test.skip import skip_if
61
62from webkitpy.thirdparty.mock import Mock
63
64
65def parse_args(extra_args=None, record_results=False, tests_included=False,
66               print_nothing=True):
67    extra_args = extra_args or []
68    if print_nothing:
69        args = ['--print', 'nothing']
70    else:
71        args = []
72    if not '--platform' in extra_args:
73        args.extend(['--platform', 'test'])
74    if not record_results:
75        args.append('--no-record-results')
76    if not '--child-processes' in extra_args and not '--worker-model' in extra_args:
77        args.extend(['--worker-model', 'inline'])
78    args.extend(extra_args)
79    if not tests_included:
80        # We use the glob to test that globbing works.
81        args.extend(['passes',
82                     'http/tests',
83                     'websocket/tests',
84                     'failures/expected/*'])
85    return run_webkit_tests.parse_args(args)
86
87
88def passing_run(extra_args=None, port_obj=None, record_results=False,
89                tests_included=False, filesystem=None):
90    options, parsed_args = parse_args(extra_args, record_results,
91                                      tests_included)
92    if not port_obj:
93        port_obj = port.get(port_name=options.platform, options=options,
94                            user=mocktool.MockUser(), filesystem=filesystem)
95    res = run_webkit_tests.run(port_obj, options, parsed_args)
96    return res == 0
97
98
99def logging_run(extra_args=None, port_obj=None, record_results=False, tests_included=False, filesystem=None):
100    options, parsed_args = parse_args(extra_args=extra_args,
101                                      record_results=record_results,
102                                      tests_included=tests_included,
103                                      print_nothing=False)
104    user = mocktool.MockUser()
105    if not port_obj:
106        port_obj = port.get(port_name=options.platform, options=options,
107                            user=user, filesystem=filesystem)
108
109    res, buildbot_output, regular_output = run_and_capture(port_obj, options,
110                                                           parsed_args)
111    return (res, buildbot_output, regular_output, user)
112
113
114def run_and_capture(port_obj, options, parsed_args):
115    oc = outputcapture.OutputCapture()
116    try:
117        oc.capture_output()
118        buildbot_output = array_stream.ArrayStream()
119        regular_output = array_stream.ArrayStream()
120        res = run_webkit_tests.run(port_obj, options, parsed_args,
121                                   buildbot_output=buildbot_output,
122                                   regular_output=regular_output)
123    finally:
124        oc.restore_output()
125    return (res, buildbot_output, regular_output)
126
127
128def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
129                  filesystem=None, include_reference_html=False):
130    extra_args = extra_args or []
131    if not tests_included:
132        # Not including http tests since they get run out of order (that
133        # behavior has its own test, see test_get_test_file_queue)
134        extra_args = ['passes', 'failures'] + extra_args
135    options, parsed_args = parse_args(extra_args, tests_included=True)
136
137    user = mocktool.MockUser()
138
139    test_batches = []
140
141
142    class RecordingTestDriver(TestDriver):
143        def __init__(self, port, worker_number):
144            TestDriver.__init__(self, port, worker_number)
145            self._current_test_batch = None
146
147        def poll(self):
148            # So that we don't create a new driver for every test
149            return None
150
151        def stop(self):
152            self._current_test_batch = None
153
154        def run_test(self, test_input):
155            if self._current_test_batch is None:
156                self._current_test_batch = []
157                test_batches.append(self._current_test_batch)
158            test_name = self._port.relative_test_filename(test_input.filename)
159            # In case of reftest, one test calls the driver's run_test() twice.
160            # We should not add a reference html used by reftests to tests unless include_reference_html parameter
161            # is explicitly given.
162            if include_reference_html or not is_reference_html_file(test_input.filename):
163                self._current_test_batch.append(test_name)
164            return TestDriver.run_test(self, test_input)
165
166    class RecordingTestPort(TestPort):
167        def create_driver(self, worker_number):
168            return RecordingTestDriver(self, worker_number)
169
170    recording_port = RecordingTestPort(options=options, user=user, filesystem=filesystem)
171    run_and_capture(recording_port, options, parsed_args)
172
173    if flatten_batches:
174        return list(itertools.chain(*test_batches))
175
176    return test_batches
177
178
179class MainTest(unittest.TestCase):
180    def test_accelerated_compositing(self):
181        # This just tests that we recognize the command line args
182        self.assertTrue(passing_run(['--accelerated-compositing']))
183        self.assertTrue(passing_run(['--no-accelerated-compositing']))
184
185    def test_accelerated_2d_canvas(self):
186        # This just tests that we recognize the command line args
187        self.assertTrue(passing_run(['--accelerated-2d-canvas']))
188        self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
189
190    def test_basic(self):
191        self.assertTrue(passing_run())
192
193    def test_batch_size(self):
194        batch_tests_run = get_tests_run(['--batch-size', '2'])
195        for batch in batch_tests_run:
196            self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
197
198    def test_child_process_1(self):
199        _, _, regular_output, _ = logging_run(
200             ['--print', 'config', '--worker-model', 'threads', '--child-processes', '1'])
201        self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
202
203    def test_child_processes_2(self):
204        _, _, regular_output, _ = logging_run(
205             ['--print', 'config', '--worker-model', 'threads', '--child-processes', '2'])
206        self.assertTrue(any(['Running 2 ' in line for line in regular_output.get()]))
207
208    def test_child_processes_min(self):
209        _, _, regular_output, _ = logging_run(
210             ['--print', 'config', '--worker-model', 'threads', '--child-processes', '2', 'passes'],
211             tests_included=True)
212        self.assertTrue(any(['Running 1 ' in line for line in regular_output.get()]))
213
214    def test_dryrun(self):
215        batch_tests_run = get_tests_run(['--dry-run'])
216        self.assertEqual(batch_tests_run, [])
217
218        batch_tests_run = get_tests_run(['-n'])
219        self.assertEqual(batch_tests_run, [])
220
221    def test_exception_raised(self):
222        self.assertRaises(ValueError, logging_run,
223            ['failures/expected/exception.html'], tests_included=True)
224
225    def test_full_results_html(self):
226        # FIXME: verify html?
227        res, out, err, user = logging_run(['--full-results-html'])
228        self.assertEqual(res, 0)
229
230    def test_help_printing(self):
231        res, out, err, user = logging_run(['--help-printing'])
232        self.assertEqual(res, 0)
233        self.assertTrue(out.empty())
234        self.assertFalse(err.empty())
235
236    def test_hung_thread(self):
237        res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
238                                          'failures/expected/hang.html'],
239                                          tests_included=True)
240        self.assertEqual(res, 0)
241        self.assertFalse(out.empty())
242        self.assertFalse(err.empty())
243
244    def test_keyboard_interrupt(self):
245        # Note that this also tests running a test marked as SKIP if
246        # you specify it explicitly.
247        self.assertRaises(KeyboardInterrupt, logging_run,
248            ['failures/expected/keyboard.html'], tests_included=True)
249
250    def test_keyboard_interrupt_inline_worker_model(self):
251        self.assertRaises(KeyboardInterrupt, logging_run,
252            ['failures/expected/keyboard.html', '--worker-model', 'inline'],
253            tests_included=True)
254
255    def test_last_results(self):
256        fs = port.unit_test_filesystem()
257        # We do a logging run here instead of a passing run in order to
258        # suppress the output from the json generator.
259        res, buildbot_output, regular_output, user = logging_run(['--clobber-old-results'], record_results=True, filesystem=fs)
260        res, buildbot_output, regular_output, user = logging_run(
261            ['--print-last-failures'], filesystem=fs)
262        self.assertEqual(regular_output.get(), ['\n\n'])
263        self.assertEqual(buildbot_output.get(), [])
264
265    def test_lint_test_files(self):
266        res, out, err, user = logging_run(['--lint-test-files'])
267        self.assertEqual(res, 0)
268        self.assertTrue(out.empty())
269        self.assertTrue(any(['Lint succeeded' in msg for msg in err.get()]))
270
271    def test_lint_test_files__errors(self):
272        options, parsed_args = parse_args(['--lint-test-files'])
273        user = mocktool.MockUser()
274        port_obj = port.get(options.platform, options=options, user=user)
275        port_obj.test_expectations = lambda: "# syntax error"
276        res, out, err = run_and_capture(port_obj, options, parsed_args)
277
278        self.assertEqual(res, -1)
279        self.assertTrue(out.empty())
280        self.assertTrue(any(['Lint failed' in msg for msg in err.get()]))
281
282    def test_no_tests_found(self):
283        res, out, err, user = logging_run(['resources'], tests_included=True)
284        self.assertEqual(res, -1)
285        self.assertTrue(out.empty())
286        self.assertTrue('No tests to run.\n' in err.get())
287
288    def test_no_tests_found_2(self):
289        res, out, err, user = logging_run(['foo'], tests_included=True)
290        self.assertEqual(res, -1)
291        self.assertTrue(out.empty())
292        self.assertTrue('No tests to run.\n' in err.get())
293
294    def test_randomize_order(self):
295        # FIXME: verify order was shuffled
296        self.assertTrue(passing_run(['--randomize-order']))
297
298    def test_run_chunk(self):
299        # Test that we actually select the right chunk
300        all_tests_run = get_tests_run(flatten_batches=True)
301        chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
302        self.assertEquals(all_tests_run[4:8], chunk_tests_run)
303
304        # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
305        tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
306        chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
307        self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
308
309    def test_run_force(self):
310        # This raises an exception because we run
311        # failures/expected/exception.html, which is normally SKIPped.
312        self.assertRaises(ValueError, logging_run, ['--force'])
313
314    def test_run_part(self):
315        # Test that we actually select the right part
316        tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
317        tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
318        self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
319
320        # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
321        # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
322        # last part repeats the first two tests).
323        chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
324        self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
325
326    def test_run_singly(self):
327        batch_tests_run = get_tests_run(['--run-singly'])
328        for batch in batch_tests_run:
329            self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
330
331    def test_run_singly_actually_runs_tests(self):
332        res, _, _, _ = logging_run(['--run-singly', 'failures/unexpected'])
333        self.assertEquals(res, 5)
334
335    def test_single_file(self):
336        tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True)
337        self.assertEquals(['passes/text.html'], tests_run)
338
339    def test_single_file_with_prefix(self):
340        tests_run = get_tests_run(['LayoutTests/passes/text.html'], tests_included=True, flatten_batches=True)
341        self.assertEquals(['passes/text.html'], tests_run)
342
343    def test_single_skipped_file(self):
344        tests_run = get_tests_run(['failures/expected/keybaord.html'], tests_included=True, flatten_batches=True)
345        self.assertEquals([], tests_run)
346
347    def test_stderr_is_saved(self):
348        fs = port.unit_test_filesystem()
349        self.assertTrue(passing_run(filesystem=fs))
350        self.assertEquals(fs.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
351                          'stuff going to stderr')
352
353    def test_test_list(self):
354        fs = port.unit_test_filesystem()
355        filename = '/tmp/foo.txt'
356        fs.write_text_file(filename, 'passes/text.html')
357        tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
358        self.assertEquals(['passes/text.html'], tests_run)
359        fs.remove(filename)
360        res, out, err, user = logging_run(['--test-list=%s' % filename],
361                                          tests_included=True, filesystem=fs)
362        self.assertEqual(res, -1)
363        self.assertFalse(err.empty())
364
365    def test_test_list_with_prefix(self):
366        fs = port.unit_test_filesystem()
367        filename = '/tmp/foo.txt'
368        fs.write_text_file(filename, 'LayoutTests/passes/text.html')
369        tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True, filesystem=fs)
370        self.assertEquals(['passes/text.html'], tests_run)
371
372    def test_unexpected_failures(self):
373        # Run tests including the unexpected failures.
374        self._url_opened = None
375        res, out, err, user = logging_run(tests_included=True)
376
377        # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
378        # FIXME: It's nice to have a routine in port/test.py that returns this number.
379        unexpected_tests_count = 5
380
381        self.assertEqual(res, unexpected_tests_count)
382        self.assertFalse(out.empty())
383        self.assertFalse(err.empty())
384        self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
385
386    def test_exit_after_n_failures_upload(self):
387        fs = port.unit_test_filesystem()
388        res, buildbot_output, regular_output, user = logging_run([
389                'failures/unexpected/text-image-checksum.html',
390                'passes/text.html',
391                '--exit-after-n-failures', '1',
392            ],
393            tests_included=True,
394            record_results=True,
395            filesystem=fs)
396        self.assertTrue('/tmp/layout-test-results/incremental_results.json' in fs.files)
397
398    def test_exit_after_n_failures(self):
399        # Unexpected failures should result in tests stopping.
400        tests_run = get_tests_run([
401                'failures/unexpected/text-image-checksum.html',
402                'passes/text.html',
403                '--exit-after-n-failures', '1',
404            ],
405            tests_included=True,
406            flatten_batches=True)
407        self.assertEquals(['failures/unexpected/text-image-checksum.html'], tests_run)
408
409        # But we'll keep going for expected ones.
410        tests_run = get_tests_run([
411                'failures/expected/text.html',
412                'passes/text.html',
413                '--exit-after-n-failures', '1',
414            ],
415            tests_included=True,
416            flatten_batches=True)
417        self.assertEquals(['failures/expected/text.html', 'passes/text.html'], tests_run)
418
419    def test_exit_after_n_crashes(self):
420        # Unexpected crashes should result in tests stopping.
421        tests_run = get_tests_run([
422                'failures/unexpected/crash.html',
423                'passes/text.html',
424                '--exit-after-n-crashes-or-timeouts', '1',
425            ],
426            tests_included=True,
427            flatten_batches=True)
428        self.assertEquals(['failures/unexpected/crash.html'], tests_run)
429
430        # Same with timeouts.
431        tests_run = get_tests_run([
432                'failures/unexpected/timeout.html',
433                'passes/text.html',
434                '--exit-after-n-crashes-or-timeouts', '1',
435            ],
436            tests_included=True,
437            flatten_batches=True)
438        self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
439
440        # But we'll keep going for expected ones.
441        tests_run = get_tests_run([
442                'failures/expected/crash.html',
443                'passes/text.html',
444                '--exit-after-n-crashes-or-timeouts', '1',
445            ],
446            tests_included=True,
447            flatten_batches=True)
448        self.assertEquals(['failures/expected/crash.html', 'passes/text.html'], tests_run)
449
450    def test_exit_after_n_crashes_inline_worker_model(self):
451        tests_run = get_tests_run([
452                'failures/unexpected/timeout.html',
453                'passes/text.html',
454                '--exit-after-n-crashes-or-timeouts', '1',
455                '--worker-model', 'inline',
456            ],
457            tests_included=True,
458            flatten_batches=True)
459        self.assertEquals(['failures/unexpected/timeout.html'], tests_run)
460
461    def test_results_directory_absolute(self):
462        # We run a configuration that should fail, to generate output, then
463        # look for what the output results url was.
464
465        fs = port.unit_test_filesystem()
466        with fs.mkdtemp() as tmpdir:
467            res, out, err, user = logging_run(['--results-directory=' + str(tmpdir)],
468                                              tests_included=True, filesystem=fs)
469            self.assertEqual(user.opened_urls, [fs.join(tmpdir, 'results.html')])
470
471    def test_results_directory_default(self):
472        # We run a configuration that should fail, to generate output, then
473        # look for what the output results url was.
474
475        # This is the default location.
476        res, out, err, user = logging_run(tests_included=True)
477        self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
478
479    def test_results_directory_relative(self):
480        # We run a configuration that should fail, to generate output, then
481        # look for what the output results url was.
482        fs = port.unit_test_filesystem()
483        fs.maybe_make_directory('/tmp/cwd')
484        fs.chdir('/tmp/cwd')
485        res, out, err, user = logging_run(['--results-directory=foo'],
486                                          tests_included=True, filesystem=fs)
487        self.assertEqual(user.opened_urls, ['/tmp/cwd/foo/results.html'])
488
489    # These next tests test that we run the tests in ascending alphabetical
490    # order per directory. HTTP tests are sharded separately from other tests,
491    # so we have to test both.
492    def assert_run_order(self, worker_model, child_processes='1'):
493        tests_run = get_tests_run(['--worker-model', worker_model,
494            '--child-processes', child_processes, 'passes'],
495            tests_included=True, flatten_batches=True)
496        self.assertEquals(tests_run, sorted(tests_run))
497
498        tests_run = get_tests_run(['--worker-model', worker_model,
499            '--child-processes', child_processes, 'http/tests/passes'],
500            tests_included=True, flatten_batches=True)
501        self.assertEquals(tests_run, sorted(tests_run))
502
503    def test_run_order__inline(self):
504        self.assert_run_order('inline')
505
506    def test_tolerance(self):
507        class ImageDiffTestPort(TestPort):
508            def diff_image(self, expected_contents, actual_contents,
509                   diff_filename=None):
510                self.tolerance_used_for_diff_image = self._options.tolerance
511                return True
512
513        def get_port_for_run(args):
514            options, parsed_args = run_webkit_tests.parse_args(args)
515            test_port = ImageDiffTestPort(options=options, user=mocktool.MockUser())
516            passing_run(args, port_obj=test_port, tests_included=True)
517            return test_port
518
519        base_args = ['--pixel-tests', 'failures/expected/*']
520
521        # If we pass in an explicit tolerance argument, then that will be used.
522        test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
523        self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
524        test_port = get_port_for_run(base_args + ['--tolerance', '0'])
525        self.assertEqual(0, test_port.tolerance_used_for_diff_image)
526
527        # Otherwise the port's default tolerance behavior (including ignoring it)
528        # should be used.
529        test_port = get_port_for_run(base_args)
530        self.assertEqual(None, test_port.tolerance_used_for_diff_image)
531
532    def test_worker_model__inline(self):
533        self.assertTrue(passing_run(['--worker-model', 'inline']))
534
535    def test_worker_model__inline_with_child_processes(self):
536        res, out, err, user = logging_run(['--worker-model', 'inline',
537                                           '--child-processes', '2'])
538        self.assertEqual(res, 0)
539        self.assertTrue('--worker-model=inline overrides --child-processes\n' in err.get())
540
541    def test_worker_model__processes(self):
542        # FIXME: remove this when we fix test-webkitpy to work properly
543        # with the multiprocessing module (bug 54520).
544        if multiprocessing and sys.platform not in ('cygwin', 'win32'):
545            self.assertTrue(passing_run(['--worker-model', 'processes']))
546
547    def test_worker_model__processes_and_dry_run(self):
548        if multiprocessing and sys.platform not in ('cygwin', 'win32'):
549            self.assertTrue(passing_run(['--worker-model', 'processes', '--dry-run']))
550
551    def test_worker_model__threads(self):
552        self.assertTrue(passing_run(['--worker-model', 'threads']))
553
554    def test_worker_model__unknown(self):
555        self.assertRaises(ValueError, logging_run,
556                          ['--worker-model', 'unknown'])
557
558    def test_reftest_run(self):
559        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
560        self.assertEquals(['passes/reftest.html'], tests_run)
561
562    def test_reftest_expected_html_should_be_ignored(self):
563        tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
564        self.assertEquals([], tests_run)
565
566    def test_reftest_driver_should_run_expected_html(self):
567        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True,
568                                  include_reference_html=True)
569        self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
570
571    def test_reftest_driver_should_run_expected_mismatch_html(self):
572        tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True,
573                                  include_reference_html=True)
574        self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
575
576    def test_additional_platform_directory(self):
577        self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
578        self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
579        self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo',
580            '--additional-platform-directory', '/tmp/bar']))
581
582        res, buildbot_output, regular_output, user = logging_run(
583             ['--additional-platform-directory', 'foo'])
584        self.assertTrue('--additional-platform-directory=foo is ignored since it is not absolute\n'
585                        in regular_output.get())
586
587
588MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
589
590
591class RebaselineTest(unittest.TestCase):
592    def assertBaselines(self, file_list, file):
593        "assert that the file_list contains the baselines."""
594        for ext in [".txt", ".png", ".checksum"]:
595            baseline = file + "-expected" + ext
596            self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
597
598    # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
599    # supposed to be.
600
601    def test_reset_results(self):
602        # Test that we update expectations in place. If the expectation
603        # is missing, update the expected generic location.
604        fs = port.unit_test_filesystem()
605        passing_run(['--pixel-tests',
606                        '--reset-results',
607                        'passes/image.html',
608                        'failures/expected/missing_image.html'],
609                        tests_included=True, filesystem=fs)
610        file_list = fs.written_files.keys()
611        file_list.remove('/tmp/layout-test-results/tests_run0.txt')
612        self.assertEqual(len(file_list), 6)
613        self.assertBaselines(file_list,
614            "/passes/image")
615        self.assertBaselines(file_list,
616            "/failures/expected/missing_image")
617
618    def test_new_baseline(self):
619        # Test that we update the platform expectations. If the expectation
620        # is mssing, then create a new expectation in the platform dir.
621        fs = port.unit_test_filesystem()
622        passing_run(['--pixel-tests',
623                        '--new-baseline',
624                        'passes/image.html',
625                        'failures/expected/missing_image.html'],
626                    tests_included=True, filesystem=fs)
627        file_list = fs.written_files.keys()
628        file_list.remove('/tmp/layout-test-results/tests_run0.txt')
629        self.assertEqual(len(file_list), 6)
630        self.assertBaselines(file_list,
631            "/platform/test-mac-leopard/passes/image")
632        self.assertBaselines(file_list,
633            "/platform/test-mac-leopard/failures/expected/missing_image")
634
635
636class DryrunTest(unittest.TestCase):
637    # FIXME: it's hard to know which platforms are safe to test; the
638    # chromium platforms require a chromium checkout, and the mac platform
639    # requires fcntl, so it can't be tested on win32, etc. There is
640    # probably a better way of handling this.
641    def disabled_test_darwin(self):
642        if sys.platform != "darwin":
643            return
644
645        self.assertTrue(passing_run(['--platform', 'dryrun', 'fast/html'],
646                        tests_included=True))
647        self.assertTrue(passing_run(['--platform', 'dryrun-mac', 'fast/html'],
648                        tests_included=True))
649
650    def test_test(self):
651        self.assertTrue(passing_run(['--platform', 'dryrun-test',
652                                           '--pixel-tests']))
653
654
655if __name__ == '__main__':
656    unittest.main()
657