1#!/usr/bin/env python
2# Copyright (C) 2010 Google Inc. All rights reserved.
3# Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Run layout tests."""
32
33import errno
34import logging
35import optparse
36import os
37import signal
38import sys
39
40from layout_package import json_results_generator
41from layout_package import printing
42from layout_package import test_runner
43from layout_package import test_runner2
44
45from webkitpy.common.system import user
46from webkitpy.thirdparty import simplejson
47
48import port
49
50_log = logging.getLogger(__name__)
51
52
53def run(port, options, args, regular_output=sys.stderr,
54        buildbot_output=sys.stdout):
55    """Run the tests.
56
57    Args:
58      port: Port object for port-specific behavior
59      options: a dictionary of command line options
60      args: a list of sub directories or files to test
61      regular_output: a stream-like object that we can send logging/debug
62          output to
63      buildbot_output: a stream-like object that we can write all output that
64          is intended to be parsed by the buildbot to
65    Returns:
66      the number of unexpected results that occurred, or -1 if there is an
67          error.
68
69    """
70    warnings = _set_up_derived_options(port, options)
71
72    printer = printing.Printer(port, options, regular_output, buildbot_output,
73        int(options.child_processes), options.experimental_fully_parallel)
74    for w in warnings:
75        _log.warning(w)
76
77    if options.help_printing:
78        printer.help_printing()
79        printer.cleanup()
80        return 0
81
82    last_unexpected_results = _gather_unexpected_results(port)
83    if options.print_last_failures:
84        printer.write("\n".join(last_unexpected_results) + "\n")
85        printer.cleanup()
86        return 0
87
88    # We wrap any parts of the run that are slow or likely to raise exceptions
89    # in a try/finally to ensure that we clean up the logging configuration.
90    num_unexpected_results = -1
91    try:
92        runner = test_runner2.TestRunner2(port, options, printer)
93        runner._print_config()
94
95        printer.print_update("Collecting tests ...")
96        try:
97            runner.collect_tests(args, last_unexpected_results)
98        except IOError, e:
99            if e.errno == errno.ENOENT:
100                return -1
101            raise
102
103        if options.lint_test_files:
104            return runner.lint()
105
106        printer.print_update("Parsing expectations ...")
107        runner.parse_expectations()
108
109        printer.print_update("Checking build ...")
110        if not port.check_build(runner.needs_http()):
111            _log.error("Build check failed")
112            return -1
113
114        result_summary = runner.set_up_run()
115        if result_summary:
116            num_unexpected_results = runner.run(result_summary)
117            runner.clean_up_run()
118            _log.debug("Testing completed, Exit status: %d" %
119                       num_unexpected_results)
120    finally:
121        printer.cleanup()
122
123    return num_unexpected_results
124
125
126def _set_up_derived_options(port_obj, options):
127    """Sets the options values that depend on other options values."""
128    # We return a list of warnings to print after the printer is initialized.
129    warnings = []
130
131    if options.worker_model is None:
132        options.worker_model = port_obj.default_worker_model()
133
134    if options.worker_model == 'inline':
135        if options.child_processes and int(options.child_processes) > 1:
136            warnings.append("--worker-model=inline overrides --child-processes")
137        options.child_processes = "1"
138    if not options.child_processes:
139        options.child_processes = os.environ.get("WEBKIT_TEST_CHILD_PROCESSES",
140                                                 str(port_obj.default_child_processes()))
141
142    if not options.configuration:
143        options.configuration = port_obj.default_configuration()
144
145    if options.pixel_tests is None:
146        options.pixel_tests = True
147
148    if not options.use_apache:
149        options.use_apache = sys.platform in ('darwin', 'linux2')
150
151    if not options.time_out_ms:
152        if options.configuration == "Debug":
153            options.time_out_ms = str(2 * test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
154        else:
155            options.time_out_ms = str(test_runner.TestRunner.DEFAULT_TEST_TIMEOUT_MS)
156
157    options.slow_time_out_ms = str(5 * int(options.time_out_ms))
158
159    if options.additional_platform_directory:
160        normalized_platform_directories = []
161        for path in options.additional_platform_directory:
162            if not port_obj._filesystem.isabs(path):
163                warnings.append("--additional-platform-directory=%s is ignored since it is not absolute" % path)
164                continue
165            normalized_platform_directories.append(port_obj._filesystem.normpath(path))
166        options.additional_platform_directory = normalized_platform_directories
167
168    return warnings
169
170
171def _gather_unexpected_results(port):
172    """Returns the unexpected results from the previous run, if any."""
173    filesystem = port._filesystem
174    results_directory = port.results_directory()
175    options = port._options
176    last_unexpected_results = []
177    if options.print_last_failures or options.retest_last_failures:
178        unexpected_results_filename = filesystem.join(results_directory, "unexpected_results.json")
179        if filesystem.exists(unexpected_results_filename):
180            results = json_results_generator.load_json(filesystem, unexpected_results_filename)
181            last_unexpected_results = results['tests'].keys()
182    return last_unexpected_results
183
184
185def _compat_shim_callback(option, opt_str, value, parser):
186    print "Ignoring unsupported option: %s" % opt_str
187
188
189def _compat_shim_option(option_name, **kwargs):
190    return optparse.make_option(option_name, action="callback",
191        callback=_compat_shim_callback,
192        help="Ignored, for old-run-webkit-tests compat only.", **kwargs)
193
194
195def parse_args(args=None):
196    """Provides a default set of command line args.
197
198    Returns a tuple of options, args from optparse"""
199
200    # FIXME: All of these options should be stored closer to the code which
201    # FIXME: actually uses them. configuration_options should move
202    # FIXME: to WebKitPort and be shared across all scripts.
203    configuration_options = [
204        optparse.make_option("-t", "--target", dest="configuration",
205                             help="(DEPRECATED)"),
206        # FIXME: --help should display which configuration is default.
207        optparse.make_option('--debug', action='store_const', const='Debug',
208                             dest="configuration",
209                             help='Set the configuration to Debug'),
210        optparse.make_option('--release', action='store_const',
211                             const='Release', dest="configuration",
212                             help='Set the configuration to Release'),
213        # old-run-webkit-tests also accepts -c, --configuration CONFIGURATION.
214    ]
215
216    print_options = printing.print_options()
217
218    # FIXME: These options should move onto the ChromiumPort.
219    chromium_options = [
220        optparse.make_option("--chromium", action="store_true", default=False,
221            help="use the Chromium port"),
222        optparse.make_option("--startup-dialog", action="store_true",
223            default=False, help="create a dialog on DumpRenderTree startup"),
224        optparse.make_option("--gp-fault-error-box", action="store_true",
225            default=False, help="enable Windows GP fault error box"),
226        optparse.make_option("--js-flags",
227            type="string", help="JavaScript flags to pass to tests"),
228        optparse.make_option("--stress-opt", action="store_true",
229            default=False,
230            help="Enable additional stress test to JavaScript optimization"),
231        optparse.make_option("--stress-deopt", action="store_true",
232            default=False,
233            help="Enable additional stress test to JavaScript optimization"),
234        optparse.make_option("--nocheck-sys-deps", action="store_true",
235            default=False,
236            help="Don't check the system dependencies (themes)"),
237        optparse.make_option("--accelerated-compositing",
238            action="store_true",
239            help="Use hardware-accelerated compositing for rendering"),
240        optparse.make_option("--no-accelerated-compositing",
241            action="store_false",
242            dest="accelerated_compositing",
243            help="Don't use hardware-accelerated compositing for rendering"),
244        optparse.make_option("--accelerated-2d-canvas",
245            action="store_true",
246            help="Use hardware-accelerated 2D Canvas calls"),
247        optparse.make_option("--no-accelerated-2d-canvas",
248            action="store_false",
249            dest="accelerated_2d_canvas",
250            help="Don't use hardware-accelerated 2D Canvas calls"),
251        optparse.make_option("--enable-hardware-gpu",
252            action="store_true",
253            default=False,
254            help="Run graphics tests on real GPU hardware vs software"),
255    ]
256
257    # Missing Mac-specific old-run-webkit-tests options:
258    # FIXME: Need: -g, --guard for guard malloc support on Mac.
259    # FIXME: Need: -l --leaks    Enable leaks checking.
260    # FIXME: Need: --sample-on-timeout Run sample on timeout
261
262    old_run_webkit_tests_compat = [
263        # NRWT doesn't generate results by default anyway.
264        _compat_shim_option("--no-new-test-results"),
265        # NRWT doesn't sample on timeout yet anyway.
266        _compat_shim_option("--no-sample-on-timeout"),
267        # FIXME: NRWT needs to support remote links eventually.
268        _compat_shim_option("--use-remote-links-to-tests"),
269    ]
270
271    results_options = [
272        # NEED for bots: --use-remote-links-to-tests Link to test files
273        # within the SVN repository in the results.
274        optparse.make_option("-p", "--pixel-tests", action="store_true",
275            dest="pixel_tests", help="Enable pixel-to-pixel PNG comparisons"),
276        optparse.make_option("--no-pixel-tests", action="store_false",
277            dest="pixel_tests", help="Disable pixel-to-pixel PNG comparisons"),
278        optparse.make_option("--tolerance",
279            help="Ignore image differences less than this percentage (some "
280                "ports may ignore this option)", type="float"),
281        optparse.make_option("--results-directory", help="Location of test results"),
282        optparse.make_option("--build-directory",
283            help="Path to the directory under which build files are kept (should not include configuration)"),
284        optparse.make_option("--new-baseline", action="store_true",
285            default=False, help="Save all generated results as new baselines "
286                 "into the platform directory, overwriting whatever's "
287                 "already there."),
288        optparse.make_option("--reset-results", action="store_true",
289            default=False, help="Reset any existing baselines to the "
290                 "generated results"),
291        optparse.make_option("--additional-drt-flag", action="append",
292            default=[], help="Additional command line flag to pass to DumpRenderTree "
293                 "Specify multiple times to add multiple flags."),
294        optparse.make_option("--additional-platform-directory", action="append",
295            default=[], help="Additional directory where to look for test "
296                 "baselines (will take precendence over platform baselines). "
297                 "Specify multiple times to add multiple search path entries."),
298        optparse.make_option("--no-show-results", action="store_false",
299            default=True, dest="show_results",
300            help="Don't launch a browser with results after the tests "
301                 "are done"),
302        # FIXME: We should have a helper function to do this sort of
303        # deprectated mapping and automatically log, etc.
304        optparse.make_option("--noshow-results", action="store_false",
305            dest="show_results",
306            help="Deprecated, same as --no-show-results."),
307        optparse.make_option("--no-launch-safari", action="store_false",
308            dest="show_results",
309            help="old-run-webkit-tests compat, same as --noshow-results."),
310        # old-run-webkit-tests:
311        # --[no-]launch-safari    Launch (or do not launch) Safari to display
312        #                         test results (default: launch)
313        optparse.make_option("--full-results-html", action="store_true",
314            default=False,
315            help="Show all failures in results.html, rather than only "
316                 "regressions"),
317        optparse.make_option("--clobber-old-results", action="store_true",
318            default=False, help="Clobbers test results from previous runs."),
319        optparse.make_option("--platform",
320            help="Override the platform for expected results"),
321        optparse.make_option("--no-record-results", action="store_false",
322            default=True, dest="record_results",
323            help="Don't record the results."),
324        # old-run-webkit-tests also has HTTP toggle options:
325        # --[no-]http                     Run (or do not run) http tests
326        #                                 (default: run)
327    ]
328
329    test_options = [
330        optparse.make_option("--build", dest="build",
331            action="store_true", default=True,
332            help="Check to ensure the DumpRenderTree build is up-to-date "
333                 "(default)."),
334        optparse.make_option("--no-build", dest="build",
335            action="store_false", help="Don't check to see if the "
336                                       "DumpRenderTree build is up-to-date."),
337        optparse.make_option("-n", "--dry-run", action="store_true",
338            default=False,
339            help="Do everything but actually run the tests or upload results."),
340        # old-run-webkit-tests has --valgrind instead of wrapper.
341        optparse.make_option("--wrapper",
342            help="wrapper command to insert before invocations of "
343                 "DumpRenderTree; option is split on whitespace before "
344                 "running. (Example: --wrapper='valgrind --smc-check=all')"),
345        # old-run-webkit-tests:
346        # -i|--ignore-tests               Comma-separated list of directories
347        #                                 or tests to ignore
348        optparse.make_option("--test-list", action="append",
349            help="read list of tests to run from file", metavar="FILE"),
350        # old-run-webkit-tests uses --skipped==[default|ignore|only]
351        # instead of --force:
352        optparse.make_option("--force", action="store_true", default=False,
353            help="Run all tests, even those marked SKIP in the test list"),
354        optparse.make_option("--use-apache", action="store_true",
355            default=False, help="Whether to use apache instead of lighttpd."),
356        optparse.make_option("--time-out-ms",
357            help="Set the timeout for each test"),
358        # old-run-webkit-tests calls --randomize-order --random:
359        optparse.make_option("--randomize-order", action="store_true",
360            default=False, help=("Run tests in random order (useful "
361                                "for tracking down corruption)")),
362        optparse.make_option("--run-chunk",
363            help=("Run a specified chunk (n:l), the nth of len l, "
364                 "of the layout tests")),
365        optparse.make_option("--run-part", help=("Run a specified part (n:m), "
366                  "the nth of m parts, of the layout tests")),
367        # old-run-webkit-tests calls --batch-size: --nthly n
368        #   Restart DumpRenderTree every n tests (default: 1000)
369        optparse.make_option("--batch-size",
370            help=("Run a the tests in batches (n), after every n tests, "
371                  "DumpRenderTree is relaunched."), type="int", default=0),
372        # old-run-webkit-tests calls --run-singly: -1|--singly
373        # Isolate each test case run (implies --nthly 1 --verbose)
374        optparse.make_option("--run-singly", action="store_true",
375            default=False, help="run a separate DumpRenderTree for each test"),
376        optparse.make_option("--child-processes",
377            help="Number of DumpRenderTrees to run in parallel."),
378        # FIXME: Display default number of child processes that will run.
379        optparse.make_option("--worker-model", action="store",
380            default=None, help=("controls worker model. Valid values are "
381                                "'inline', 'threads', and 'processes'.")),
382        optparse.make_option("--experimental-fully-parallel",
383            action="store_true", default=False,
384            help="run all tests in parallel"),
385        optparse.make_option("--exit-after-n-failures", type="int", default=500,
386            help="Exit after the first N failures instead of running all "
387            "tests"),
388        optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int",
389            default=20, help="Exit after the first N crashes instead of "
390            "running all tests"),
391        # FIXME: consider: --iterations n
392        #      Number of times to run the set of tests (e.g. ABCABCABC)
393        optparse.make_option("--print-last-failures", action="store_true",
394            default=False, help="Print the tests in the last run that "
395            "had unexpected failures (or passes) and then exit."),
396        optparse.make_option("--retest-last-failures", action="store_true",
397            default=False, help="re-test the tests in the last run that "
398            "had unexpected failures (or passes)."),
399        optparse.make_option("--retry-failures", action="store_true",
400            default=True,
401            help="Re-try any tests that produce unexpected results (default)"),
402        optparse.make_option("--no-retry-failures", action="store_false",
403            dest="retry_failures",
404            help="Don't re-try any tests that produce unexpected results."),
405    ]
406
407    misc_options = [
408        optparse.make_option("--lint-test-files", action="store_true",
409        default=False, help=("Makes sure the test files parse for all "
410                            "configurations. Does not run any tests.")),
411    ]
412
413    # FIXME: Move these into json_results_generator.py
414    results_json_options = [
415        optparse.make_option("--master-name", help="The name of the buildbot master."),
416        optparse.make_option("--builder-name", default="DUMMY_BUILDER_NAME",
417            help=("The name of the builder shown on the waterfall running "
418                  "this script e.g. WebKit.")),
419        optparse.make_option("--build-name", default="DUMMY_BUILD_NAME",
420            help=("The name of the builder used in its path, e.g. "
421                  "webkit-rel.")),
422        optparse.make_option("--build-number", default="DUMMY_BUILD_NUMBER",
423            help=("The build number of the builder running this script.")),
424        optparse.make_option("--test-results-server", default="",
425            help=("If specified, upload results json files to this appengine "
426                  "server.")),
427    ]
428
429    option_list = (configuration_options + print_options +
430                   chromium_options + results_options + test_options +
431                   misc_options + results_json_options +
432                   old_run_webkit_tests_compat)
433    option_parser = optparse.OptionParser(option_list=option_list)
434
435    return option_parser.parse_args(args)
436
437
438def main():
439    options, args = parse_args()
440    port_obj = port.get(options.platform, options)
441    return run(port_obj, options, args)
442
443
444if '__main__' == __name__:
445    try:
446        sys.exit(main())
447    except KeyboardInterrupt:
448        # this mirrors what the shell normally does
449        sys.exit(signal.SIGINT + 128)
450