1# Copyright (C) 2011 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30import logging
31import time
32
33from webkitpy.layout_tests.port import base
34from webkitpy.layout_tests.layout_package import test_failures
35from webkitpy.layout_tests.layout_package import test_result_writer
36from webkitpy.layout_tests.layout_package.test_results import TestResult
37
38
39_log = logging.getLogger(__name__)
40
41
42def run_single_test(port, options, test_input, driver, worker_name):
43    runner = SingleTestRunner(options, port, driver, test_input, worker_name)
44    return runner.run()
45
46
47class SingleTestRunner:
48
49    def __init__(self, options, port, driver, test_input, worker_name):
50        self._options = options
51        self._port = port
52        self._driver = driver
53        self._filename = test_input.filename
54        self._timeout = test_input.timeout
55        self._worker_name = worker_name
56        self._testname = port.relative_test_filename(test_input.filename)
57
58        self._is_reftest = False
59        self._is_mismatch_reftest = False
60        self._reference_filename = None
61
62        fs = port._filesystem
63        reftest_expected_filename = port.reftest_expected_filename(self._filename)
64        if fs.exists(reftest_expected_filename):
65            self._is_reftest = True
66            self._reference_filename = reftest_expected_filename
67
68        reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._filename)
69        if fs.exists(reftest_expected_mismatch_filename):
70            if self._is_reftest:
71                _log.error('It is not allowed that one test file has both'
72                           ' expected.html file and expected-mismatch.html file'
73                           ' at the same time. Please remove either %s or %s.',
74                           reftest_expected_filename, reftest_expected_mismatch_filename)
75            else:
76                self._is_reftest = True
77                self._is_mismatch_reftest = True
78                self._reference_filename = reftest_expected_mismatch_filename
79
80        if self._is_reftest:
81            # Detect and report a test which has a wrong combination of expectation files.
82            # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
83            # 'foo-expected.txt', we should warn users. One test file must be used exclusively
84            # in either layout tests or reftests, but not in both.
85            for suffix in ('.txt', '.checksum', '.png', '.wav'):
86                expected_filename = self._port.expected_filename(self._filename, suffix)
87                if fs.exists(expected_filename):
88                    _log.error('The reftest (%s) can not have an expectation file (%s).'
89                               ' Please remove that file.', self._testname, expected_filename)
90
91    def _expected_driver_output(self):
92        return base.DriverOutput(self._port.expected_text(self._filename),
93                                 self._port.expected_image(self._filename),
94                                 self._port.expected_checksum(self._filename),
95                                 self._port.expected_audio(self._filename))
96
97    def _should_fetch_expected_checksum(self):
98        return (self._options.pixel_tests and
99                not (self._options.new_baseline or self._options.reset_results))
100
101    def _driver_input(self):
102        # The image hash is used to avoid doing an image dump if the
103        # checksums match, so it should be set to a blank value if we
104        # are generating a new baseline.  (Otherwise, an image from a
105        # previous run will be copied into the baseline."""
106        image_hash = None
107        if self._should_fetch_expected_checksum():
108            image_hash = self._port.expected_checksum(self._filename)
109        return base.DriverInput(self._filename, self._timeout, image_hash)
110
111    def run(self):
112        if self._options.new_baseline or self._options.reset_results:
113            if self._is_reftest:
114                # Returns a dummy TestResult. We don't have to rebase for reftests.
115                return TestResult(self._filename)
116            else:
117                return self._run_rebaseline()
118        if self._is_reftest:
119            return self._run_reftest()
120        return self._run_compare_test()
121
122    def _run_compare_test(self):
123        driver_output = self._driver.run_test(self._driver_input())
124        expected_driver_output = self._expected_driver_output()
125        test_result = self._compare_output(driver_output, expected_driver_output)
126        test_result_writer.write_test_result(self._port, self._filename,
127                                             driver_output, expected_driver_output, test_result.failures)
128        return test_result
129
130    def _run_rebaseline(self):
131        driver_output = self._driver.run_test(self._driver_input())
132        failures = self._handle_error(driver_output)
133        test_result_writer.write_test_result(self._port, self._filename,
134                                             driver_output, None, failures)
135        # FIXME: It the test crashed or timed out, it might be bettter to avoid
136        # to write new baselines.
137        self._save_baselines(driver_output)
138        return TestResult(self._filename, failures, driver_output.test_time)
139
140    def _save_baselines(self, driver_output):
141        # Although all test_shell/DumpRenderTree output should be utf-8,
142        # we do not ever decode it inside run-webkit-tests.  For some tests
143        # DumpRenderTree may not output utf-8 text (e.g. webarchives).
144        self._save_baseline_data(driver_output.text, ".txt",
145                                 generate_new_baseline=self._options.new_baseline)
146        if driver_output.audio:
147            self._save_baseline_data(driver_output.audio, '.wav',
148                                     generate_new_baseline=self._options.new_baseline)
149        if self._options.pixel_tests and driver_output.image_hash:
150            self._save_baseline_data(driver_output.image, ".png",
151                                     generate_new_baseline=self._options.new_baseline)
152            self._save_baseline_data(driver_output.image_hash, ".checksum",
153                                     generate_new_baseline=self._options.new_baseline)
154
155    def _save_baseline_data(self, data, modifier, generate_new_baseline=True):
156        """Saves a new baseline file into the port's baseline directory.
157
158        The file will be named simply "<test>-expected<modifier>", suitable for
159        use as the expected results in a later run.
160
161        Args:
162          data: result to be saved as the new baseline
163          modifier: type of the result file, e.g. ".txt" or ".png"
164          generate_new_baseline: whether to enerate a new, platform-specific
165            baseline, or update the existing one
166        """
167        assert data is not None
168        port = self._port
169        fs = port._filesystem
170        if generate_new_baseline:
171            relative_dir = fs.dirname(self._testname)
172            baseline_path = port.baseline_path()
173            output_dir = fs.join(baseline_path, relative_dir)
174            output_file = fs.basename(fs.splitext(self._filename)[0] +
175                "-expected" + modifier)
176            fs.maybe_make_directory(output_dir)
177            output_path = fs.join(output_dir, output_file)
178            _log.debug('writing new baseline result "%s"' % (output_path))
179        else:
180            output_path = port.expected_filename(self._filename, modifier)
181            _log.debug('resetting baseline result "%s"' % output_path)
182
183        port.update_baseline(output_path, data)
184
185    def _handle_error(self, driver_output, reference_filename=None):
186        """Returns test failures if some unusual errors happen in driver's run.
187
188        Args:
189          driver_output: The output from the driver.
190          reference_filename: The full path to the reference file which produced the driver_output.
191              This arg is optional and should be used only in reftests until we have a better way to know
192              which html file is used for producing the driver_output.
193        """
194        failures = []
195        fs = self._port._filesystem
196        if driver_output.timeout:
197            failures.append(test_failures.FailureTimeout(bool(reference_filename)))
198
199        if reference_filename:
200            testname = self._port.relative_test_filename(reference_filename)
201        else:
202            testname = self._testname
203
204        if driver_output.crash:
205            failures.append(test_failures.FailureCrash(bool(reference_filename)))
206            _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname,
207                                                      driver_output.error))
208        elif driver_output.error:
209            _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, testname,
210                                                           driver_output.error))
211        return failures
212
213    def _compare_output(self, driver_output, expected_driver_output):
214        failures = []
215        failures.extend(self._handle_error(driver_output))
216
217        if driver_output.crash:
218            # Don't continue any more if we already have a crash.
219            # In case of timeouts, we continue since we still want to see the text and image output.
220            return TestResult(self._filename, failures, driver_output.test_time)
221
222        failures.extend(self._compare_text(driver_output.text, expected_driver_output.text))
223        failures.extend(self._compare_audio(driver_output.audio, expected_driver_output.audio))
224        if self._options.pixel_tests:
225            failures.extend(self._compare_image(driver_output, expected_driver_output))
226        return TestResult(self._filename, failures, driver_output.test_time)
227
228    def _compare_text(self, actual_text, expected_text):
229        failures = []
230        if (expected_text and actual_text and
231            # Assuming expected_text is already normalized.
232            self._port.compare_text(self._get_normalized_output_text(actual_text), expected_text)):
233            failures.append(test_failures.FailureTextMismatch())
234        elif actual_text and not expected_text:
235            failures.append(test_failures.FailureMissingResult())
236        return failures
237
238    def _compare_audio(self, actual_audio, expected_audio):
239        failures = []
240        if (expected_audio and actual_audio and
241            self._port.compare_audio(actual_audio, expected_audio)):
242            failures.append(test_failures.FailureAudioMismatch())
243        elif actual_audio and not expected_audio:
244            failures.append(test_failures.FailureMissingAudio())
245        return failures
246
247    def _get_normalized_output_text(self, output):
248        """Returns the normalized text output, i.e. the output in which
249        the end-of-line characters are normalized to "\n"."""
250        # Running tests on Windows produces "\r\n".  The "\n" part is helpfully
251        # changed to "\r\n" by our system (Python/Cygwin), resulting in
252        # "\r\r\n", when, in fact, we wanted to compare the text output with
253        # the normalized text expectation files.
254        return output.replace("\r\r\n", "\r\n").replace("\r\n", "\n")
255
256    def _compare_image(self, driver_output, expected_driver_outputs):
257        failures = []
258        # If we didn't produce a hash file, this test must be text-only.
259        if driver_output.image_hash is None:
260            return failures
261        if not expected_driver_outputs.image:
262            failures.append(test_failures.FailureMissingImage())
263        elif not expected_driver_outputs.image_hash:
264            failures.append(test_failures.FailureMissingImageHash())
265        elif driver_output.image_hash != expected_driver_outputs.image_hash:
266            failures.append(test_failures.FailureImageHashMismatch())
267        return failures
268
269    def _run_reftest(self):
270        driver_output1 = self._driver.run_test(self._driver_input())
271        driver_output2 = self._driver.run_test(
272            base.DriverInput(self._reference_filename, self._timeout, driver_output1.image_hash))
273        test_result = self._compare_output_with_reference(driver_output1, driver_output2)
274
275        test_result_writer.write_test_result(self._port, self._filename,
276                                             driver_output1, driver_output2, test_result.failures)
277        return test_result
278
279    def _compare_output_with_reference(self, driver_output1, driver_output2):
280        total_test_time = driver_output1.test_time + driver_output2.test_time
281        failures = []
282        failures.extend(self._handle_error(driver_output1))
283        if failures:
284            # Don't continue any more if we already have crash or timeout.
285            return TestResult(self._filename, failures, total_test_time)
286        failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename))
287        if failures:
288            return TestResult(self._filename, failures, total_test_time)
289
290        if self._is_mismatch_reftest:
291            if driver_output1.image_hash == driver_output2.image_hash:
292                failures.append(test_failures.FailureReftestMismatchDidNotOccur())
293        elif driver_output1.image_hash != driver_output2.image_hash:
294            failures.append(test_failures.FailureReftestMismatch())
295        return TestResult(self._filename, failures, total_test_time)
296