1# Copyright (c) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29"""Starts a local HTTP server which displays layout test failures (given a test
30results directory), provides comparisons of expected and actual results (both
31images and text) and allows one-click rebaselining of tests."""
32from __future__ import with_statement
33
34import codecs
35import datetime
36import fnmatch
37import mimetypes
38import os
39import os.path
40import shutil
41import threading
42import time
43import urlparse
44import BaseHTTPServer
45
46from optparse import make_option
47from wsgiref.handlers import format_date_time
48
49from webkitpy.common import system
50from webkitpy.layout_tests.layout_package import json_results_generator
51from webkitpy.layout_tests.port import factory
52from webkitpy.layout_tests.port.webkit import WebKitPort
53from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
54from webkitpy.thirdparty import simplejson
55
56STATE_NEEDS_REBASELINE = 'needs_rebaseline'
57STATE_REBASELINE_FAILED = 'rebaseline_failed'
58STATE_REBASELINE_SUCCEEDED = 'rebaseline_succeeded'
59
60class RebaselineHTTPServer(BaseHTTPServer.HTTPServer):
61    def __init__(self, httpd_port, test_config, results_json, platforms_json):
62        BaseHTTPServer.HTTPServer.__init__(self, ("", httpd_port), RebaselineHTTPRequestHandler)
63        self.test_config = test_config
64        self.results_json = results_json
65        self.platforms_json = platforms_json
66
67
68class RebaselineHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
69    STATIC_FILE_NAMES = frozenset([
70        "index.html",
71        "loupe.js",
72        "main.js",
73        "main.css",
74        "queue.js",
75        "util.js",
76    ])
77
78    STATIC_FILE_DIRECTORY = os.path.join(
79        os.path.dirname(__file__), "data", "rebaselineserver")
80
81    def do_GET(self):
82        self._handle_request()
83
84    def do_POST(self):
85        self._handle_request()
86
87    def _handle_request(self):
88        # Parse input.
89        if "?" in self.path:
90            path, query_string = self.path.split("?", 1)
91            self.query = urlparse.parse_qs(query_string)
92        else:
93            path = self.path
94            self.query = {}
95        function_or_file_name = path[1:] or "index.html"
96
97        # See if a static file matches.
98        if function_or_file_name in RebaselineHTTPRequestHandler.STATIC_FILE_NAMES:
99            self._serve_static_file(function_or_file_name)
100            return
101
102        # See if a class method matches.
103        function_name = function_or_file_name.replace(".", "_")
104        if not hasattr(self, function_name):
105            self.send_error(404, "Unknown function %s" % function_name)
106            return
107        if function_name[0] == "_":
108            self.send_error(
109                401, "Not allowed to invoke private or protected methods")
110            return
111        function = getattr(self, function_name)
112        function()
113
114    def _serve_static_file(self, static_path):
115        self._serve_file(os.path.join(
116            RebaselineHTTPRequestHandler.STATIC_FILE_DIRECTORY, static_path))
117
118    def rebaseline(self):
119        test = self.query['test'][0]
120        baseline_target = self.query['baseline-target'][0]
121        baseline_move_to = self.query['baseline-move-to'][0]
122        test_json = self.server.results_json['tests'][test]
123
124        if test_json['state'] != STATE_NEEDS_REBASELINE:
125            self.send_error(400, "Test %s is in unexpected state: %s" %
126                (test, test_json["state"]))
127            return
128
129        log = []
130        success = _rebaseline_test(
131            test,
132            baseline_target,
133            baseline_move_to,
134            self.server.test_config,
135            log=lambda l: log.append(l))
136
137        if success:
138            test_json['state'] = STATE_REBASELINE_SUCCEEDED
139            self.send_response(200)
140        else:
141            test_json['state'] = STATE_REBASELINE_FAILED
142            self.send_response(500)
143
144        self.send_header('Content-type', 'text/plain')
145        self.end_headers()
146        self.wfile.write('\n'.join(log))
147
148    def quitquitquit(self):
149        self.send_response(200)
150        self.send_header("Content-type", "text/plain")
151        self.end_headers()
152        self.wfile.write("Quit.\n")
153
154        # Shutdown has to happen on another thread from the server's thread,
155        # otherwise there's a deadlock
156        threading.Thread(target=lambda: self.server.shutdown()).start()
157
158    def test_result(self):
159        test_name, _ = os.path.splitext(self.query['test'][0])
160        mode = self.query['mode'][0]
161        if mode == 'expected-image':
162            file_name = test_name + '-expected.png'
163        elif mode == 'actual-image':
164            file_name = test_name + '-actual.png'
165        if mode == 'expected-checksum':
166            file_name = test_name + '-expected.checksum'
167        elif mode == 'actual-checksum':
168            file_name = test_name + '-actual.checksum'
169        elif mode == 'diff-image':
170            file_name = test_name + '-diff.png'
171        if mode == 'expected-text':
172            file_name = test_name + '-expected.txt'
173        elif mode == 'actual-text':
174            file_name = test_name + '-actual.txt'
175        elif mode == 'diff-text':
176            file_name = test_name + '-diff.txt'
177        elif mode == 'diff-text-pretty':
178            file_name = test_name + '-pretty-diff.html'
179
180        file_path = os.path.join(self.server.test_config.results_directory, file_name)
181
182        # Let results be cached for 60 seconds, so that they can be pre-fetched
183        # by the UI
184        self._serve_file(file_path, cacheable_seconds=60)
185
186    def results_json(self):
187        self._serve_json(self.server.results_json)
188
189    def platforms_json(self):
190        self._serve_json(self.server.platforms_json)
191
192    def _serve_json(self, json):
193        self.send_response(200)
194        self.send_header('Content-type', 'application/json')
195        self.end_headers()
196        simplejson.dump(json, self.wfile)
197
198    def _serve_file(self, file_path, cacheable_seconds=0):
199        if not os.path.exists(file_path):
200            self.send_error(404, "File not found")
201            return
202        with codecs.open(file_path, "rb") as static_file:
203            self.send_response(200)
204            self.send_header("Content-Length", os.path.getsize(file_path))
205            mime_type, encoding = mimetypes.guess_type(file_path)
206            if mime_type:
207                self.send_header("Content-type", mime_type)
208
209            if cacheable_seconds:
210                expires_time = (datetime.datetime.now() +
211                    datetime.timedelta(0, cacheable_seconds))
212                expires_formatted = format_date_time(
213                    time.mktime(expires_time.timetuple()))
214                self.send_header("Expires", expires_formatted)
215            self.end_headers()
216
217            shutil.copyfileobj(static_file, self.wfile)
218
219
220class TestConfig(object):
221    def __init__(self, test_port, layout_tests_directory, results_directory, platforms, filesystem, scm):
222        self.test_port = test_port
223        self.layout_tests_directory = layout_tests_directory
224        self.results_directory = results_directory
225        self.platforms = platforms
226        self.filesystem = filesystem
227        self.scm = scm
228
229
230def _get_actual_result_files(test_file, test_config):
231    test_name, _ = os.path.splitext(test_file)
232    test_directory = os.path.dirname(test_file)
233
234    test_results_directory = test_config.filesystem.join(
235        test_config.results_directory, test_directory)
236    actual_pattern = os.path.basename(test_name) + '-actual.*'
237    actual_files = []
238    for filename in test_config.filesystem.listdir(test_results_directory):
239        if fnmatch.fnmatch(filename, actual_pattern):
240            actual_files.append(filename)
241    actual_files.sort()
242    return tuple(actual_files)
243
244
245def _rebaseline_test(test_file, baseline_target, baseline_move_to, test_config, log):
246    test_name, _ = os.path.splitext(test_file)
247    test_directory = os.path.dirname(test_name)
248
249    log('Rebaselining %s...' % test_name)
250
251    actual_result_files = _get_actual_result_files(test_file, test_config)
252    filesystem = test_config.filesystem
253    scm = test_config.scm
254    layout_tests_directory = test_config.layout_tests_directory
255    results_directory = test_config.results_directory
256    target_expectations_directory = filesystem.join(
257        layout_tests_directory, 'platform', baseline_target, test_directory)
258    test_results_directory = test_config.filesystem.join(
259        test_config.results_directory, test_directory)
260
261    # If requested, move current baselines out
262    current_baselines = _get_test_baselines(test_file, test_config)
263    if baseline_target in current_baselines and baseline_move_to != 'none':
264        log('  Moving current %s baselines to %s' %
265            (baseline_target, baseline_move_to))
266
267        # See which ones we need to move (only those that are about to be
268        # updated), and make sure we're not clobbering any files in the
269        # destination.
270        current_extensions = set(current_baselines[baseline_target].keys())
271        actual_result_extensions = [
272            os.path.splitext(f)[1] for f in actual_result_files]
273        extensions_to_move = current_extensions.intersection(
274            actual_result_extensions)
275
276        if extensions_to_move.intersection(
277            current_baselines.get(baseline_move_to, {}).keys()):
278            log('    Already had baselines in %s, could not move existing '
279                '%s ones' % (baseline_move_to, baseline_target))
280            return False
281
282        # Do the actual move.
283        if extensions_to_move:
284            if not _move_test_baselines(
285                test_file,
286                list(extensions_to_move),
287                baseline_target,
288                baseline_move_to,
289                test_config,
290                log):
291                return False
292        else:
293            log('    No current baselines to move')
294
295    log('  Updating baselines for %s' % baseline_target)
296    filesystem.maybe_make_directory(target_expectations_directory)
297    for source_file in actual_result_files:
298        source_path = filesystem.join(test_results_directory, source_file)
299        destination_file = source_file.replace('-actual', '-expected')
300        destination_path = filesystem.join(
301            target_expectations_directory, destination_file)
302        filesystem.copyfile(source_path, destination_path)
303        exit_code = scm.add(destination_path, return_exit_code=True)
304        if exit_code:
305            log('    Could not update %s in SCM, exit code %d' %
306                (destination_file, exit_code))
307            return False
308        else:
309            log('    Updated %s' % destination_file)
310
311    return True
312
313
314def _move_test_baselines(test_file, extensions_to_move, source_platform, destination_platform, test_config, log):
315    test_file_name = os.path.splitext(os.path.basename(test_file))[0]
316    test_directory = os.path.dirname(test_file)
317    filesystem = test_config.filesystem
318
319    # Want predictable output order for unit tests.
320    extensions_to_move.sort()
321
322    source_directory = os.path.join(
323        test_config.layout_tests_directory,
324        'platform',
325        source_platform,
326        test_directory)
327    destination_directory = os.path.join(
328        test_config.layout_tests_directory,
329        'platform',
330        destination_platform,
331        test_directory)
332    filesystem.maybe_make_directory(destination_directory)
333
334    for extension in extensions_to_move:
335        file_name = test_file_name + '-expected' + extension
336        source_path = filesystem.join(source_directory, file_name)
337        destination_path = filesystem.join(destination_directory, file_name)
338        filesystem.copyfile(source_path, destination_path)
339        exit_code = test_config.scm.add(destination_path, return_exit_code=True)
340        if exit_code:
341            log('    Could not update %s in SCM, exit code %d' %
342                (file_name, exit_code))
343            return False
344        else:
345            log('    Moved %s' % file_name)
346
347    return True
348
349def _get_test_baselines(test_file, test_config):
350    class AllPlatformsPort(WebKitPort):
351        def __init__(self):
352            WebKitPort.__init__(self, filesystem=test_config.filesystem)
353            self._platforms_by_directory = dict(
354                [(self._webkit_baseline_path(p), p) for p in test_config.platforms])
355
356        def baseline_search_path(self):
357            return self._platforms_by_directory.keys()
358
359        def platform_from_directory(self, directory):
360            return self._platforms_by_directory[directory]
361
362    test_path = test_config.filesystem.join(
363        test_config.layout_tests_directory, test_file)
364
365    all_platforms_port = AllPlatformsPort()
366
367    all_test_baselines = {}
368    for baseline_extension in ('.txt', '.checksum', '.png'):
369        test_baselines = test_config.test_port.expected_baselines(
370            test_path, baseline_extension)
371        baselines = all_platforms_port.expected_baselines(
372            test_path, baseline_extension, all_baselines=True)
373        for platform_directory, expected_filename in baselines:
374            if not platform_directory:
375                continue
376            if platform_directory == test_config.layout_tests_directory:
377                platform = 'base'
378            else:
379                platform = all_platforms_port.platform_from_directory(
380                    platform_directory)
381            platform_baselines = all_test_baselines.setdefault(platform, {})
382            was_used_for_test = (
383                platform_directory, expected_filename) in test_baselines
384            platform_baselines[baseline_extension] = was_used_for_test
385
386    return all_test_baselines
387
388
389class RebaselineServer(AbstractDeclarativeCommand):
390    name = "rebaseline-server"
391    help_text = __doc__
392    argument_names = "/path/to/results/directory"
393
394    def __init__(self):
395        options = [
396            make_option("--httpd-port", action="store", type="int", default=8127, help="Port to use for the the rebaseline HTTP server"),
397        ]
398        AbstractDeclarativeCommand.__init__(self, options=options)
399
400    def execute(self, options, args, tool):
401        results_directory = args[0]
402        filesystem = system.filesystem.FileSystem()
403        scm = self._tool.scm()
404
405        if options.dry_run:
406
407            def no_op_copyfile(src, dest):
408                pass
409
410            def no_op_add(path, return_exit_code=False):
411                if return_exit_code:
412                    return 0
413
414            filesystem.copyfile = no_op_copyfile
415            scm.add = no_op_add
416
417        print 'Parsing unexpected_results.json...'
418        results_json_path = filesystem.join(results_directory, 'unexpected_results.json')
419        results_json = json_results_generator.load_json(filesystem, results_json_path)
420
421        port = factory.get()
422        layout_tests_directory = port.layout_tests_dir()
423        platforms = filesystem.listdir(
424            filesystem.join(layout_tests_directory, 'platform'))
425        test_config = TestConfig(
426            port,
427            layout_tests_directory,
428            results_directory,
429            platforms,
430            filesystem,
431            scm)
432
433        print 'Gathering current baselines...'
434        for test_file, test_json in results_json['tests'].items():
435            test_json['state'] = STATE_NEEDS_REBASELINE
436            test_path = filesystem.join(layout_tests_directory, test_file)
437            test_json['baselines'] = _get_test_baselines(test_file, test_config)
438
439        server_url = "http://localhost:%d/" % options.httpd_port
440        print "Starting server at %s" % server_url
441        print ("Use the 'Exit' link in the UI, %squitquitquit "
442            "or Ctrl-C to stop") % server_url
443
444        threading.Timer(
445            .1, lambda: self._tool.user.open_url(server_url)).start()
446
447        httpd = RebaselineHTTPServer(
448            httpd_port=options.httpd_port,
449            test_config=test_config,
450            results_json=results_json,
451            platforms_json={
452                'platforms': platforms,
453                'defaultPlatform': port.name(),
454            })
455        httpd.serve_forever()
456