layout_tests/port/base.py

#!/usr/bin/env python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the Google name nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Abstract base class of Port-specific entrypoints for the layout tests
test infrastructure (the Port and Driver classes)."""

from __future__ import with_statement

import cgi
import difflib
import errno
import os
import shlex
import sys
import time

# Handle Python < 2.6 where multiprocessing isn't available.
try:
    import multiprocessing
except ImportError:
    multiprocessing = None

from webkitpy.common import system
from webkitpy.common.system import filesystem
from webkitpy.common.system import logutils
from webkitpy.common.system import path
from webkitpy.common.system.executive import Executive, ScriptError
from webkitpy.common.system.user import User
from webkitpy.layout_tests import read_checksum_from_png
from webkitpy.layout_tests.port import apache_http_server
from webkitpy.layout_tests.port import config as port_config
from webkitpy.layout_tests.port import http_lock
from webkitpy.layout_tests.port import http_server
from webkitpy.layout_tests.port import test_files
from webkitpy.layout_tests.port import websocket_server

_log = logutils.get_logger(__file__)


class DummyOptions(object):
    """Fake implementation of optparse.Values. Cloned from
    webkitpy.tool.mocktool.MockOptions.

    """

    def __init__(self, **kwargs):
        # The caller can set option values using keyword arguments. We don't
        # set any values by default because we don't know how this
        # object will be used. Generally speaking unit tests should
        # subclass this or provider wrapper functions that set a common
        # set of options.
        for key, value in kwargs.items():
            self.__dict__[key] = value


# FIXME: This class should merge with webkitpy.webkit_port at some point.
class Port(object):
    """Abstract class for Port-specific hooks for the layout_test package."""

    def __init__(self, port_name=None, options=None,
                 executive=None,
                 user=None,
                 filesystem=None,
                 config=None,
                 **kwargs):
        self._name = port_name

        # These are default values that should be overridden in a subclasses.
        # FIXME: These should really be passed in.
        self._operating_system = 'mac'
        self._version = ''
        self._architecture = 'x86'
        self._graphics_type = 'cpu'

        # FIXME: Ideally we'd have a package-wide way to get a
        # well-formed options object that had all of the necessary
        # options defined on it.
        self._options = options or DummyOptions()

        self._executive = executive or Executive()
        self._user = user or User()
        self._filesystem = filesystem or system.filesystem.FileSystem()
        self._config = config or port_config.Config(self._executive, self._filesystem)

        self._helper = None
        self._http_server = None
        self._webkit_base_dir = None
        self._websocket_server = None
        self._http_lock = None

        # Python's Popen has a bug that causes any pipes opened to a
        # process that can't be executed to be leaked.  Since this
        # code is specifically designed to tolerate exec failures
        # to gracefully handle cases where wdiff is not installed,
        # the bug results in a massive file descriptor leak. As a
        # workaround, if an exec failure is ever experienced for
        # wdiff, assume it's not available.  This will leak one
        # file descriptor but that's better than leaking each time
        # wdiff would be run.
        #
        # http://mail.python.org/pipermail/python-list/
        #    2008-August/505753.html
        # http://bugs.python.org/issue3210
        self._wdiff_available = True

        # FIXME: prettypatch.py knows this path, why is it copied here?
        self._pretty_patch_path = self.path_from_webkit_base("Websites",
            "bugs.webkit.org", "PrettyPatch", "prettify.rb")
        self._pretty_patch_available = None

        if not hasattr(self._options, 'configuration') or self._options.configuration is None:
            self._options.configuration = self.default_configuration()
        self._test_configuration = None
        self._multiprocessing_is_available = (multiprocessing is not None)
        self._results_directory = None

    def wdiff_available(self):
        return bool(self._wdiff_available)

    def pretty_patch_available(self):
        return bool(self._pretty_patch_available)

    def default_child_processes(self):
        """Return the number of DumpRenderTree instances to use for this
        port."""
        return self._executive.cpu_count()

    def default_worker_model(self):
        if self._multiprocessing_is_available:
            return 'processes'
        return 'threads'

    def baseline_path(self):
        """Return the absolute path to the directory to store new baselines
        in for this port."""
        raise NotImplementedError('Port.baseline_path')

    def baseline_search_path(self):
        """Return a list of absolute paths to directories to search under for
        baselines. The directories are searched in order."""
        raise NotImplementedError('Port.baseline_search_path')

    def check_build(self, needs_http):
        """This routine is used to ensure that the build is up to date
        and all the needed binaries are present."""
        raise NotImplementedError('Port.check_build')

    def check_sys_deps(self, needs_http):
        """If the port needs to do some runtime checks to ensure that the
        tests can be run successfully, it should override this routine.
        This step can be skipped with --nocheck-sys-deps.

        Returns whether the system is properly configured."""
        return True

    def check_image_diff(self, override_step=None, logging=True):
        """This routine is used to check whether image_diff binary exists."""
        raise NotImplementedError('Port.check_image_diff')

    def check_pretty_patch(self, logging=True):
        """Checks whether we can use the PrettyPatch ruby script."""
        # check if Ruby is installed
        try:
            result = self._executive.run_command(['ruby', '--version'])
        except OSError, e:
            if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
                if logging:
                    _log.error("Ruby is not installed; can't generate pretty patches.")
                    _log.error('')
                return False

        if not self.path_exists(self._pretty_patch_path):
            if logging:
                _log.error("Unable to find %s; can't generate pretty patches." % self._pretty_patch_path)
                _log.error('')
            return False

        return True

    def compare_text(self, expected_text, actual_text):
        """Return whether or not the two strings are *not* equal. This
        routine is used to diff text output.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""
        return expected_text != actual_text

    def compare_audio(self, expected_audio, actual_audio):
        """Return whether the two audio files are *not* equal."""
        return expected_audio != actual_audio

    def diff_image(self, expected_contents, actual_contents,
                   diff_filename=None, tolerance=0):
        """Compare two images and produce a delta image file.

        Return True if the two images are different, False if they are the same.
        Also produce a delta image of the two images and write that into
        |diff_filename| if it is not None.

        |tolerance| should be a percentage value (0.0 - 100.0).
        If it is omitted, the port default tolerance value is used.

        """
        raise NotImplementedError('Port.diff_image')


    def diff_text(self, expected_text, actual_text,
                  expected_filename, actual_filename):
        """Returns a string containing the diff of the two text strings
        in 'unified diff' format.

        While this is a generic routine, we include it in the Port
        interface so that it can be overriden for testing purposes."""

        # The filenames show up in the diff output, make sure they're
        # raw bytes and not unicode, so that they don't trigger join()
        # trying to decode the input.
        def to_raw_bytes(str):
            if isinstance(str, unicode):
                return str.encode('utf-8')
            return str
        expected_filename = to_raw_bytes(expected_filename)
        actual_filename = to_raw_bytes(actual_filename)
        diff = difflib.unified_diff(expected_text.splitlines(True),
                                    actual_text.splitlines(True),
                                    expected_filename,
                                    actual_filename)
        return ''.join(diff)

    def driver_name(self):
        """Returns the name of the actual binary that is performing the test,
        so that it can be referred to in log messages. In most cases this
        will be DumpRenderTree, but if a port uses a binary with a different
        name, it can be overridden here."""
        return "DumpRenderTree"

    def expected_baselines(self, filename, suffix, all_baselines=False):
        """Given a test name, finds where the baseline results are located.

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g.
            '.txt' or '.png'.  This should not be None, but may be an empty
            string.
        all_baselines: If True, return an ordered list of all baseline paths
            for the given platform. If False, return only the first one.
        Returns
        a list of ( platform_dir, results_filename ), where
            platform_dir - abs path to the top of the results tree (or test
                tree)
            results_filename - relative path from top of tree to the results
                file
            (port.join() of the two gives you the full path to the file,
                unless None was returned.)
        Return values will be in the format appropriate for the current
        platform (e.g., "\\" for path separators on Windows). If the results
        file is not found, then None will be returned for the directory,
        but the expected relative pathname will still be returned.

        This routine is generic but lives here since it is used in
        conjunction with the other baseline and filename routines that are
        platform specific.
        """
        testname = self._filesystem.splitext(self.relative_test_filename(filename))[0]

        baseline_filename = testname + '-expected' + suffix

        baseline_search_path = self.get_option('additional_platform_directory', []) + self.baseline_search_path()

        baselines = []
        for platform_dir in baseline_search_path:
            if self.path_exists(self._filesystem.join(platform_dir,
                                                      baseline_filename)):
                baselines.append((platform_dir, baseline_filename))

            if not all_baselines and baselines:
                return baselines

        # If it wasn't found in a platform directory, return the expected
        # result in the test directory, even if no such file actually exists.
        platform_dir = self.layout_tests_dir()
        if self.path_exists(self._filesystem.join(platform_dir,
                                                  baseline_filename)):
            baselines.append((platform_dir, baseline_filename))

        if baselines:
            return baselines

        return [(None, baseline_filename)]

    def expected_filename(self, filename, suffix):
        """Given a test name, returns an absolute path to its expected results.

        If no expected results are found in any of the searched directories,
        the directory in which the test itself is located will be returned.
        The return value is in the format appropriate for the platform
        (e.g., "\\" for path separators on windows).

        Args:
        filename: absolute filename to test file
        suffix: file suffix of the expected results, including dot; e.g. '.txt'
            or '.png'.  This should not be None, but may be an empty string.
        platform: the most-specific directory name to use to build the
            search list of directories, e.g., 'chromium-win', or
            'chromium-mac-leopard' (we follow the WebKit format)

        This routine is generic but is implemented here to live alongside
        the other baseline and filename manipulation routines.
        """
        platform_dir, baseline_filename = self.expected_baselines(
            filename, suffix)[0]
        if platform_dir:
            return self._filesystem.join(platform_dir, baseline_filename)
        return self._filesystem.join(self.layout_tests_dir(), baseline_filename)

    def expected_checksum(self, test):
        """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
        png_path = self.expected_filename(test, '.png')
        checksum_path = self._filesystem.splitext(png_path)[0] + '.checksum'

        if self.path_exists(checksum_path):
            return self._filesystem.read_binary_file(checksum_path)

        if self.path_exists(png_path):
            with self._filesystem.open_binary_file_for_reading(png_path) as filehandle:
                return read_checksum_from_png.read_checksum(filehandle)

        return None

    def expected_image(self, test):
        """Returns the image we expect the test to produce."""
        path = self.expected_filename(test, '.png')
        if not self.path_exists(path):
            return None
        return self._filesystem.read_binary_file(path)

    def expected_audio(self, test):
        path = self.expected_filename(test, '.wav')
        if not self.path_exists(path):
            return None
        return self._filesystem.read_binary_file(path)

    def expected_text(self, test):
        """Returns the text output we expect the test to produce, or None
        if we don't expect there to be any text output.
        End-of-line characters are normalized to '\n'."""
        # FIXME: DRT output is actually utf-8, but since we don't decode the
        # output from DRT (instead treating it as a binary string), we read the
        # baselines as a binary string, too.
        path = self.expected_filename(test, '.txt')
        if not self.path_exists(path):
            return None
        text = self._filesystem.read_binary_file(path)
        return text.replace("\r\n", "\n")

    def reftest_expected_filename(self, filename):
        """Return the filename of reference we expect the test matches."""
        return self.expected_filename(filename, '.html')

    def reftest_expected_mismatch_filename(self, filename):
        """Return the filename of reference we don't expect the test matches."""
        return self.expected_filename(filename, '-mismatch.html')

    def filename_to_uri(self, filename):
        """Convert a test file (which is an absolute path) to a URI."""
        LAYOUTTEST_HTTP_DIR = "http/tests/"
        LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"

        relative_path = self.relative_test_filename(filename)
        port = None
        use_ssl = False

        if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
            or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
            relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
            port = 8000

        # Make http/tests/local run as local files. This is to mimic the
        # logic in run-webkit-tests.
        #
        # TODO(dpranke): remove the SSL reference?
        if (port and not relative_path.startswith("local/")):
            if relative_path.startswith("ssl/"):
                port += 443
                protocol = "https"
            else:
                protocol = "http"
            return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)

        return path.abspath_to_uri(self._filesystem.abspath(filename))

    def tests(self, paths):
        """Return the list of tests found (relative to layout_tests_dir()."""
        return test_files.find(self, paths)

    def test_dirs(self):
        """Returns the list of top-level test directories.

        Used by --clobber-old-results."""
        layout_tests_dir = self.layout_tests_dir()
        return filter(lambda x: self._filesystem.isdir(self._filesystem.join(layout_tests_dir, x)),
                      self._filesystem.listdir(layout_tests_dir))

    def path_isdir(self, path):
        """Return True if the path refers to a directory of tests."""
        # Used by test_expectations.py to apply rules to whole directories.
        return self._filesystem.isdir(path)

    def path_exists(self, path):
        """Return True if the path refers to an existing test or baseline."""
        # Used by test_expectations.py to determine if an entry refers to a
        # valid test and by printing.py to determine if baselines exist.
        return self._filesystem.exists(path)

    def driver_cmd_line(self):
        """Prints the DRT command line that will be used."""
        driver = self.create_driver(0)
        return driver.cmd_line()

    def update_baseline(self, path, data):
        """Updates the baseline for a test.

        Args:
            path: the actual path to use for baseline, not the path to
              the test. This function is used to update either generic or
              platform-specific baselines, but we can't infer which here.
            data: contents of the baseline.
        """
        self._filesystem.write_binary_file(path, data)

    def uri_to_test_name(self, uri):
        """Return the base layout test name for a given URI.

        This returns the test name for a given URI, e.g., if you passed in
        "file:///src/LayoutTests/fast/html/keygen.html" it would return
        "fast/html/keygen.html".

        """
        test = uri
        if uri.startswith("file:///"):
            prefix = path.abspath_to_uri(self.layout_tests_dir()) + "/"
            return test[len(prefix):]

        if uri.startswith("http://127.0.0.1:8880/"):
            # websocket tests
            return test.replace('http://127.0.0.1:8880/', '')

        if uri.startswith("http://"):
            # regular HTTP test
            return test.replace('http://127.0.0.1:8000/', 'http/tests/')

        if uri.startswith("https://"):
            return test.replace('https://127.0.0.1:8443/', 'http/tests/')

        raise NotImplementedError('unknown url type: %s' % uri)

    def layout_tests_dir(self):
        """Return the absolute path to the top of the LayoutTests directory."""
        return self.path_from_webkit_base('LayoutTests')

    def skips_layout_test(self, test_name):
        """Figures out if the givent test is being skipped or not.

        Test categories are handled as well."""
        for test_or_category in self.skipped_layout_tests():
            if test_or_category == test_name:
                return True
            category = self._filesystem.join(self.layout_tests_dir(),
                                             test_or_category)
            if (self._filesystem.isdir(category) and
                test_name.startswith(test_or_category)):
                return True
        return False

    def maybe_make_directory(self, *path):
        """Creates the specified directory if it doesn't already exist."""
        self._filesystem.maybe_make_directory(*path)

    def name(self):
        """Return the name of the port (e.g., 'mac', 'chromium-win-xp')."""
        return self._name

    def operating_system(self):
        return self._operating_system

    def version(self):
        """Returns a string indicating the version of a given platform, e.g.
        'leopard' or 'xp'.

        This is used to help identify the exact port when parsing test
        expectations, determining search paths, and logging information."""
        return self._version

    def graphics_type(self):
        """Returns whether the port uses accelerated graphics ('gpu') or not
        ('cpu')."""
        return self._graphics_type

    def architecture(self):
        return self._architecture

    def real_name(self):
        """Returns the actual name of the port, not the delegate's."""
        return self.name()

    def get_option(self, name, default_value=None):
        # FIXME: Eventually we should not have to do a test for
        # hasattr(), and we should be able to just do
        # self.options.value. See additional FIXME in the constructor.
        if hasattr(self._options, name):
            return getattr(self._options, name)
        return default_value

    def set_option_default(self, name, default_value):
        if not hasattr(self._options, name):
            return setattr(self._options, name, default_value)

    def path_from_webkit_base(self, *comps):
        """Returns the full path to path made by joining the top of the
        WebKit source tree and the list of path components in |*comps|."""
        return self._config.path_from_webkit_base(*comps)

    def script_path(self, script_name):
        return self._config.script_path(script_name)

    def script_shell_command(self, script_name):
        return self._config.script_shell_command(script_name)

    def path_to_test_expectations_file(self):
        """Update the test expectations to the passed-in string.

        This is used by the rebaselining tool. Raises NotImplementedError
        if the port does not use expectations files."""
        raise NotImplementedError('Port.path_to_test_expectations_file')

    def relative_test_filename(self, filename):
        """Relative unix-style path for a filename under the LayoutTests
        directory. Filenames outside the LayoutTests directory should raise
        an error."""
        # FIXME: On Windows, does this return test_names with forward slashes,
        # or windows-style relative paths?
        assert filename.startswith(self.layout_tests_dir()), "%s did not start with %s" % (filename, self.layout_tests_dir())
        return filename[len(self.layout_tests_dir()) + 1:]

    def abspath_for_test(self, test_name):
        """Returns the full path to the file for a given test name. This is the
        inverse of relative_test_filename()."""
        return self._filesystem.normpath(self._filesystem.join(self.layout_tests_dir(), test_name))

    def results_directory(self):
        """Absolute path to the place to store the test results (uses --results-directory)."""
        if not self._results_directory:
            option_val = self.get_option('results_directory') or self.default_results_directory()
            self._results_directory = self._filesystem.abspath(option_val)
        return self._results_directory

    def default_results_directory(self):
        """Absolute path to the default place to store the test results."""
        raise NotImplementedError()

    def setup_test_run(self):
        """Perform port-specific work at the beginning of a test run."""
        pass

    def setup_environ_for_server(self):
        """Perform port-specific work at the beginning of a server launch.

        Returns:
           Operating-system's environment.
        """
        return os.environ.copy()

    def show_results_html_file(self, results_filename):
        """This routine should display the HTML file pointed at by
        results_filename in a users' browser."""
        return self._user.open_url(results_filename)

    def create_driver(self, worker_number):
        """Return a newly created base.Driver subclass for starting/stopping
        the test driver."""
        raise NotImplementedError('Port.create_driver')

    def start_helper(self):
        """If a port needs to reconfigure graphics settings or do other
        things to ensure a known test configuration, it should override this
        method."""
        pass

    def start_http_server(self):
        """Start a web server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
        if self.get_option('use_apache'):
            self._http_server = apache_http_server.LayoutTestApacheHttpd(self,
                self.results_directory())
        else:
            self._http_server = http_server.Lighttpd(self, self.results_directory())
        self._http_server.start()

    def start_websocket_server(self):
        """Start a websocket server if it is available. Do nothing if
        it isn't. This routine is allowed to (and may) fail if a server
        is already running."""
        self._websocket_server = websocket_server.PyWebSocket(self, self.results_directory())
        self._websocket_server.start()

    def acquire_http_lock(self):
        self._http_lock = http_lock.HttpLock(None)
        self._http_lock.wait_for_httpd_lock()

    def stop_helper(self):
        """Shut down the test helper if it is running. Do nothing if
        it isn't, or it isn't available. If a port overrides start_helper()
        it must override this routine as well."""
        pass

    def stop_http_server(self):
        """Shut down the http server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._http_server:
            self._http_server.stop()

    def stop_websocket_server(self):
        """Shut down the websocket server if it is running. Do nothing if
        it isn't, or it isn't available."""
        if self._websocket_server:
            self._websocket_server.stop()

    def release_http_lock(self):
        if self._http_lock:
            self._http_lock.cleanup_http_lock()

    #
    # TEST EXPECTATION-RELATED METHODS
    #

    def test_configuration(self):
        """Returns the current TestConfiguration for the port."""
        if not self._test_configuration:
            self._test_configuration = TestConfiguration(self)
        return self._test_configuration

    def all_test_configurations(self):
        return self.test_configuration().all_test_configurations()

    def all_baseline_variants(self):
        """Returns a list of platform names sufficient to cover all the baselines.

        The list should be sorted so that a later platform  will reuse
        an earlier platform's baselines if they are the same (e.g.,
        'snowleopard' should precede 'leopard')."""
        raise NotImplementedError

    def test_expectations(self):
        """Returns the test expectations for this port.

        Basically this string should contain the equivalent of a
        test_expectations file. See test_expectations.py for more details."""
        return self._filesystem.read_text_file(self.path_to_test_expectations_file())

    def test_expectations_overrides(self):
        """Returns an optional set of overrides for the test_expectations.

        This is used by ports that have code in two repositories, and where
        it is possible that you might need "downstream" expectations that
        temporarily override the "upstream" expectations until the port can
        sync up the two repos."""
        return None

    def test_repository_paths(self):
        """Returns a list of (repository_name, repository_path) tuples
        of its depending code base.  By default it returns a list that only
        contains a ('webkit', <webkitRepossitoryPath>) tuple.
        """
        return [('webkit', self.layout_tests_dir())]


    _WDIFF_DEL = '##WDIFF_DEL##'
    _WDIFF_ADD = '##WDIFF_ADD##'
    _WDIFF_END = '##WDIFF_END##'

    def _format_wdiff_output_as_html(self, wdiff):
        wdiff = cgi.escape(wdiff)
        wdiff = wdiff.replace(self._WDIFF_DEL, "<span class=del>")
        wdiff = wdiff.replace(self._WDIFF_ADD, "<span class=add>")
        wdiff = wdiff.replace(self._WDIFF_END, "</span>")
        html = "<head><style>.del { background: #faa; } "
        html += ".add { background: #afa; }</style></head>"
        html += "<pre>%s</pre>" % wdiff
        return html

    def _wdiff_command(self, actual_filename, expected_filename):
        executable = self._path_to_wdiff()
        return [executable,
                "--start-delete=%s" % self._WDIFF_DEL,
                "--end-delete=%s" % self._WDIFF_END,
                "--start-insert=%s" % self._WDIFF_ADD,
                "--end-insert=%s" % self._WDIFF_END,
                actual_filename,
                expected_filename]

    @staticmethod
    def _handle_wdiff_error(script_error):
        # Exit 1 means the files differed, any other exit code is an error.
        if script_error.exit_code != 1:
            raise script_error

    def _run_wdiff(self, actual_filename, expected_filename):
        """Runs wdiff and may throw exceptions.
        This is mostly a hook for unit testing."""
        # Diffs are treated as binary as they may include multiple files
        # with conflicting encodings.  Thus we do not decode the output.
        command = self._wdiff_command(actual_filename, expected_filename)
        wdiff = self._executive.run_command(command, decode_output=False,
            error_handler=self._handle_wdiff_error)
        return self._format_wdiff_output_as_html(wdiff)

    def wdiff_text(self, actual_filename, expected_filename):
        """Returns a string of HTML indicating the word-level diff of the
        contents of the two filenames. Returns an empty string if word-level
        diffing isn't available."""
        if not self._wdiff_available:
            return ""
        try:
            # It's possible to raise a ScriptError we pass wdiff invalid paths.
            return self._run_wdiff(actual_filename, expected_filename)
        except OSError, e:
            if e.errno in [errno.ENOENT, errno.EACCES, errno.ECHILD]:
                # Silently ignore cases where wdiff is missing.
                self._wdiff_available = False
                return ""
            raise

    # This is a class variable so we can test error output easily.
    _pretty_patch_error_html = "Failed to run PrettyPatch, see error log."

    def pretty_patch_text(self, diff_path):
        if self._pretty_patch_available is None:
            self._pretty_patch_available = self.check_pretty_patch(logging=False)
        if not self._pretty_patch_available:
            return self._pretty_patch_error_html
        command = ("ruby", "-I", self._filesystem.dirname(self._pretty_patch_path),
                   self._pretty_patch_path, diff_path)
        try:
            # Diffs are treated as binary (we pass decode_output=False) as they
            # may contain multiple files of conflicting encodings.
            return self._executive.run_command(command, decode_output=False)
        except OSError, e:
            # If the system is missing ruby log the error and stop trying.
            self._pretty_patch_available = False
            _log.error("Failed to run PrettyPatch (%s): %s" % (command, e))
            return self._pretty_patch_error_html
        except ScriptError, e:
            # If ruby failed to run for some reason, log the command
            # output and stop trying.
            self._pretty_patch_available = False
            _log.error("Failed to run PrettyPatch (%s):\n%s" % (command,
                       e.message_with_output()))
            return self._pretty_patch_error_html

    def default_configuration(self):
        return self._config.default_configuration()

    #
    # PROTECTED ROUTINES
    #
    # The routines below should only be called by routines in this class
    # or any of its subclasses.
    #
    def _webkit_build_directory(self, args):
        return self._config.build_directory(args[0])

    def _path_to_apache(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache')

    def _path_to_apache_config_file(self):
        """Returns the full path to the apache binary.

        This is needed only by ports that use the apache_http_server module."""
        raise NotImplementedError('Port.path_to_apache_config_file')

    def _path_to_driver(self, configuration=None):
        """Returns the full path to the test driver (DumpRenderTree)."""
        raise NotImplementedError('Port._path_to_driver')

    def _path_to_webcore_library(self):
        """Returns the full path to a built copy of WebCore."""
        raise NotImplementedError('Port.path_to_webcore_library')

    def _path_to_helper(self):
        """Returns the full path to the layout_test_helper binary, which
        is used to help configure the system for the test run, or None
        if no helper is needed.

        This is likely only used by start/stop_helper()."""
        raise NotImplementedError('Port._path_to_helper')

    def _path_to_image_diff(self):
        """Returns the full path to the image_diff binary, or None if it
        is not available.

        This is likely used only by diff_image()"""
        raise NotImplementedError('Port.path_to_image_diff')

    def _path_to_lighttpd(self):
        """Returns the path to the LigHTTPd binary.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd')

    def _path_to_lighttpd_modules(self):
        """Returns the path to the LigHTTPd modules directory.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_modules')

    def _path_to_lighttpd_php(self):
        """Returns the path to the LigHTTPd PHP executable.

        This is needed only by ports that use the http_server.py module."""
        raise NotImplementedError('Port._path_to_lighttpd_php')

    def _path_to_wdiff(self):
        """Returns the full path to the wdiff binary, or None if it is
        not available.

        This is likely used only by wdiff_text()"""
        raise NotImplementedError('Port._path_to_wdiff')

    def _shut_down_http_server(self, pid):
        """Forcefully and synchronously kills the web server.

        This routine should only be called from http_server.py or its
        subclasses."""
        raise NotImplementedError('Port._shut_down_http_server')

    def _webkit_baseline_path(self, platform):
        """Return the  full path to the top of the baseline tree for a
        given platform."""
        return self._filesystem.join(self.layout_tests_dir(), 'platform',
                                     platform)


class DriverInput(object):
    """Holds the input parameters for a driver."""

    def __init__(self, filename, timeout, image_hash):
        """Initializes a DriverInput object.

        Args:
          filename: Full path to the test.
          timeout: Timeout in msecs the driver should use while running the test
          image_hash: A image checksum which is used to avoid doing an image dump if
                     the checksums match.
        """
        self.filename = filename
        self.timeout = timeout
        self.image_hash = image_hash


class DriverOutput(object):
    """Groups information about a output from driver for easy passing of data."""

    def __init__(self, text, image, image_hash, audio,
                 crash=False, test_time=0, timeout=False, error=''):
        """Initializes a TestOutput object.

        Args:
          text: a text output
          image: an image output
          image_hash: a string containing the checksum of the image
          audio: contents of an audio stream, if any (in WAV format)
          crash: a boolean indicating whether the driver crashed on the test
          test_time: the time the test took to execute
          timeout: a boolean indicating whehter the test timed out
          error: any unexpected or additional (or error) text output
        """
        self.text = text
        self.image = image
        self.image_hash = image_hash
        self.audio = audio
        self.crash = crash
        self.test_time = test_time
        self.timeout = timeout
        self.error = error


class Driver:
    """Abstract interface for the DumpRenderTree interface."""

    def __init__(self, port, worker_number):
        """Initialize a Driver to subsequently run tests.

        Typically this routine will spawn DumpRenderTree in a config
        ready for subsequent input.

        port - reference back to the port object.
        worker_number - identifier for a particular worker/driver instance
        """
        raise NotImplementedError('Driver.__init__')

    def run_test(self, driver_input):
        """Run a single test and return the results.

        Note that it is okay if a test times out or crashes and leaves
        the driver in an indeterminate state. The upper layers of the program
        are responsible for cleaning up and ensuring things are okay.

        Args:
          driver_input: a DriverInput object

        Returns a DriverOutput object.
          Note that DriverOutput.image will be '' (empty string) if a test crashes.
        """
        raise NotImplementedError('Driver.run_test')

    # FIXME: This is static so we can test it w/o creating a Base instance.
    @classmethod
    def _command_wrapper(cls, wrapper_option):
        # Hook for injecting valgrind or other runtime instrumentation,
        # used by e.g. tools/valgrind/valgrind_tests.py.
        wrapper = []
        browser_wrapper = os.environ.get("BROWSER_WRAPPER", None)
        if browser_wrapper:
            # FIXME: There seems to be no reason to use BROWSER_WRAPPER over --wrapper.
            # Remove this code any time after the date listed below.
            _log.error("BROWSER_WRAPPER is deprecated, please use --wrapper instead.")
            _log.error("BROWSER_WRAPPER will be removed any time after June 1st 2010 and your scripts will break.")
            wrapper += [browser_wrapper]

        if wrapper_option:
            wrapper += shlex.split(wrapper_option)
        return wrapper

    def poll(self):
        """Returns None if the Driver is still running. Returns the returncode
        if it has exited."""
        raise NotImplementedError('Driver.poll')

    def stop(self):
        raise NotImplementedError('Driver.stop')


class TestConfiguration(object):
    def __init__(self, port=None, os=None, version=None, architecture=None,
                 build_type=None, graphics_type=None):
        self.os = os or port.operating_system()
        self.version = version or port.version()
        self.architecture = architecture or port.architecture()
        self.build_type = build_type or port._options.configuration.lower()
        self.graphics_type = graphics_type or port.graphics_type()

    def items(self):
        return self.__dict__.items()

    def keys(self):
        return self.__dict__.keys()

    def __str__(self):
        return ("<%(os)s, %(version)s, %(architecture)s, %(build_type)s, %(graphics_type)s>" %
                self.__dict__)

    def __repr__(self):
        return "TestConfig(os='%(os)s', version='%(version)s', architecture='%(architecture)s', build_type='%(build_type)s', graphics_type='%(graphics_type)s')" % self.__dict__

    def values(self):
        """Returns the configuration values of this instance as a tuple."""
        return self.__dict__.values()

    def all_test_configurations(self):
        """Returns a sequence of the TestConfigurations the port supports."""
        # By default, we assume we want to test every graphics type in
        # every configuration on every system.
        test_configurations = []
        for system in self.all_systems():
            for build_type in self.all_build_types():
                for graphics_type in self.all_graphics_types():
                    test_configurations.append(TestConfiguration(
                        os=system[0],
                        version=system[1],
                        architecture=system[2],
                        build_type=build_type,
                        graphics_type=graphics_type))
        return test_configurations

    def all_systems(self):
        return (('mac', 'leopard', 'x86'),
                ('mac', 'snowleopard', 'x86'),
                ('win', 'xp', 'x86'),
                ('win', 'vista', 'x86'),
                ('win', 'win7', 'x86'),
                ('linux', 'hardy', 'x86'),
                ('linux', 'hardy', 'x86_64'))

    def all_build_types(self):
        return ('debug', 'release')

    def all_graphics_types(self):
        return ('cpu', 'gpu')