xref: /external/webkit/Tools/Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py

#!/usr/bin/python
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Unit tests for printing.py."""

import optparse
import unittest
import logging

from webkitpy.common import array_stream
from webkitpy.common.system import logtesting
from webkitpy.layout_tests import port

from webkitpy.layout_tests.layout_package import printing
from webkitpy.layout_tests.layout_package import result_summary
from webkitpy.layout_tests.layout_package import test_expectations
from webkitpy.layout_tests.layout_package import test_failures
from webkitpy.layout_tests.layout_package import test_results
from webkitpy.layout_tests.layout_package import test_runner


def get_options(args):
    print_options = printing.print_options()
    option_parser = optparse.OptionParser(option_list=print_options)
    return option_parser.parse_args(args)


class TestUtilityFunctions(unittest.TestCase):
    def test_configure_logging(self):
        options, args = get_options([])
        stream = array_stream.ArrayStream()
        handler = printing._configure_logging(stream, options.verbose)
        logging.info("this should be logged")
        self.assertFalse(stream.empty())

        stream.reset()
        logging.debug("this should not be logged")
        self.assertTrue(stream.empty())

        printing._restore_logging(handler)

        stream.reset()
        options, args = get_options(['--verbose'])
        handler = printing._configure_logging(stream, options.verbose)
        logging.debug("this should be logged")
        self.assertFalse(stream.empty())
        printing._restore_logging(handler)

    def test_print_options(self):
        options, args = get_options([])
        self.assertTrue(options is not None)

    def test_parse_print_options(self):
        def test_switches(args, expected_switches_str,
                          verbose=False, child_processes=1,
                          is_fully_parallel=False):
            options, args = get_options(args)
            if expected_switches_str:
                expected_switches = set(expected_switches_str.split(','))
            else:
                expected_switches = set()
            switches = printing.parse_print_options(options.print_options,
                                                    verbose,
                                                    child_processes,
                                                    is_fully_parallel)
            self.assertEqual(expected_switches, switches)

        # test that we default to the default set of switches
        test_switches([], printing.PRINT_DEFAULT)

        # test that verbose defaults to everything
        test_switches([], printing.PRINT_EVERYTHING, verbose=True)

        # test that --print default does what it's supposed to
        test_switches(['--print', 'default'], printing.PRINT_DEFAULT)

        # test that --print nothing does what it's supposed to
        test_switches(['--print', 'nothing'], None)

        # test that --print everything does what it's supposed to
        test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING)

        # this tests that '--print X' overrides '--verbose'
        test_switches(['--print', 'actual'], 'actual', verbose=True)



class  Testprinter(unittest.TestCase):
    def get_printer(self, args=None, single_threaded=False,
                   is_fully_parallel=False):
        args = args or []
        printing_options = printing.print_options()
        option_parser = optparse.OptionParser(option_list=printing_options)
        options, args = option_parser.parse_args(args)
        self._port = port.get('test', options)
        nproc = 2
        if single_threaded:
            nproc = 1

        regular_output = array_stream.ArrayStream()
        buildbot_output = array_stream.ArrayStream()
        printer = printing.Printer(self._port, options, regular_output,
                                   buildbot_output, single_threaded,
                                   is_fully_parallel)
        return printer, regular_output, buildbot_output

    def get_result(self, test, result_type=test_expectations.PASS, run_time=0):
        failures = []
        if result_type == test_expectations.TIMEOUT:
            failures = [test_failures.FailureTimeout()]
        elif result_type == test_expectations.CRASH:
            failures = [test_failures.FailureCrash()]
        path = self._port._filesystem.join(self._port.layout_tests_dir(), test)
        return test_results.TestResult(path, failures=failures, test_run_time=run_time)

    def get_result_summary(self, tests, expectations_str):
        test_paths = [self._port._filesystem.join(self._port.layout_tests_dir(), test) for
                      test in tests]
        expectations = test_expectations.TestExpectations(
            self._port, test_paths, expectations_str,
            self._port.test_configuration(),
            is_lint_mode=False)

        rs = result_summary.ResultSummary(expectations, test_paths)
        return test_paths, rs, expectations

    def test_help_printer(self):
        # Here and below we'll call the "regular" printer err and the
        # buildbot printer out; this corresponds to how things run on the
        # bots with stderr and stdout.
        printer, err, out = self.get_printer()

        # This routine should print something to stdout. testing what it is
        # is kind of pointless.
        printer.help_printing()
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def do_switch_tests(self, method_name, switch, to_buildbot,
                        message='hello', exp_err=None, exp_bot=None):
        def do_helper(method_name, switch, message, exp_err, exp_bot):
            printer, err, bot = self.get_printer(['--print', switch])
            getattr(printer, method_name)(message)
            self.assertEqual(err.get(), exp_err)
            self.assertEqual(bot.get(), exp_bot)

        if to_buildbot:
            if exp_err is None:
                exp_err = []
            if exp_bot is None:
                exp_bot = [message + "\n"]
        else:
            if exp_err is None:
                exp_err = [message + "\n"]
            if exp_bot is None:
                exp_bot = []
        do_helper(method_name, 'nothing', 'hello', [], [])
        do_helper(method_name, switch, 'hello', exp_err, exp_bot)
        do_helper(method_name, 'everything', 'hello', exp_err, exp_bot)

    def test_configure_and_cleanup(self):
        # This test verifies that calling cleanup repeatedly and deleting
        # the object is safe.
        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.cleanup()
        printer.cleanup()
        printer = None

    def test_print_actual(self):
        # Actual results need to be logged to the buildbot's stream.
        self.do_switch_tests('print_actual', 'actual', to_buildbot=True)

    def test_print_actual_buildbot(self):
        # FIXME: Test that the format of the actual results matches what the
        # buildbot is expecting.
        pass

    def test_print_config(self):
        self.do_switch_tests('print_config', 'config', to_buildbot=False)

    def test_print_expected(self):
        self.do_switch_tests('print_expected', 'expected', to_buildbot=False)

    def test_print_timing(self):
        self.do_switch_tests('print_timing', 'timing', to_buildbot=False)

    def test_print_update(self):
        # Note that there shouldn't be a carriage return here; updates()
        # are meant to be overwritten.
        self.do_switch_tests('print_update', 'updates', to_buildbot=False,
                             message='hello', exp_err=['hello'])

    def test_print_one_line_summary(self):
        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print', 'one-line-summary'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])

        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.print_one_line_summary(1, 1, 0)
        self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"])

        err.reset()
        printer.print_one_line_summary(2, 1, 1)
        self.assertEquals(err.get(),
                          ["1 test ran as expected, 1 didn't:\n", "\n"])

        err.reset()
        printer.print_one_line_summary(3, 2, 1)
        self.assertEquals(err.get(),
                          ["2 tests ran as expected, 1 didn't:\n", "\n"])

        err.reset()
        printer.print_one_line_summary(3, 2, 0)
        self.assertEquals(err.get(),
                          ['\n', "2 tests ran as expected (1 didn't run).\n",
                           '\n'])


    def test_print_test_result(self):
        # Note here that we don't use meaningful exp_str and got_str values;
        # the actual contents of the string are treated opaquely by
        # print_test_result() when tracing, and usually we don't want
        # to test what exactly is printed, just that something
        # was printed (or that nothing was printed).
        #
        # FIXME: this is actually some goofy layering; it would be nice
        # we could refactor it so that the args weren't redundant. Maybe
        # the TestResult should contain what was expected, and the
        # strings could be derived from the TestResult?
        printer, err, out = self.get_printer(['--print', 'nothing'])
        result = self.get_result('passes/image.html')
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print', 'unexpected'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertEquals(err.get(),
                          ['  passes/image.html -> unexpected pass\n'])

        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertEquals(err.get(),
                          ['  passes/image.html -> unexpected pass\n'])

        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertTrue(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        printer, err, out = self.get_printer(['--print',
                                              'trace-unexpected'])
        result = self.get_result("passes/text.html")
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        err.reset()
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        printer, err, out = self.get_printer(['--print', 'trace-everything'])
        result = self.get_result('passes/image.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_text.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_check.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        result = self.get_result('failures/expected/missing_image.html')
        printer.print_test_result(result, expected=True, exp_str='',
                                  got_str='')
        self.assertFalse(err.empty())

        err.reset()
        printer.print_test_result(result, expected=False, exp_str='',
                                  got_str='')

    def test_print_progress(self):
        expectations = ''

        # test that we print nothing
        printer, err, out = self.get_printer(['--print', 'nothing'])
        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        paths, rs, exp = self.get_result_summary(tests, expectations)

        printer.print_progress(rs, False, paths)
        self.assertTrue(out.empty())
        self.assertTrue(err.empty())

        printer.print_progress(rs, True, paths)
        self.assertTrue(out.empty())
        self.assertTrue(err.empty())

        # test regular functionality
        printer, err, out = self.get_printer(['--print',
                                              'one-line-progress'])
        printer.print_progress(rs, False, paths)
        self.assertTrue(out.empty())
        self.assertFalse(err.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def test_print_progress__detailed(self):
        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        expectations = 'BUGX : failures/expected/timeout.html = TIMEOUT'

        # first, test that it is disabled properly
        # should still print one-line-progress
        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress'], single_threaded=False)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # now test the enabled paths
        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress'], single_threaded=True)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
        rs.add(self.get_result('failures/expected/timeout.html'), True)
        rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
        err.reset()
        out.reset()
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # We only clear the meter when retrying w/ detailed-progress.
        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        printer, err, out = self.get_printer(
            ['--print', 'detailed-progress,unexpected'], single_threaded=True)
        paths, rs, exp = self.get_result_summary(tests, expectations)
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False)
        rs.add(self.get_result('failures/expected/timeout.html'), True)
        rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True)
        err.reset()
        out.reset()
        printer.print_progress(rs, False, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

        # We only clear the meter when retrying w/ detailed-progress.
        err.reset()
        out.reset()
        printer.print_progress(rs, True, paths)
        self.assertFalse(err.empty())
        self.assertTrue(out.empty())

    def test_write_nothing(self):
        printer, err, out = self.get_printer(['--print', 'nothing'])
        printer.write("foo")
        self.assertTrue(err.empty())

    def test_write_misc(self):
        printer, err, out = self.get_printer(['--print', 'misc'])
        printer.write("foo")
        self.assertFalse(err.empty())
        err.reset()
        printer.write("foo", "config")
        self.assertTrue(err.empty())

    def test_write_everything(self):
        printer, err, out = self.get_printer(['--print', 'everything'])
        printer.write("foo")
        self.assertFalse(err.empty())
        err.reset()
        printer.write("foo", "config")
        self.assertFalse(err.empty())

    def test_write_verbose(self):
        printer, err, out = self.get_printer(['--verbose'])
        printer.write("foo")
        self.assertTrue(not err.empty() and "foo" in err.get()[0])
        self.assertTrue(out.empty())

    def test_print_unexpected_results(self):
        # This routine is the only one that prints stuff that the bots
        # care about.
        #
        # FIXME: there's some weird layering going on here. It seems
        # like we shouldn't be both using an expectations string and
        # having to specify whether or not the result was expected.
        # This whole set of tests should probably be rewritten.
        #
        # FIXME: Plus, the fact that we're having to call into
        # run_webkit_tests is clearly a layering inversion.
        def get_unexpected_results(expected, passing, flaky):
            """Return an unexpected results summary matching the input description.

            There are a lot of different combinations of test results that
            can be tested; this routine produces various combinations based
            on the values of the input flags.

            Args
                expected: whether the tests ran as expected
                passing: whether the tests should all pass
                flaky: whether the tests should be flaky (if False, they
                    produce the same results on both runs; if True, they
                    all pass on the second run).

            """
            paths, rs, exp = self.get_result_summary(tests, expectations)
            if expected:
                rs.add(self.get_result('passes/text.html', test_expectations.PASS),
                       expected)
                rs.add(self.get_result('failures/expected/timeout.html',
                       test_expectations.TIMEOUT), expected)
                rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH),
                   expected)
            elif passing:
                rs.add(self.get_result('passes/text.html'), expected)
                rs.add(self.get_result('failures/expected/timeout.html'), expected)
                rs.add(self.get_result('failures/expected/crash.html'), expected)
            else:
                rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT),
                       expected)
                rs.add(self.get_result('failures/expected/timeout.html',
                       test_expectations.CRASH), expected)
                rs.add(self.get_result('failures/expected/crash.html',
                                  test_expectations.TIMEOUT),
                   expected)
            retry = rs
            if flaky:
                paths, retry, exp = self.get_result_summary(tests,
                                                expectations)
                retry.add(self.get_result('passes/text.html'), True)
                retry.add(self.get_result('failures/expected/timeout.html'), True)
                retry.add(self.get_result('failures/expected/crash.html'), True)
            unexpected_results = test_runner.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True)
            return unexpected_results

        tests = ['passes/text.html', 'failures/expected/timeout.html',
                 'failures/expected/crash.html']
        expectations = ''

        printer, err, out = self.get_printer(['--print', 'nothing'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertTrue(out.empty())

        printer, err, out = self.get_printer(['--print',
                                              'unexpected-results'])

        # test everything running as expected
        ur = get_unexpected_results(expected=True, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertTrue(out.empty())

        # test failures
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # test unexpected flaky results
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=True, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # test unexpected passes
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=True)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        err.reset()
        out.reset()
        printer, err, out = self.get_printer(['--print', 'everything'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        expectations = """
BUGX : failures/expected/crash.html = CRASH
BUGX : failures/expected/timeout.html = TIMEOUT
"""
        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        err.reset()
        out.reset()
        ur = get_unexpected_results(expected=False, passing=True, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

        # Test handling of --verbose as well.
        err.reset()
        out.reset()
        printer, err, out = self.get_printer(['--verbose'])
        ur = get_unexpected_results(expected=False, passing=False, flaky=False)
        printer.print_unexpected_results(ur)
        self.assertTrue(err.empty())
        self.assertFalse(out.empty())

    def test_print_unexpected_results_buildbot(self):
        # FIXME: Test that print_unexpected_results() produces the printer the
        # buildbot is expecting.
        pass

if __name__ == '__main__':
    unittest.main()