1#!/usr/bin/python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""Unit tests for printing.py.""" 31 32import optparse 33import unittest 34import logging 35 36from webkitpy.common import array_stream 37from webkitpy.common.system import logtesting 38from webkitpy.layout_tests import port 39 40from webkitpy.layout_tests.layout_package import printing 41from webkitpy.layout_tests.layout_package import result_summary 42from webkitpy.layout_tests.layout_package import test_expectations 43from webkitpy.layout_tests.layout_package import test_failures 44from webkitpy.layout_tests.layout_package import test_results 45from webkitpy.layout_tests.layout_package import test_runner 46 47 48def get_options(args): 49 print_options = printing.print_options() 50 option_parser = optparse.OptionParser(option_list=print_options) 51 return option_parser.parse_args(args) 52 53 54class TestUtilityFunctions(unittest.TestCase): 55 def test_configure_logging(self): 56 options, args = get_options([]) 57 stream = array_stream.ArrayStream() 58 handler = printing._configure_logging(stream, options.verbose) 59 logging.info("this should be logged") 60 self.assertFalse(stream.empty()) 61 62 stream.reset() 63 logging.debug("this should not be logged") 64 self.assertTrue(stream.empty()) 65 66 printing._restore_logging(handler) 67 68 stream.reset() 69 options, args = get_options(['--verbose']) 70 handler = printing._configure_logging(stream, options.verbose) 71 logging.debug("this should be logged") 72 self.assertFalse(stream.empty()) 73 printing._restore_logging(handler) 74 75 def test_print_options(self): 76 options, args = get_options([]) 77 self.assertTrue(options is not None) 78 79 def test_parse_print_options(self): 80 def test_switches(args, expected_switches_str, 81 verbose=False, child_processes=1, 82 is_fully_parallel=False): 83 options, args = get_options(args) 84 if expected_switches_str: 85 expected_switches = set(expected_switches_str.split(',')) 86 else: 87 expected_switches = set() 88 switches = printing.parse_print_options(options.print_options, 89 verbose, 90 child_processes, 91 is_fully_parallel) 92 self.assertEqual(expected_switches, switches) 93 94 # test that we default to the default set of switches 95 test_switches([], printing.PRINT_DEFAULT) 96 97 # test that verbose defaults to everything 98 test_switches([], printing.PRINT_EVERYTHING, verbose=True) 99 100 # test that --print default does what it's supposed to 101 test_switches(['--print', 'default'], printing.PRINT_DEFAULT) 102 103 # test that --print nothing does what it's supposed to 104 test_switches(['--print', 'nothing'], None) 105 106 # test that --print everything does what it's supposed to 107 test_switches(['--print', 'everything'], printing.PRINT_EVERYTHING) 108 109 # this tests that '--print X' overrides '--verbose' 110 test_switches(['--print', 'actual'], 'actual', verbose=True) 111 112 113 114class Testprinter(unittest.TestCase): 115 def get_printer(self, args=None, single_threaded=False, 116 is_fully_parallel=False): 117 args = args or [] 118 printing_options = printing.print_options() 119 option_parser = optparse.OptionParser(option_list=printing_options) 120 options, args = option_parser.parse_args(args) 121 self._port = port.get('test', options) 122 nproc = 2 123 if single_threaded: 124 nproc = 1 125 126 regular_output = array_stream.ArrayStream() 127 buildbot_output = array_stream.ArrayStream() 128 printer = printing.Printer(self._port, options, regular_output, 129 buildbot_output, single_threaded, 130 is_fully_parallel) 131 return printer, regular_output, buildbot_output 132 133 def get_result(self, test, result_type=test_expectations.PASS, run_time=0): 134 failures = [] 135 if result_type == test_expectations.TIMEOUT: 136 failures = [test_failures.FailureTimeout()] 137 elif result_type == test_expectations.CRASH: 138 failures = [test_failures.FailureCrash()] 139 path = self._port._filesystem.join(self._port.layout_tests_dir(), test) 140 return test_results.TestResult(path, failures=failures, test_run_time=run_time) 141 142 def get_result_summary(self, tests, expectations_str): 143 test_paths = [self._port._filesystem.join(self._port.layout_tests_dir(), test) for 144 test in tests] 145 expectations = test_expectations.TestExpectations( 146 self._port, test_paths, expectations_str, 147 self._port.test_configuration(), 148 is_lint_mode=False) 149 150 rs = result_summary.ResultSummary(expectations, test_paths) 151 return test_paths, rs, expectations 152 153 def test_help_printer(self): 154 # Here and below we'll call the "regular" printer err and the 155 # buildbot printer out; this corresponds to how things run on the 156 # bots with stderr and stdout. 157 printer, err, out = self.get_printer() 158 159 # This routine should print something to stdout. testing what it is 160 # is kind of pointless. 161 printer.help_printing() 162 self.assertFalse(err.empty()) 163 self.assertTrue(out.empty()) 164 165 def do_switch_tests(self, method_name, switch, to_buildbot, 166 message='hello', exp_err=None, exp_bot=None): 167 def do_helper(method_name, switch, message, exp_err, exp_bot): 168 printer, err, bot = self.get_printer(['--print', switch]) 169 getattr(printer, method_name)(message) 170 self.assertEqual(err.get(), exp_err) 171 self.assertEqual(bot.get(), exp_bot) 172 173 if to_buildbot: 174 if exp_err is None: 175 exp_err = [] 176 if exp_bot is None: 177 exp_bot = [message + "\n"] 178 else: 179 if exp_err is None: 180 exp_err = [message + "\n"] 181 if exp_bot is None: 182 exp_bot = [] 183 do_helper(method_name, 'nothing', 'hello', [], []) 184 do_helper(method_name, switch, 'hello', exp_err, exp_bot) 185 do_helper(method_name, 'everything', 'hello', exp_err, exp_bot) 186 187 def test_configure_and_cleanup(self): 188 # This test verifies that calling cleanup repeatedly and deleting 189 # the object is safe. 190 printer, err, out = self.get_printer(['--print', 'everything']) 191 printer.cleanup() 192 printer.cleanup() 193 printer = None 194 195 def test_print_actual(self): 196 # Actual results need to be logged to the buildbot's stream. 197 self.do_switch_tests('print_actual', 'actual', to_buildbot=True) 198 199 def test_print_actual_buildbot(self): 200 # FIXME: Test that the format of the actual results matches what the 201 # buildbot is expecting. 202 pass 203 204 def test_print_config(self): 205 self.do_switch_tests('print_config', 'config', to_buildbot=False) 206 207 def test_print_expected(self): 208 self.do_switch_tests('print_expected', 'expected', to_buildbot=False) 209 210 def test_print_timing(self): 211 self.do_switch_tests('print_timing', 'timing', to_buildbot=False) 212 213 def test_print_update(self): 214 # Note that there shouldn't be a carriage return here; updates() 215 # are meant to be overwritten. 216 self.do_switch_tests('print_update', 'updates', to_buildbot=False, 217 message='hello', exp_err=['hello']) 218 219 def test_print_one_line_summary(self): 220 printer, err, out = self.get_printer(['--print', 'nothing']) 221 printer.print_one_line_summary(1, 1, 0) 222 self.assertTrue(err.empty()) 223 224 printer, err, out = self.get_printer(['--print', 'one-line-summary']) 225 printer.print_one_line_summary(1, 1, 0) 226 self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) 227 228 printer, err, out = self.get_printer(['--print', 'everything']) 229 printer.print_one_line_summary(1, 1, 0) 230 self.assertEquals(err.get(), ["All 1 tests ran as expected.\n", "\n"]) 231 232 err.reset() 233 printer.print_one_line_summary(2, 1, 1) 234 self.assertEquals(err.get(), 235 ["1 test ran as expected, 1 didn't:\n", "\n"]) 236 237 err.reset() 238 printer.print_one_line_summary(3, 2, 1) 239 self.assertEquals(err.get(), 240 ["2 tests ran as expected, 1 didn't:\n", "\n"]) 241 242 err.reset() 243 printer.print_one_line_summary(3, 2, 0) 244 self.assertEquals(err.get(), 245 ['\n', "2 tests ran as expected (1 didn't run).\n", 246 '\n']) 247 248 249 def test_print_test_result(self): 250 # Note here that we don't use meaningful exp_str and got_str values; 251 # the actual contents of the string are treated opaquely by 252 # print_test_result() when tracing, and usually we don't want 253 # to test what exactly is printed, just that something 254 # was printed (or that nothing was printed). 255 # 256 # FIXME: this is actually some goofy layering; it would be nice 257 # we could refactor it so that the args weren't redundant. Maybe 258 # the TestResult should contain what was expected, and the 259 # strings could be derived from the TestResult? 260 printer, err, out = self.get_printer(['--print', 'nothing']) 261 result = self.get_result('passes/image.html') 262 printer.print_test_result(result, expected=False, exp_str='', 263 got_str='') 264 self.assertTrue(err.empty()) 265 266 printer, err, out = self.get_printer(['--print', 'unexpected']) 267 printer.print_test_result(result, expected=True, exp_str='', 268 got_str='') 269 self.assertTrue(err.empty()) 270 printer.print_test_result(result, expected=False, exp_str='', 271 got_str='') 272 self.assertEquals(err.get(), 273 [' passes/image.html -> unexpected pass\n']) 274 275 printer, err, out = self.get_printer(['--print', 'everything']) 276 printer.print_test_result(result, expected=True, exp_str='', 277 got_str='') 278 self.assertTrue(err.empty()) 279 280 printer.print_test_result(result, expected=False, exp_str='', 281 got_str='') 282 self.assertEquals(err.get(), 283 [' passes/image.html -> unexpected pass\n']) 284 285 printer, err, out = self.get_printer(['--print', 'nothing']) 286 printer.print_test_result(result, expected=False, exp_str='', 287 got_str='') 288 self.assertTrue(err.empty()) 289 290 printer, err, out = self.get_printer(['--print', 291 'trace-unexpected']) 292 printer.print_test_result(result, expected=True, exp_str='', 293 got_str='') 294 self.assertTrue(err.empty()) 295 296 printer, err, out = self.get_printer(['--print', 297 'trace-unexpected']) 298 printer.print_test_result(result, expected=False, exp_str='', 299 got_str='') 300 self.assertFalse(err.empty()) 301 302 printer, err, out = self.get_printer(['--print', 303 'trace-unexpected']) 304 result = self.get_result("passes/text.html") 305 printer.print_test_result(result, expected=False, exp_str='', 306 got_str='') 307 self.assertFalse(err.empty()) 308 309 err.reset() 310 printer.print_test_result(result, expected=False, exp_str='', 311 got_str='') 312 self.assertFalse(err.empty()) 313 314 printer, err, out = self.get_printer(['--print', 'trace-everything']) 315 result = self.get_result('passes/image.html') 316 printer.print_test_result(result, expected=True, exp_str='', 317 got_str='') 318 result = self.get_result('failures/expected/missing_text.html') 319 printer.print_test_result(result, expected=True, exp_str='', 320 got_str='') 321 result = self.get_result('failures/expected/missing_check.html') 322 printer.print_test_result(result, expected=True, exp_str='', 323 got_str='') 324 result = self.get_result('failures/expected/missing_image.html') 325 printer.print_test_result(result, expected=True, exp_str='', 326 got_str='') 327 self.assertFalse(err.empty()) 328 329 err.reset() 330 printer.print_test_result(result, expected=False, exp_str='', 331 got_str='') 332 333 def test_print_progress(self): 334 expectations = '' 335 336 # test that we print nothing 337 printer, err, out = self.get_printer(['--print', 'nothing']) 338 tests = ['passes/text.html', 'failures/expected/timeout.html', 339 'failures/expected/crash.html'] 340 paths, rs, exp = self.get_result_summary(tests, expectations) 341 342 printer.print_progress(rs, False, paths) 343 self.assertTrue(out.empty()) 344 self.assertTrue(err.empty()) 345 346 printer.print_progress(rs, True, paths) 347 self.assertTrue(out.empty()) 348 self.assertTrue(err.empty()) 349 350 # test regular functionality 351 printer, err, out = self.get_printer(['--print', 352 'one-line-progress']) 353 printer.print_progress(rs, False, paths) 354 self.assertTrue(out.empty()) 355 self.assertFalse(err.empty()) 356 357 err.reset() 358 out.reset() 359 printer.print_progress(rs, True, paths) 360 self.assertFalse(err.empty()) 361 self.assertTrue(out.empty()) 362 363 def test_print_progress__detailed(self): 364 tests = ['passes/text.html', 'failures/expected/timeout.html', 365 'failures/expected/crash.html'] 366 expectations = 'BUGX : failures/expected/timeout.html = TIMEOUT' 367 368 # first, test that it is disabled properly 369 # should still print one-line-progress 370 printer, err, out = self.get_printer( 371 ['--print', 'detailed-progress'], single_threaded=False) 372 paths, rs, exp = self.get_result_summary(tests, expectations) 373 printer.print_progress(rs, False, paths) 374 self.assertFalse(err.empty()) 375 self.assertTrue(out.empty()) 376 377 # now test the enabled paths 378 printer, err, out = self.get_printer( 379 ['--print', 'detailed-progress'], single_threaded=True) 380 paths, rs, exp = self.get_result_summary(tests, expectations) 381 printer.print_progress(rs, False, paths) 382 self.assertFalse(err.empty()) 383 self.assertTrue(out.empty()) 384 385 err.reset() 386 out.reset() 387 printer.print_progress(rs, True, paths) 388 self.assertFalse(err.empty()) 389 self.assertTrue(out.empty()) 390 391 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) 392 rs.add(self.get_result('failures/expected/timeout.html'), True) 393 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) 394 err.reset() 395 out.reset() 396 printer.print_progress(rs, False, paths) 397 self.assertFalse(err.empty()) 398 self.assertTrue(out.empty()) 399 400 # We only clear the meter when retrying w/ detailed-progress. 401 err.reset() 402 out.reset() 403 printer.print_progress(rs, True, paths) 404 self.assertFalse(err.empty()) 405 self.assertTrue(out.empty()) 406 407 printer, err, out = self.get_printer( 408 ['--print', 'detailed-progress,unexpected'], single_threaded=True) 409 paths, rs, exp = self.get_result_summary(tests, expectations) 410 printer.print_progress(rs, False, paths) 411 self.assertFalse(err.empty()) 412 self.assertTrue(out.empty()) 413 414 err.reset() 415 out.reset() 416 printer.print_progress(rs, True, paths) 417 self.assertFalse(err.empty()) 418 self.assertTrue(out.empty()) 419 420 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), False) 421 rs.add(self.get_result('failures/expected/timeout.html'), True) 422 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), True) 423 err.reset() 424 out.reset() 425 printer.print_progress(rs, False, paths) 426 self.assertFalse(err.empty()) 427 self.assertTrue(out.empty()) 428 429 # We only clear the meter when retrying w/ detailed-progress. 430 err.reset() 431 out.reset() 432 printer.print_progress(rs, True, paths) 433 self.assertFalse(err.empty()) 434 self.assertTrue(out.empty()) 435 436 def test_write_nothing(self): 437 printer, err, out = self.get_printer(['--print', 'nothing']) 438 printer.write("foo") 439 self.assertTrue(err.empty()) 440 441 def test_write_misc(self): 442 printer, err, out = self.get_printer(['--print', 'misc']) 443 printer.write("foo") 444 self.assertFalse(err.empty()) 445 err.reset() 446 printer.write("foo", "config") 447 self.assertTrue(err.empty()) 448 449 def test_write_everything(self): 450 printer, err, out = self.get_printer(['--print', 'everything']) 451 printer.write("foo") 452 self.assertFalse(err.empty()) 453 err.reset() 454 printer.write("foo", "config") 455 self.assertFalse(err.empty()) 456 457 def test_write_verbose(self): 458 printer, err, out = self.get_printer(['--verbose']) 459 printer.write("foo") 460 self.assertTrue(not err.empty() and "foo" in err.get()[0]) 461 self.assertTrue(out.empty()) 462 463 def test_print_unexpected_results(self): 464 # This routine is the only one that prints stuff that the bots 465 # care about. 466 # 467 # FIXME: there's some weird layering going on here. It seems 468 # like we shouldn't be both using an expectations string and 469 # having to specify whether or not the result was expected. 470 # This whole set of tests should probably be rewritten. 471 # 472 # FIXME: Plus, the fact that we're having to call into 473 # run_webkit_tests is clearly a layering inversion. 474 def get_unexpected_results(expected, passing, flaky): 475 """Return an unexpected results summary matching the input description. 476 477 There are a lot of different combinations of test results that 478 can be tested; this routine produces various combinations based 479 on the values of the input flags. 480 481 Args 482 expected: whether the tests ran as expected 483 passing: whether the tests should all pass 484 flaky: whether the tests should be flaky (if False, they 485 produce the same results on both runs; if True, they 486 all pass on the second run). 487 488 """ 489 paths, rs, exp = self.get_result_summary(tests, expectations) 490 if expected: 491 rs.add(self.get_result('passes/text.html', test_expectations.PASS), 492 expected) 493 rs.add(self.get_result('failures/expected/timeout.html', 494 test_expectations.TIMEOUT), expected) 495 rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), 496 expected) 497 elif passing: 498 rs.add(self.get_result('passes/text.html'), expected) 499 rs.add(self.get_result('failures/expected/timeout.html'), expected) 500 rs.add(self.get_result('failures/expected/crash.html'), expected) 501 else: 502 rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), 503 expected) 504 rs.add(self.get_result('failures/expected/timeout.html', 505 test_expectations.CRASH), expected) 506 rs.add(self.get_result('failures/expected/crash.html', 507 test_expectations.TIMEOUT), 508 expected) 509 retry = rs 510 if flaky: 511 paths, retry, exp = self.get_result_summary(tests, 512 expectations) 513 retry.add(self.get_result('passes/text.html'), True) 514 retry.add(self.get_result('failures/expected/timeout.html'), True) 515 retry.add(self.get_result('failures/expected/crash.html'), True) 516 unexpected_results = test_runner.summarize_results(self._port, exp, rs, retry, test_timings={}, only_unexpected=True) 517 return unexpected_results 518 519 tests = ['passes/text.html', 'failures/expected/timeout.html', 520 'failures/expected/crash.html'] 521 expectations = '' 522 523 printer, err, out = self.get_printer(['--print', 'nothing']) 524 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 525 printer.print_unexpected_results(ur) 526 self.assertTrue(err.empty()) 527 self.assertTrue(out.empty()) 528 529 printer, err, out = self.get_printer(['--print', 530 'unexpected-results']) 531 532 # test everything running as expected 533 ur = get_unexpected_results(expected=True, passing=False, flaky=False) 534 printer.print_unexpected_results(ur) 535 self.assertTrue(err.empty()) 536 self.assertTrue(out.empty()) 537 538 # test failures 539 err.reset() 540 out.reset() 541 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 542 printer.print_unexpected_results(ur) 543 self.assertTrue(err.empty()) 544 self.assertFalse(out.empty()) 545 546 # test unexpected flaky results 547 err.reset() 548 out.reset() 549 ur = get_unexpected_results(expected=False, passing=True, flaky=False) 550 printer.print_unexpected_results(ur) 551 self.assertTrue(err.empty()) 552 self.assertFalse(out.empty()) 553 554 # test unexpected passes 555 err.reset() 556 out.reset() 557 ur = get_unexpected_results(expected=False, passing=False, flaky=True) 558 printer.print_unexpected_results(ur) 559 self.assertTrue(err.empty()) 560 self.assertFalse(out.empty()) 561 562 err.reset() 563 out.reset() 564 printer, err, out = self.get_printer(['--print', 'everything']) 565 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 566 printer.print_unexpected_results(ur) 567 self.assertTrue(err.empty()) 568 self.assertFalse(out.empty()) 569 570 expectations = """ 571BUGX : failures/expected/crash.html = CRASH 572BUGX : failures/expected/timeout.html = TIMEOUT 573""" 574 err.reset() 575 out.reset() 576 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 577 printer.print_unexpected_results(ur) 578 self.assertTrue(err.empty()) 579 self.assertFalse(out.empty()) 580 581 err.reset() 582 out.reset() 583 ur = get_unexpected_results(expected=False, passing=True, flaky=False) 584 printer.print_unexpected_results(ur) 585 self.assertTrue(err.empty()) 586 self.assertFalse(out.empty()) 587 588 # Test handling of --verbose as well. 589 err.reset() 590 out.reset() 591 printer, err, out = self.get_printer(['--verbose']) 592 ur = get_unexpected_results(expected=False, passing=False, flaky=False) 593 printer.print_unexpected_results(ur) 594 self.assertTrue(err.empty()) 595 self.assertFalse(out.empty()) 596 597 def test_print_unexpected_results_buildbot(self): 598 # FIXME: Test that print_unexpected_results() produces the printer the 599 # buildbot is expecting. 600 pass 601 602if __name__ == '__main__': 603 unittest.main() 604