test_expectations.py revision f05b935882198ccf7d81675736e3aeb089c5113a
1#!/usr/bin/env python 2# Copyright (C) 2010 Google Inc. All rights reserved. 3# 4# Redistribution and use in source and binary forms, with or without 5# modification, are permitted provided that the following conditions are 6# met: 7# 8# * Redistributions of source code must retain the above copyright 9# notice, this list of conditions and the following disclaimer. 10# * Redistributions in binary form must reproduce the above 11# copyright notice, this list of conditions and the following disclaimer 12# in the documentation and/or other materials provided with the 13# distribution. 14# * Neither the name of Google Inc. nor the names of its 15# contributors may be used to endorse or promote products derived from 16# this software without specific prior written permission. 17# 18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 30"""A helper class for reading in and dealing with tests expectations 31for layout tests. 32""" 33 34import logging 35import os 36import re 37import sys 38 39import webkitpy.thirdparty.simplejson as simplejson 40 41_log = logging.getLogger("webkitpy.layout_tests.layout_package." 42 "test_expectations") 43 44# Test expectation and modifier constants. 45(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, TIMEOUT, CRASH, SKIP, WONTFIX, 46 SLOW, REBASELINE, MISSING, FLAKY, NOW, NONE) = range(15) 47 48# Test expectation file update action constants 49(NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, ADD_PLATFORMS_EXCEPT_THIS) = range(4) 50 51 52def result_was_expected(result, expected_results, test_needs_rebaselining, 53 test_is_skipped): 54 """Returns whether we got a result we were expecting. 55 Args: 56 result: actual result of a test execution 57 expected_results: set of results listed in test_expectations 58 test_needs_rebaselining: whether test was marked as REBASELINE 59 test_is_skipped: whether test was marked as SKIP""" 60 if result in expected_results: 61 return True 62 if result in (IMAGE, TEXT, IMAGE_PLUS_TEXT) and FAIL in expected_results: 63 return True 64 if result == MISSING and test_needs_rebaselining: 65 return True 66 if result == SKIP and test_is_skipped: 67 return True 68 return False 69 70 71def remove_pixel_failures(expected_results): 72 """Returns a copy of the expected results for a test, except that we 73 drop any pixel failures and return the remaining expectations. For example, 74 if we're not running pixel tests, then tests expected to fail as IMAGE 75 will PASS.""" 76 expected_results = expected_results.copy() 77 if IMAGE in expected_results: 78 expected_results.remove(IMAGE) 79 expected_results.add(PASS) 80 if IMAGE_PLUS_TEXT in expected_results: 81 expected_results.remove(IMAGE_PLUS_TEXT) 82 expected_results.add(TEXT) 83 return expected_results 84 85 86class TestExpectations: 87 TEST_LIST = "test_expectations.txt" 88 89 def __init__(self, port, tests, expectations, test_platform_name, 90 is_debug_mode, is_lint_mode, overrides=None): 91 """Loads and parses the test expectations given in the string. 92 Args: 93 port: handle to object containing platform-specific functionality 94 test: list of all of the test files 95 expectations: test expectations as a string 96 test_platform_name: name of the platform to match expectations 97 against. Note that this may be different than 98 port.test_platform_name() when is_lint_mode is True. 99 is_debug_mode: whether to use the DEBUG or RELEASE modifiers 100 in the expectations 101 is_lint_mode: If True, just parse the expectations string 102 looking for errors. 103 overrides: test expectations that are allowed to override any 104 entries in |expectations|. This is used by callers 105 that need to manage two sets of expectations (e.g., upstream 106 and downstream expectations). 107 """ 108 self._expected_failures = TestExpectationsFile(port, expectations, 109 tests, test_platform_name, is_debug_mode, is_lint_mode, 110 overrides=overrides) 111 112 # TODO(ojan): Allow for removing skipped tests when getting the list of 113 # tests to run, but not when getting metrics. 114 # TODO(ojan): Replace the Get* calls here with the more sane API exposed 115 # by TestExpectationsFile below. Maybe merge the two classes entirely? 116 117 def get_expectations_json_for_all_platforms(self): 118 return ( 119 self._expected_failures.get_expectations_json_for_all_platforms()) 120 121 def get_rebaselining_failures(self): 122 return (self._expected_failures.get_test_set(REBASELINE, FAIL) | 123 self._expected_failures.get_test_set(REBASELINE, IMAGE) | 124 self._expected_failures.get_test_set(REBASELINE, TEXT) | 125 self._expected_failures.get_test_set(REBASELINE, 126 IMAGE_PLUS_TEXT)) 127 128 def get_options(self, test): 129 return self._expected_failures.get_options(test) 130 131 def get_expectations(self, test): 132 return self._expected_failures.get_expectations(test) 133 134 def get_expectations_string(self, test): 135 """Returns the expectatons for the given test as an uppercase string. 136 If there are no expectations for the test, then "PASS" is returned.""" 137 expectations = self.get_expectations(test) 138 retval = [] 139 140 for expectation in expectations: 141 retval.append(self.expectation_to_string(expectation)) 142 143 return " ".join(retval) 144 145 def expectation_to_string(self, expectation): 146 """Return the uppercased string equivalent of a given expectation.""" 147 for item in TestExpectationsFile.EXPECTATIONS.items(): 148 if item[1] == expectation: 149 return item[0].upper() 150 raise ValueError(expectation) 151 152 def get_tests_with_result_type(self, result_type): 153 return self._expected_failures.get_tests_with_result_type(result_type) 154 155 def get_tests_with_timeline(self, timeline): 156 return self._expected_failures.get_tests_with_timeline(timeline) 157 158 def matches_an_expected_result(self, test, result, 159 pixel_tests_are_enabled): 160 expected_results = self._expected_failures.get_expectations(test) 161 if not pixel_tests_are_enabled: 162 expected_results = remove_pixel_failures(expected_results) 163 return result_was_expected(result, expected_results, 164 self.is_rebaselining(test), self.has_modifier(test, SKIP)) 165 166 def is_rebaselining(self, test): 167 return self._expected_failures.has_modifier(test, REBASELINE) 168 169 def has_modifier(self, test, modifier): 170 return self._expected_failures.has_modifier(test, modifier) 171 172 def remove_platform_from_expectations(self, tests, platform): 173 return self._expected_failures.remove_platform_from_expectations( 174 tests, platform) 175 176 177def strip_comments(line): 178 """Strips comments from a line and return None if the line is empty 179 or else the contents of line with leading and trailing spaces removed 180 and all other whitespace collapsed""" 181 182 commentIndex = line.find('//') 183 if commentIndex is -1: 184 commentIndex = len(line) 185 186 line = re.sub(r'\s+', ' ', line[:commentIndex].strip()) 187 if line == '': 188 return None 189 else: 190 return line 191 192 193class ParseError(Exception): 194 def __init__(self, fatal, errors): 195 self.fatal = fatal 196 self.errors = errors 197 198 def __str__(self): 199 return '\n'.join(map(str, self.errors)) 200 201 def __repr__(self): 202 return 'ParseError(fatal=%s, errors=%s)' % (fatal, errors) 203 204 205class ModifiersAndExpectations: 206 """A holder for modifiers and expectations on a test that serializes to 207 JSON.""" 208 209 def __init__(self, modifiers, expectations): 210 self.modifiers = modifiers 211 self.expectations = expectations 212 213 214class ExpectationsJsonEncoder(simplejson.JSONEncoder): 215 """JSON encoder that can handle ModifiersAndExpectations objects.""" 216 def default(self, obj): 217 # A ModifiersAndExpectations object has two fields, each of which 218 # is a dict. Since JSONEncoders handle all the builtin types directly, 219 # the only time this routine should be called is on the top level 220 # object (i.e., the encoder shouldn't recurse). 221 assert isinstance(obj, ModifiersAndExpectations) 222 return {"modifiers": obj.modifiers, 223 "expectations": obj.expectations} 224 225 226class TestExpectationsFile: 227 """Test expectation files consist of lines with specifications of what 228 to expect from layout test cases. The test cases can be directories 229 in which case the expectations apply to all test cases in that 230 directory and any subdirectory. The format of the file is along the 231 lines of: 232 233 LayoutTests/fast/js/fixme.js = FAIL 234 LayoutTests/fast/js/flaky.js = FAIL PASS 235 LayoutTests/fast/js/crash.js = CRASH TIMEOUT FAIL PASS 236 ... 237 238 To add other options: 239 SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 240 DEBUG : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 241 DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 242 LINUX DEBUG SKIP : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 243 LINUX WIN : LayoutTests/fast/js/no-good.js = TIMEOUT PASS 244 245 SKIP: Doesn't run the test. 246 SLOW: The test takes a long time to run, but does not timeout indefinitely. 247 WONTFIX: For tests that we never intend to pass on a given platform. 248 DEBUG: Expectations apply only to the debug build. 249 RELEASE: Expectations apply only to release build. 250 LINUX/WIN/WIN-XP/WIN-VISTA/WIN-7/MAC: Expectations apply only to these 251 platforms. 252 253 Notes: 254 -A test cannot be both SLOW and TIMEOUT 255 -A test should only be one of IMAGE, TEXT, IMAGE+TEXT, or FAIL. FAIL is 256 a migratory state that currently means either IMAGE, TEXT, or 257 IMAGE+TEXT. Once we have finished migrating the expectations, we will 258 change FAIL to have the meaning of IMAGE+TEXT and remove the IMAGE+TEXT 259 identifier. 260 -A test can be included twice, but not via the same path. 261 -If a test is included twice, then the more precise path wins. 262 -CRASH tests cannot be WONTFIX 263 """ 264 265 EXPECTATIONS = {'pass': PASS, 266 'fail': FAIL, 267 'text': TEXT, 268 'image': IMAGE, 269 'image+text': IMAGE_PLUS_TEXT, 270 'timeout': TIMEOUT, 271 'crash': CRASH, 272 'missing': MISSING} 273 274 EXPECTATION_DESCRIPTIONS = {SKIP: ('skipped', 'skipped'), 275 PASS: ('pass', 'passes'), 276 FAIL: ('failure', 'failures'), 277 TEXT: ('text diff mismatch', 278 'text diff mismatch'), 279 IMAGE: ('image mismatch', 'image mismatch'), 280 IMAGE_PLUS_TEXT: ('image and text mismatch', 281 'image and text mismatch'), 282 CRASH: ('DumpRenderTree crash', 283 'DumpRenderTree crashes'), 284 TIMEOUT: ('test timed out', 'tests timed out'), 285 MISSING: ('no expected result found', 286 'no expected results found')} 287 288 EXPECTATION_ORDER = (PASS, CRASH, TIMEOUT, MISSING, IMAGE_PLUS_TEXT, 289 TEXT, IMAGE, FAIL, SKIP) 290 291 BUILD_TYPES = ('debug', 'release') 292 293 MODIFIERS = {'skip': SKIP, 294 'wontfix': WONTFIX, 295 'slow': SLOW, 296 'rebaseline': REBASELINE, 297 'none': NONE} 298 299 TIMELINES = {'wontfix': WONTFIX, 300 'now': NOW} 301 302 RESULT_TYPES = {'skip': SKIP, 303 'pass': PASS, 304 'fail': FAIL, 305 'flaky': FLAKY} 306 307 def __init__(self, port, expectations, full_test_list, test_platform_name, 308 is_debug_mode, is_lint_mode, overrides=None): 309 """ 310 expectations: Contents of the expectations file 311 full_test_list: The list of all tests to be run pending processing of 312 the expections for those tests. 313 test_platform_name: name of the platform to match expectations 314 against. Note that this may be different than 315 port.test_platform_name() when is_lint_mode is True. 316 is_debug_mode: Whether we testing a test_shell built debug mode. 317 is_lint_mode: Whether this is just linting test_expecatations.txt. 318 overrides: test expectations that are allowed to override any 319 entries in |expectations|. This is used by callers 320 that need to manage two sets of expectations (e.g., upstream 321 and downstream expectations). 322 """ 323 324 self._port = port 325 self._expectations = expectations 326 self._full_test_list = full_test_list 327 self._test_platform_name = test_platform_name 328 self._is_debug_mode = is_debug_mode 329 self._is_lint_mode = is_lint_mode 330 self._overrides = overrides 331 self._errors = [] 332 self._non_fatal_errors = [] 333 334 # Maps relative test paths as listed in the expectations file to a 335 # list of maps containing modifiers and expectations for each time 336 # the test is listed in the expectations file. 337 self._all_expectations = {} 338 339 # Maps a test to its list of expectations. 340 self._test_to_expectations = {} 341 342 # Maps a test to its list of options (string values) 343 self._test_to_options = {} 344 345 # Maps a test to its list of modifiers: the constants associated with 346 # the options minus any bug or platform strings 347 self._test_to_modifiers = {} 348 349 # Maps a test to the base path that it was listed with in the list. 350 self._test_list_paths = {} 351 352 self._modifier_to_tests = self._dict_of_sets(self.MODIFIERS) 353 self._expectation_to_tests = self._dict_of_sets(self.EXPECTATIONS) 354 self._timeline_to_tests = self._dict_of_sets(self.TIMELINES) 355 self._result_type_to_tests = self._dict_of_sets(self.RESULT_TYPES) 356 357 self._read(self._get_iterable_expectations(self._expectations), 358 overrides_allowed=False) 359 360 # List of tests that are in the overrides file (used for checking for 361 # duplicates inside the overrides file itself). Note that just because 362 # a test is in this set doesn't mean it's necessarily overridding a 363 # expectation in the regular expectations; the test might not be 364 # mentioned in the regular expectations file at all. 365 self._overridding_tests = set() 366 367 if overrides: 368 self._read(self._get_iterable_expectations(self._overrides), 369 overrides_allowed=True) 370 371 self._handle_any_read_errors() 372 self._process_tests_without_expectations() 373 374 def _handle_any_read_errors(self): 375 if len(self._errors) or len(self._non_fatal_errors): 376 if self._is_debug_mode: 377 build_type = 'DEBUG' 378 else: 379 build_type = 'RELEASE' 380 _log.error('') 381 _log.error("FAILURES FOR PLATFORM: %s, BUILD_TYPE: %s" % 382 (self._test_platform_name.upper(), build_type)) 383 384 for error in self._errors: 385 _log.error(error) 386 for error in self._non_fatal_errors: 387 _log.error(error) 388 389 if len(self._errors): 390 raise ParseError(fatal=True, errors=self._errors) 391 if len(self._non_fatal_errors) and self._is_lint_mode: 392 raise ParseError(fatal=False, errors=self._non_fatal_errors) 393 394 def _process_tests_without_expectations(self): 395 expectations = set([PASS]) 396 options = [] 397 modifiers = [] 398 if self._full_test_list: 399 for test in self._full_test_list: 400 if not test in self._test_list_paths: 401 self._add_test(test, modifiers, expectations, options, 402 overrides_allowed=False) 403 404 def _dict_of_sets(self, strings_to_constants): 405 """Takes a dict of strings->constants and returns a dict mapping 406 each constant to an empty set.""" 407 d = {} 408 for c in strings_to_constants.values(): 409 d[c] = set() 410 return d 411 412 def _get_iterable_expectations(self, expectations_str): 413 """Returns an object that can be iterated over. Allows for not caring 414 about whether we're iterating over a file or a new-line separated 415 string.""" 416 iterable = [x + "\n" for x in expectations_str.split("\n")] 417 # Strip final entry if it's empty to avoid added in an extra 418 # newline. 419 if iterable[-1] == "\n": 420 return iterable[:-1] 421 return iterable 422 423 def get_test_set(self, modifier, expectation=None, include_skips=True): 424 if expectation is None: 425 tests = self._modifier_to_tests[modifier] 426 else: 427 tests = (self._expectation_to_tests[expectation] & 428 self._modifier_to_tests[modifier]) 429 430 if not include_skips: 431 tests = tests - self.get_test_set(SKIP, expectation) 432 433 return tests 434 435 def get_tests_with_result_type(self, result_type): 436 return self._result_type_to_tests[result_type] 437 438 def get_tests_with_timeline(self, timeline): 439 return self._timeline_to_tests[timeline] 440 441 def get_options(self, test): 442 """This returns the entire set of options for the given test 443 (the modifiers plus the BUGXXXX identifier). This is used by the 444 LTTF dashboard.""" 445 return self._test_to_options[test] 446 447 def has_modifier(self, test, modifier): 448 return test in self._modifier_to_tests[modifier] 449 450 def get_expectations(self, test): 451 return self._test_to_expectations[test] 452 453 def get_expectations_json_for_all_platforms(self): 454 # Specify separators in order to get compact encoding. 455 return ExpectationsJsonEncoder(separators=(',', ':')).encode( 456 self._all_expectations) 457 458 def get_non_fatal_errors(self): 459 return self._non_fatal_errors 460 461 def remove_platform_from_expectations(self, tests, platform): 462 """Returns a copy of the expectations with the tests matching the 463 platform removed. 464 465 If a test is in the test list and has an option that matches the given 466 platform, remove the matching platform and save the updated test back 467 to the file. If no other platforms remaining after removal, delete the 468 test from the file. 469 470 Args: 471 tests: list of tests that need to update.. 472 platform: which platform option to remove. 473 474 Returns: 475 the updated string. 476 """ 477 478 assert(platform) 479 f_orig = self._get_iterable_expectations(self._expectations) 480 f_new = [] 481 482 tests_removed = 0 483 tests_updated = 0 484 lineno = 0 485 for line in f_orig: 486 lineno += 1 487 action = self._get_platform_update_action(line, lineno, tests, 488 platform) 489 assert(action in (NO_CHANGE, REMOVE_TEST, REMOVE_PLATFORM, 490 ADD_PLATFORMS_EXCEPT_THIS)) 491 if action == NO_CHANGE: 492 # Save the original line back to the file 493 _log.debug('No change to test: %s', line) 494 f_new.append(line) 495 elif action == REMOVE_TEST: 496 tests_removed += 1 497 _log.info('Test removed: %s', line) 498 elif action == REMOVE_PLATFORM: 499 parts = line.split(':') 500 new_options = parts[0].replace(platform.upper() + ' ', '', 1) 501 new_line = ('%s:%s' % (new_options, parts[1])) 502 f_new.append(new_line) 503 tests_updated += 1 504 _log.info('Test updated: ') 505 _log.info(' old: %s', line) 506 _log.info(' new: %s', new_line) 507 elif action == ADD_PLATFORMS_EXCEPT_THIS: 508 parts = line.split(':') 509 new_options = parts[0] 510 for p in self._port.test_platform_names(): 511 p = p.upper() 512 # This is a temp solution for rebaselining tool. 513 # Do not add tags WIN-7 and WIN-VISTA to test expectations 514 # if the original line does not specify the platform 515 # option. 516 # TODO(victorw): Remove WIN-VISTA and WIN-7 once we have 517 # reliable Win 7 and Win Vista buildbots setup. 518 if not p in (platform.upper(), 'WIN-VISTA', 'WIN-7'): 519 new_options += p + ' ' 520 new_line = ('%s:%s' % (new_options, parts[1])) 521 f_new.append(new_line) 522 tests_updated += 1 523 _log.info('Test updated: ') 524 _log.info(' old: %s', line) 525 _log.info(' new: %s', new_line) 526 527 _log.info('Total tests removed: %d', tests_removed) 528 _log.info('Total tests updated: %d', tests_updated) 529 530 return "".join(f_new) 531 532 def parse_expectations_line(self, line, lineno): 533 """Parses a line from test_expectations.txt and returns a tuple 534 with the test path, options as a list, expectations as a list.""" 535 line = strip_comments(line) 536 if not line: 537 return (None, None, None) 538 539 options = [] 540 if line.find(":") is -1: 541 test_and_expectation = line.split("=") 542 else: 543 parts = line.split(":") 544 options = self._get_options_list(parts[0]) 545 test_and_expectation = parts[1].split('=') 546 547 test = test_and_expectation[0].strip() 548 if (len(test_and_expectation) is not 2): 549 self._add_error(lineno, "Missing expectations.", 550 test_and_expectation) 551 expectations = None 552 else: 553 expectations = self._get_options_list(test_and_expectation[1]) 554 555 return (test, options, expectations) 556 557 def _get_platform_update_action(self, line, lineno, tests, platform): 558 """Check the platform option and return the action needs to be taken. 559 560 Args: 561 line: current line in test expectations file. 562 lineno: current line number of line 563 tests: list of tests that need to update.. 564 platform: which platform option to remove. 565 566 Returns: 567 NO_CHANGE: no change to the line (comments, test not in the list etc) 568 REMOVE_TEST: remove the test from file. 569 REMOVE_PLATFORM: remove this platform option from the test. 570 ADD_PLATFORMS_EXCEPT_THIS: add all the platforms except this one. 571 """ 572 test, options, expectations = self.parse_expectations_line(line, 573 lineno) 574 if not test or test not in tests: 575 return NO_CHANGE 576 577 has_any_platform = False 578 for option in options: 579 if option in self._port.test_platform_names(): 580 has_any_platform = True 581 if not option == platform: 582 return REMOVE_PLATFORM 583 584 # If there is no platform specified, then it means apply to all 585 # platforms. Return the action to add all the platforms except this 586 # one. 587 if not has_any_platform: 588 return ADD_PLATFORMS_EXCEPT_THIS 589 590 return REMOVE_TEST 591 592 def _has_valid_modifiers_for_current_platform(self, options, lineno, 593 test_and_expectations, modifiers): 594 """Returns true if the current platform is in the options list or if 595 no platforms are listed and if there are no fatal errors in the 596 options list. 597 598 Args: 599 options: List of lowercase options. 600 lineno: The line in the file where the test is listed. 601 test_and_expectations: The path and expectations for the test. 602 modifiers: The set to populate with modifiers. 603 """ 604 has_any_platform = False 605 has_bug_id = False 606 for option in options: 607 if option in self.MODIFIERS: 608 modifiers.add(option) 609 elif option in self._port.test_platform_names(): 610 has_any_platform = True 611 elif re.match(r'bug\d', option) != None: 612 self._add_error(lineno, 'Bug must be either BUGCR, BUGWK, or BUGV8_ for test: %s' % 613 option, test_and_expectations) 614 elif option.startswith('bug'): 615 has_bug_id = True 616 elif option not in self.BUILD_TYPES: 617 self._add_error(lineno, 'Invalid modifier for test: %s' % 618 option, test_and_expectations) 619 620 if has_any_platform and not self._match_platform(options): 621 return False 622 623 if not has_bug_id and 'wontfix' not in options: 624 # TODO(ojan): Turn this into an AddError call once all the 625 # tests have BUG identifiers. 626 self._log_non_fatal_error(lineno, 'Test lacks BUG modifier.', 627 test_and_expectations) 628 629 if 'release' in options or 'debug' in options: 630 if self._is_debug_mode and 'debug' not in options: 631 return False 632 if not self._is_debug_mode and 'release' not in options: 633 return False 634 635 if self._is_lint_mode and 'rebaseline' in options: 636 self._add_error(lineno, 637 'REBASELINE should only be used for running rebaseline.py. ' 638 'Cannot be checked in.', test_and_expectations) 639 640 return True 641 642 def _match_platform(self, options): 643 """Match the list of options against our specified platform. If any 644 of the options prefix-match self._platform, return True. This handles 645 the case where a test is marked WIN and the platform is WIN-VISTA. 646 647 Args: 648 options: list of options 649 """ 650 for opt in options: 651 if self._test_platform_name.startswith(opt): 652 return True 653 return False 654 655 def _add_to_all_expectations(self, test, options, expectations): 656 # Make all paths unix-style so the dashboard doesn't need to. 657 test = test.replace('\\', '/') 658 if not test in self._all_expectations: 659 self._all_expectations[test] = [] 660 self._all_expectations[test].append( 661 ModifiersAndExpectations(options, expectations)) 662 663 def _read(self, expectations, overrides_allowed): 664 """For each test in an expectations iterable, generate the 665 expectations for it.""" 666 lineno = 0 667 for line in expectations: 668 lineno += 1 669 670 test_list_path, options, expectations = \ 671 self.parse_expectations_line(line, lineno) 672 if not expectations: 673 continue 674 675 self._add_to_all_expectations(test_list_path, 676 " ".join(options).upper(), 677 " ".join(expectations).upper()) 678 679 modifiers = set() 680 if options and not self._has_valid_modifiers_for_current_platform( 681 options, lineno, test_list_path, modifiers): 682 continue 683 684 expectations = self._parse_expectations(expectations, lineno, 685 test_list_path) 686 687 if 'slow' in options and TIMEOUT in expectations: 688 self._add_error(lineno, 689 'A test can not be both slow and timeout. If it times out ' 690 'indefinitely, then it should be just timeout.', 691 test_list_path) 692 693 full_path = os.path.join(self._port.layout_tests_dir(), 694 test_list_path) 695 full_path = os.path.normpath(full_path) 696 # WebKit's way of skipping tests is to add a -disabled suffix. 697 # So we should consider the path existing if the path or the 698 # -disabled version exists. 699 if (not self._port.path_exists(full_path) 700 and not self._port.path_exists(full_path + '-disabled')): 701 # Log a non fatal error here since you hit this case any 702 # time you update test_expectations.txt without syncing 703 # the LayoutTests directory 704 self._log_non_fatal_error(lineno, 'Path does not exist.', 705 test_list_path) 706 continue 707 708 if not self._full_test_list: 709 tests = [test_list_path] 710 else: 711 tests = self._expand_tests(test_list_path) 712 713 self._add_tests(tests, expectations, test_list_path, lineno, 714 modifiers, options, overrides_allowed) 715 716 def _get_options_list(self, listString): 717 return [part.strip().lower() for part in listString.strip().split(' ')] 718 719 def _parse_expectations(self, expectations, lineno, test_list_path): 720 result = set() 721 for part in expectations: 722 if not part in self.EXPECTATIONS: 723 self._add_error(lineno, 'Unsupported expectation: %s' % part, 724 test_list_path) 725 continue 726 expectation = self.EXPECTATIONS[part] 727 result.add(expectation) 728 return result 729 730 def _expand_tests(self, test_list_path): 731 """Convert the test specification to an absolute, normalized 732 path and make sure directories end with the OS path separator.""" 733 # FIXME: full_test_list can quickly contain a big amount of 734 # elements. We should consider at some point to use a more 735 # efficient structure instead of a list. Maybe a dictionary of 736 # lists to represent the tree of tests, leaves being test 737 # files and nodes being categories. 738 739 path = os.path.join(self._port.layout_tests_dir(), test_list_path) 740 path = os.path.normpath(path) 741 if self._port.path_isdir(path): 742 # this is a test category, return all the tests of the category. 743 path = os.path.join(path, '') 744 745 return [test for test in self._full_test_list if test.startswith(path)] 746 747 # this is a test file, do a quick check if it's in the 748 # full test suite. 749 result = [] 750 if path in self._full_test_list: 751 result = [path, ] 752 return result 753 754 def _add_tests(self, tests, expectations, test_list_path, lineno, 755 modifiers, options, overrides_allowed): 756 for test in tests: 757 if self._already_seen_test(test, test_list_path, lineno, 758 overrides_allowed): 759 continue 760 761 self._clear_expectations_for_test(test, test_list_path) 762 self._add_test(test, modifiers, expectations, options, 763 overrides_allowed) 764 765 def _add_test(self, test, modifiers, expectations, options, 766 overrides_allowed): 767 """Sets the expected state for a given test. 768 769 This routine assumes the test has not been added before. If it has, 770 use _ClearExpectationsForTest() to reset the state prior to 771 calling this. 772 773 Args: 774 test: test to add 775 modifiers: sequence of modifier keywords ('wontfix', 'slow', etc.) 776 expectations: sequence of expectations (PASS, IMAGE, etc.) 777 options: sequence of keywords and bug identifiers. 778 overrides_allowed: whether we're parsing the regular expectations 779 or the overridding expectations""" 780 self._test_to_expectations[test] = expectations 781 for expectation in expectations: 782 self._expectation_to_tests[expectation].add(test) 783 784 self._test_to_options[test] = options 785 self._test_to_modifiers[test] = set() 786 for modifier in modifiers: 787 mod_value = self.MODIFIERS[modifier] 788 self._modifier_to_tests[mod_value].add(test) 789 self._test_to_modifiers[test].add(mod_value) 790 791 if 'wontfix' in modifiers: 792 self._timeline_to_tests[WONTFIX].add(test) 793 else: 794 self._timeline_to_tests[NOW].add(test) 795 796 if 'skip' in modifiers: 797 self._result_type_to_tests[SKIP].add(test) 798 elif expectations == set([PASS]): 799 self._result_type_to_tests[PASS].add(test) 800 elif len(expectations) > 1: 801 self._result_type_to_tests[FLAKY].add(test) 802 else: 803 self._result_type_to_tests[FAIL].add(test) 804 805 if overrides_allowed: 806 self._overridding_tests.add(test) 807 808 def _clear_expectations_for_test(self, test, test_list_path): 809 """Remove prexisting expectations for this test. 810 This happens if we are seeing a more precise path 811 than a previous listing. 812 """ 813 if test in self._test_list_paths: 814 self._test_to_expectations.pop(test, '') 815 self._remove_from_sets(test, self._expectation_to_tests) 816 self._remove_from_sets(test, self._modifier_to_tests) 817 self._remove_from_sets(test, self._timeline_to_tests) 818 self._remove_from_sets(test, self._result_type_to_tests) 819 820 self._test_list_paths[test] = os.path.normpath(test_list_path) 821 822 def _remove_from_sets(self, test, dict): 823 """Removes the given test from the sets in the dictionary. 824 825 Args: 826 test: test to look for 827 dict: dict of sets of files""" 828 for set_of_tests in dict.itervalues(): 829 if test in set_of_tests: 830 set_of_tests.remove(test) 831 832 def _already_seen_test(self, test, test_list_path, lineno, 833 allow_overrides): 834 """Returns true if we've already seen a more precise path for this test 835 than the test_list_path. 836 """ 837 if not test in self._test_list_paths: 838 return False 839 840 prev_base_path = self._test_list_paths[test] 841 if (prev_base_path == os.path.normpath(test_list_path)): 842 if (not allow_overrides or test in self._overridding_tests): 843 if allow_overrides: 844 expectation_source = "override" 845 else: 846 expectation_source = "expectation" 847 self._add_error(lineno, 'Duplicate %s.' % expectation_source, 848 test) 849 return True 850 else: 851 # We have seen this path, but that's okay because its 852 # in the overrides and the earlier path was in the 853 # expectations. 854 return False 855 856 # Check if we've already seen a more precise path. 857 return prev_base_path.startswith(os.path.normpath(test_list_path)) 858 859 def _add_error(self, lineno, msg, path): 860 """Reports an error that will prevent running the tests. Does not 861 immediately raise an exception because we'd like to aggregate all the 862 errors so they can all be printed out.""" 863 self._errors.append('Line:%s %s %s' % (lineno, msg, path)) 864 865 def _log_non_fatal_error(self, lineno, msg, path): 866 """Reports an error that will not prevent running the tests. These are 867 still errors, but not bad enough to warrant breaking test running.""" 868 self._non_fatal_errors.append('Line:%s %s %s' % (lineno, msg, path)) 869