15821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 25821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 35821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# found in the LICENSE file. 45821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 55821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)"""A module to analyze test expectations for Webkit layout tests.""" 65821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 75821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)import urllib2 85821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 95821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)from webkitpy.layout_tests.models.test_expectations import * 105821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 11c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)# Default location for chromium test expectation file. 125821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# TODO(imasaki): support multiple test expectations files. 13c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles)DEFAULT_TEST_EXPECTATIONS_LOCATION = ( 14c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) 'http://src.chromium.org/blink/trunk/LayoutTests/TestExpectations') 155821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 165821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# The following is from test expectation syntax. The detail can be found in 175821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# http://www.chromium.org/developers/testing/webkit-layout-tests#TOC-Test-Expectations 185821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# <decision> ::== [SKIP] [WONTFIX] [SLOW] 195821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)DECISION_NAMES = ['SKIP', 'WONTFIX', 'SLOW'] 205821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# <config> ::== RELEASE | DEBUG 215821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)CONFIG_NAMES = ['RELEASE', 'DEBUG'] 225821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# Only hard code keywords we don't expect to change. Determine the rest from 235821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)# the format of the status line. 245821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)KNOWN_TE_KEYWORDS = DECISION_NAMES + CONFIG_NAMES 255821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 265821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 275821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles)class TestExpectations(object): 285821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """A class to model the content of test expectation file for analysis. 295821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 305821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) This class retrieves the TestExpectations file via HTTP from WebKit and uses 315821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) the WebKit layout test processor to process each line. 325821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 335821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) The resulting dictionary is stored in |all_test_expectation_info| and looks 345821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) like: 355821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 365821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) {'<test name>': [{'<modifier0>': True, '<modifier1>': True, ..., 375821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'Platforms: ['<platform0>', ... ], 'Bugs': ['....']}]} 385821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 395821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Duplicate keys are merged (though technically they shouldn't exist). 405821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 415821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Example: 425821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) crbug.com/145590 [ Android ] \ 435821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) platform/chromium/media/video-frame-size-change.html [ Timeout ] 445821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) webkit.org/b/84724 [ SnowLeopard ] \ 455821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) platform/chromium/media/video-frame-size-change.html \ 465821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) [ ImageOnlyFailure Pass ] 475821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 485821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) {'platform/chromium/media/video-frame-size-change.html': [{'IMAGE': True, 495821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'Bugs': ['BUGWK84724', 'BUGCR145590'], 'Comments': '', 505821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 'Platforms': ['SNOWLEOPARD', 'ANDROID'], 'TIMEOUT': True, 'PASS': True}]} 515821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 525821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 53c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) def __init__(self, url=DEFAULT_TEST_EXPECTATIONS_LOCATION): 545821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Read the test expectation file from the specified URL and parse it. 555821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 565821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Args: 575821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) url: A URL string for the test expectation file. 585821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 595821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Raises: 605821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) NameError when the test expectation file cannot be retrieved from |url|. 615821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 625821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.all_test_expectation_info = {} 635821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) resp = urllib2.urlopen(url) 645821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if resp.code != 200: 655821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) raise NameError('Test expectation file does not exist in %s' % url) 665821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Start parsing each line. 675821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for line in resp.read().split('\n'): 685821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) line = line.strip() 695821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Skip comments. 705821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if line.startswith('#'): 715821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 725821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) testname, te_info = self.ParseLine(line) 735821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if not testname or not te_info: 745821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) continue 755821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if testname in self.all_test_expectation_info: 765821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Merge keys if entry already exists. 775821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for k in te_info.keys(): 785821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if (isinstance(te_info[k], list) and 795821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) k in self.all_test_expectation_info[testname]): 805821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.all_test_expectation_info[testname][0][k] += te_info[k] 815821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 825821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.all_test_expectation_info[testname][0][k] = te_info[k] 835821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) else: 845821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) self.all_test_expectation_info[testname] = [te_info] 855821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 865821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) @staticmethod 875821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) def ParseLine(line): 885821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """Parses the provided line using WebKit's TextExpecations parser. 895821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 905821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Returns: 915821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) Tuple of test name, test expectations dictionary. See class documentation 925821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) for the format of the dictionary 935821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) """ 945821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) test_expectation_info = {} 955821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) parsed = TestExpectationParser._tokenize_line('TestExpectations', line, 0) 965821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) if parsed.is_invalid(): 975821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return None, None 985821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 995821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) test_expectation_info['Comments'] = parsed.comment or '' 1005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_expectation_info['Bugs'] = parsed.bugs or []; 1015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) test_expectation_info['Platforms'] = parsed.specifiers or [] 1025821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # Shovel the expectations and modifiers in as "<key>: True" entries. Ugly, 1035821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) # but required by the rest of the pipeline for parsing. 1045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) for m in parsed.expectations: 1055821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) test_expectation_info[m] = True 1065821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) 1075821806d5e7f356e8fa4b058a389a808ea183019Torne (Richard Coles) return parsed.name, test_expectation_info 108