test_expectations.py revision c2e0dbddbe15c98d52c4786dac06cb8952a8ae6d 
1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""A module to analyze test expectations for Webkit layout tests."""
6
7import urllib2
8
9from webkitpy.layout_tests.models.test_expectations import *
10
11# Default location for chromium test expectation file.
12# TODO(imasaki): support multiple test expectations files.
13DEFAULT_TEST_EXPECTATIONS_LOCATION = (
14    'http://src.chromium.org/blink/trunk/LayoutTests/TestExpectations')
15
16# The following is from test expectation syntax. The detail can be found in
17# http://www.chromium.org/developers/testing/webkit-layout-tests#TOC-Test-Expectations
18# <decision> ::== [SKIP] [WONTFIX] [SLOW]
19DECISION_NAMES = ['SKIP', 'WONTFIX', 'SLOW']
20# <config> ::== RELEASE | DEBUG
21CONFIG_NAMES = ['RELEASE', 'DEBUG']
22# Only hard code keywords we don't expect to change.  Determine the rest from
23# the format of the status line.
24KNOWN_TE_KEYWORDS = DECISION_NAMES + CONFIG_NAMES
25
26
27class TestExpectations(object):
28  """A class to model the content of test expectation file for analysis.
29
30  This class retrieves the TestExpectations file via HTTP from WebKit and uses
31  the WebKit layout test processor to process each line.
32
33  The resulting dictionary is stored in |all_test_expectation_info| and looks
34  like:
35
36    {'<test name>': [{'<modifier0>': True, '<modifier1>': True, ...,
37                     'Platforms: ['<platform0>', ... ], 'Bugs': ['....']}]}
38
39  Duplicate keys are merged (though technically they shouldn't exist).
40
41  Example:
42    crbug.com/145590 [ Android ] \
43        platform/chromium/media/video-frame-size-change.html [ Timeout ]
44    webkit.org/b/84724 [ SnowLeopard ] \
45        platform/chromium/media/video-frame-size-change.html \
46        [ ImageOnlyFailure Pass ]
47
48  {'platform/chromium/media/video-frame-size-change.html': [{'IMAGE': True,
49   'Bugs': ['BUGWK84724', 'BUGCR145590'], 'Comments': '',
50   'Platforms': ['SNOWLEOPARD', 'ANDROID'], 'TIMEOUT': True, 'PASS': True}]}
51  """
52
53  def __init__(self, url=DEFAULT_TEST_EXPECTATIONS_LOCATION):
54    """Read the test expectation file from the specified URL and parse it.
55
56    Args:
57      url: A URL string for the test expectation file.
58
59    Raises:
60      NameError when the test expectation file cannot be retrieved from |url|.
61    """
62    self.all_test_expectation_info = {}
63    resp = urllib2.urlopen(url)
64    if resp.code != 200:
65      raise NameError('Test expectation file does not exist in %s' % url)
66    # Start parsing each line.
67    for line in resp.read().split('\n'):
68      line = line.strip()
69      # Skip comments.
70      if line.startswith('#'):
71        continue
72      testname, te_info = self.ParseLine(line)
73      if not testname or not te_info:
74        continue
75      if testname in self.all_test_expectation_info:
76        # Merge keys if entry already exists.
77        for k in te_info.keys():
78          if (isinstance(te_info[k], list) and
79              k in self.all_test_expectation_info[testname]):
80            self.all_test_expectation_info[testname][0][k] += te_info[k]
81          else:
82            self.all_test_expectation_info[testname][0][k] = te_info[k]
83      else:
84        self.all_test_expectation_info[testname] = [te_info]
85
86  @staticmethod
87  def ParseLine(line):
88    """Parses the provided line using WebKit's TextExpecations parser.
89
90    Returns:
91      Tuple of test name, test expectations dictionary.  See class documentation
92      for the format of the dictionary
93    """
94    test_expectation_info = {}
95    parsed = TestExpectationParser._tokenize_line('TestExpectations', line, 0)
96    if parsed.is_invalid():
97      return None, None
98
99    test_expectation_info['Comments'] = parsed.comment or ''
100
101    # Split the modifiers dictionary into the format we want.
102    remaining_modifiers = list(parsed.modifiers)
103    test_expectation_info['Bugs'] = []
104    for m in parsed.modifiers:
105      if m.startswith('BUG'):
106        test_expectation_info['Bugs'].append(m)
107        remaining_modifiers.remove(m)
108      elif m in KNOWN_TE_KEYWORDS:
109        test_expectation_info[m] = True
110        remaining_modifiers.remove(m)
111
112    # The modifiers left over should all be platform names.
113    test_expectation_info['Platforms'] = list(remaining_modifiers)
114
115    # Shovel the expectations and modifiers in as "<key>: True" entries.  Ugly,
116    # but required by the rest of the pipeline for parsing.
117    for m in parsed.expectations + remaining_modifiers:
118      test_expectation_info[m] = True
119
120    return parsed.name, test_expectation_info
121