results.py revision b144271179aaf82cb1151e9dfd8e866747402594 
1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
10"""
11
12# System-level imports
13import fnmatch
14import os
15import re
16
17# Imports from within Skia
18import fix_pythonpath  # must do this first
19import gm_json
20import imagepairset
21
22# Keys used to link an image to a particular GM test.
23# NOTE: Keep these in sync with static/constants.js
24VALUE__HEADER__SCHEMA_VERSION = 3
25KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
26KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
27KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
28KEY__EXTRACOLUMNS__BUILDER = 'builder'
29KEY__EXTRACOLUMNS__CONFIG = 'config'
30KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType'
31KEY__EXTRACOLUMNS__TEST = 'test'
32KEY__HEADER__DATAHASH = 'dataHash'
33KEY__HEADER__IS_EDITABLE = 'isEditable'
34KEY__HEADER__IS_EXPORTED = 'isExported'
35KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
36KEY__HEADER__RESULTS_ALL = 'all'
37KEY__HEADER__RESULTS_FAILURES = 'failures'
38KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
39KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
40KEY__HEADER__TIME_UPDATED = 'timeUpdated'
41KEY__HEADER__TYPE = 'type'
42KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
43KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
44KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
45KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
46
47IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
48IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
49
50PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
51DEFAULT_ACTUALS_DIR = '.gm-actuals'
52DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
53    PARENT_DIRECTORY, '.generated-images')
54
55# Define the default set of builders we will process expectations/actuals for.
56# This allows us to ignore builders for which we don't maintain expectations
57# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
58# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
59# produces error when trying to add baselines for ASAN/TSAN builders')
60DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
61DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
62    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']
63
64
65class BaseComparisons(object):
66  """Base class for generating summary of comparisons between two image sets.
67  """
68
69  def get_results_of_type(self, results_type):
70    """Return results of some/all tests (depending on 'results_type' parameter).
71
72    Args:
73      results_type: string describing which types of results to include; must
74          be one of the RESULTS_* constants
75
76    Results are returned in a dictionary as output by ImagePairSet.as_dict().
77    """
78    return self._results[results_type]
79
80  def get_packaged_results_of_type(self, results_type, reload_seconds=None,
81                                   is_editable=False, is_exported=True):
82    """Package the results of some/all tests as a complete response_dict.
83
84    Args:
85      results_type: string indicating which set of results to return;
86          must be one of the RESULTS_* constants
87      reload_seconds: if specified, note that new results may be available once
88          these results are reload_seconds old
89      is_editable: whether clients are allowed to submit new baselines
90      is_exported: whether these results are being made available to other
91          network hosts
92    """
93    response_dict = self._results[results_type]
94    time_updated = self.get_timestamp()
95    response_dict[imagepairset.KEY__ROOT__HEADER] = {
96        KEY__HEADER__SCHEMA_VERSION: (
97            VALUE__HEADER__SCHEMA_VERSION),
98
99        # Timestamps:
100        # 1. when this data was last updated
101        # 2. when the caller should check back for new data (if ever)
102        KEY__HEADER__TIME_UPDATED: time_updated,
103        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
104            (time_updated+reload_seconds) if reload_seconds else None),
105
106        # The type we passed to get_results_of_type()
107        KEY__HEADER__TYPE: results_type,
108
109        # Hash of dataset, which the client must return with any edits--
110        # this ensures that the edits were made to a particular dataset.
111        KEY__HEADER__DATAHASH: str(hash(repr(
112            response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))),
113
114        # Whether the server will accept edits back.
115        KEY__HEADER__IS_EDITABLE: is_editable,
116
117        # Whether the service is accessible from other hosts.
118        KEY__HEADER__IS_EXPORTED: is_exported,
119    }
120    return response_dict
121
122  def get_timestamp(self):
123    """Return the time at which this object was created, in seconds past epoch
124    (UTC).
125    """
126    return self._timestamp
127
128  _match_builders_pattern_list = [
129      re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
130  _skip_builders_pattern_list = [
131      re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]
132
133  def set_match_builders_pattern_list(self, pattern_list):
134    """Override the default set of builders we should process.
135
136    The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .
137
138    Note that skip_builders_pattern_list overrides this; regardless of whether a
139    builder is in the "match" list, if it's in the "skip" list, we will skip it.
140
141    Args:
142      pattern_list: list of regex patterns; process builders that match any
143          entry within this list
144    """
145    if pattern_list == None:
146      pattern_list = []
147    self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]
148
149  def set_skip_builders_pattern_list(self, pattern_list):
150    """Override the default set of builders we should skip while processing.
151
152    The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .
153
154    This overrides match_builders_pattern_list; regardless of whether a
155    builder is in the "match" list, if it's in the "skip" list, we will skip it.
156
157    Args:
158      pattern_list: list of regex patterns; skip builders that match any
159          entry within this list
160    """
161    if pattern_list == None:
162      pattern_list = []
163    self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]
164
165  def _ignore_builder(self, builder):
166    """Returns True if we should skip processing this builder.
167
168    Args:
169      builder: name of this builder, as a string
170
171    Returns:
172      True if we should ignore expectations and actuals for this builder.
173    """
174    for pattern in self._skip_builders_pattern_list:
175      if pattern.match(builder):
176        return True
177    for pattern in self._match_builders_pattern_list:
178      if pattern.match(builder):
179        return False
180    return True
181
182  def _read_builder_dicts_from_root(self, root, pattern='*.json'):
183    """Read all JSON dictionaries within a directory tree.
184
185    Skips any dictionaries belonging to a builder we have chosen to ignore.
186
187    Args:
188      root: path to root of directory tree
189      pattern: which files to read within root (fnmatch-style pattern)
190
191    Returns:
192      A meta-dictionary containing all the JSON dictionaries found within
193      the directory tree, keyed by builder name (the basename of the directory
194      where each JSON dictionary was found).
195
196    Raises:
197      IOError if root does not refer to an existing directory
198    """
199    # I considered making this call _read_dicts_from_root(), but I decided
200    # it was better to prune out the ignored builders within the os.walk().
201    if not os.path.isdir(root):
202      raise IOError('no directory found at path %s' % root)
203    meta_dict = {}
204    for dirpath, dirnames, filenames in os.walk(root):
205      for matching_filename in fnmatch.filter(filenames, pattern):
206        builder = os.path.basename(dirpath)
207        if self._ignore_builder(builder):
208          continue
209        full_path = os.path.join(dirpath, matching_filename)
210        meta_dict[builder] = gm_json.LoadFromFile(full_path)
211    return meta_dict
212
213  def _read_dicts_from_root(self, root, pattern='*.json'):
214    """Read all JSON dictionaries within a directory tree.
215
216    Args:
217      root: path to root of directory tree
218      pattern: which files to read within root (fnmatch-style pattern)
219
220    Returns:
221      A meta-dictionary containing all the JSON dictionaries found within
222      the directory tree, keyed by the pathname (relative to root) of each JSON
223      dictionary.
224
225    Raises:
226      IOError if root does not refer to an existing directory
227    """
228    if not os.path.isdir(root):
229      raise IOError('no directory found at path %s' % root)
230    meta_dict = {}
231    for abs_dirpath, dirnames, filenames in os.walk(root):
232      rel_dirpath = os.path.relpath(abs_dirpath, root)
233      for matching_filename in fnmatch.filter(filenames, pattern):
234        abs_path = os.path.join(abs_dirpath, matching_filename)
235        rel_path = os.path.join(rel_dirpath, matching_filename)
236        meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
237    return meta_dict
238
239  @staticmethod
240  def _read_noncomment_lines(path):
241    """Return a list of all noncomment lines within a file.
242
243    (A "noncomment" line is one that does not start with a '#'.)
244
245    Args:
246      path: path to file
247    """
248    lines = []
249    with open(path, 'r') as fh:
250      for line in fh:
251        if not line.startswith('#'):
252          lines.append(line.strip())
253    return lines
254
255  @staticmethod
256  def _create_relative_url(hashtype_and_digest, test_name):
257    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
258
259    If we don't have a record of this image, returns None.
260
261    Args:
262      hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
263          don't have a record of this image
264      test_name: string; name of the GM test that created this image
265    """
266    if not hashtype_and_digest:
267      return None
268    return gm_json.CreateGmRelativeUrl(
269        test_name=test_name,
270        hash_type=hashtype_and_digest[0],
271        hash_digest=hashtype_and_digest[1])
272
273  @staticmethod
274  def combine_subdicts(input_dict):
275    """ Flatten out a dictionary structure by one level.
276
277    Input:
278      {
279        KEY_A1 : {
280          KEY_B1 : VALUE_B1,
281        },
282        KEY_A2 : {
283          KEY_B2 : VALUE_B2,
284        }
285      }
286
287    Output:
288      {
289        KEY_B1 : VALUE_B1,
290        KEY_B2 : VALUE_B2,
291      }
292
293    If this would result in any repeated keys, it will raise an Exception.
294    """
295    output_dict = {}
296    for key, subdict in input_dict.iteritems():
297      for subdict_key, subdict_value in subdict.iteritems():
298        if subdict_key in output_dict:
299          raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
300        output_dict[subdict_key] = subdict_value
301    return output_dict
302
303  @staticmethod
304  def get_multilevel(input_dict, *keys):
305    """ Returns input_dict[key1][key2][...], or None if any key is not found.
306    """
307    for key in keys:
308      if input_dict == None:
309        return None
310      input_dict = input_dict.get(key, None)
311    return input_dict
312