results.py revision 3eb77e4d5a381fa55197f6bd03c599e709146069 
1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
10"""
11
12# System-level imports
13import fnmatch
14import os
15import re
16import sys
17
18# Imports from within Skia
19#
20# We need to add the 'gm' directory, so that we can import gm_json.py within
21# that directory.  That script allows us to parse the actual-results.json file
22# written out by the GM tool.
23# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
24# so any dirs that are already in the PYTHONPATH will be preferred.
25PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
26GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
27if GM_DIRECTORY not in sys.path:
28  sys.path.append(GM_DIRECTORY)
29import gm_json
30import imagepairset
31
32# Keys used to link an image to a particular GM test.
33# NOTE: Keep these in sync with static/constants.js
34REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2
35KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
36KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
37KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
38KEY__EXTRACOLUMN__BUILDER = 'builder'
39KEY__EXTRACOLUMN__CONFIG = 'config'
40KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType'
41KEY__EXTRACOLUMN__TEST = 'test'
42KEY__HEADER = 'header'
43KEY__HEADER__DATAHASH = 'dataHash'
44KEY__HEADER__IS_EDITABLE = 'isEditable'
45KEY__HEADER__IS_EXPORTED = 'isExported'
46KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
47KEY__HEADER__RESULTS_ALL = 'all'
48KEY__HEADER__RESULTS_FAILURES = 'failures'
49KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
50KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
51KEY__HEADER__TIME_UPDATED = 'timeUpdated'
52KEY__HEADER__TYPE = 'type'
53KEY__NEW_IMAGE_URL = 'newImageUrl'
54KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
55KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
56KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
57KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
58
59IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
60IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
61
62# Ignore expectations/actuals for builders matching any of these patterns.
63# This allows us to ignore builders for which we don't maintain expectations
64# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
65# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
66# produces error when trying to add baselines for ASAN/TSAN builders')
67SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [
68    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']]
69
70DEFAULT_ACTUALS_DIR = '.gm-actuals'
71DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
72    PARENT_DIRECTORY, '.generated-images')
73
74
75class BaseComparisons(object):
76  """Base class for generating summary of comparisons between two image sets.
77  """
78
79  def __init__(self):
80    raise NotImplementedError('cannot instantiate the abstract base class')
81
82  def get_results_of_type(self, results_type):
83    """Return results of some/all tests (depending on 'results_type' parameter).
84
85    Args:
86      results_type: string describing which types of results to include; must
87          be one of the RESULTS_* constants
88
89    Results are returned in a dictionary as output by ImagePairSet.as_dict().
90    """
91    return self._results[results_type]
92
93  def get_packaged_results_of_type(self, results_type, reload_seconds=None,
94                                   is_editable=False, is_exported=True):
95    """Package the results of some/all tests as a complete response_dict.
96
97    Args:
98      results_type: string indicating which set of results to return;
99          must be one of the RESULTS_* constants
100      reload_seconds: if specified, note that new results may be available once
101          these results are reload_seconds old
102      is_editable: whether clients are allowed to submit new baselines
103      is_exported: whether these results are being made available to other
104          network hosts
105    """
106    response_dict = self._results[results_type]
107    time_updated = self.get_timestamp()
108    response_dict[KEY__HEADER] = {
109        KEY__HEADER__SCHEMA_VERSION: (
110            REBASELINE_SERVER_SCHEMA_VERSION_NUMBER),
111
112        # Timestamps:
113        # 1. when this data was last updated
114        # 2. when the caller should check back for new data (if ever)
115        KEY__HEADER__TIME_UPDATED: time_updated,
116        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
117            (time_updated+reload_seconds) if reload_seconds else None),
118
119        # The type we passed to get_results_of_type()
120        KEY__HEADER__TYPE: results_type,
121
122        # Hash of dataset, which the client must return with any edits--
123        # this ensures that the edits were made to a particular dataset.
124        KEY__HEADER__DATAHASH: str(hash(repr(
125            response_dict[imagepairset.KEY__IMAGEPAIRS]))),
126
127        # Whether the server will accept edits back.
128        KEY__HEADER__IS_EDITABLE: is_editable,
129
130        # Whether the service is accessible from other hosts.
131        KEY__HEADER__IS_EXPORTED: is_exported,
132    }
133    return response_dict
134
135  def get_timestamp(self):
136    """Return the time at which this object was created, in seconds past epoch
137    (UTC).
138    """
139    return self._timestamp
140
141  @staticmethod
142  def _ignore_builder(builder):
143    """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST.
144
145    Args:
146      builder: name of this builder, as a string
147
148    Returns:
149      True if we should ignore expectations and actuals for this builder.
150    """
151    for pattern in SKIP_BUILDERS_PATTERN_LIST:
152      if pattern.match(builder):
153        return True
154    return False
155
156  @staticmethod
157  def _read_dicts_from_root(root, pattern='*.json'):
158    """Read all JSON dictionaries within a directory tree.
159
160    Args:
161      root: path to root of directory tree
162      pattern: which files to read within root (fnmatch-style pattern)
163
164    Returns:
165      A meta-dictionary containing all the JSON dictionaries found within
166      the directory tree, keyed by the builder name of each dictionary.
167
168    Raises:
169      IOError if root does not refer to an existing directory
170    """
171    if not os.path.isdir(root):
172      raise IOError('no directory found at path %s' % root)
173    meta_dict = {}
174    for dirpath, dirnames, filenames in os.walk(root):
175      for matching_filename in fnmatch.filter(filenames, pattern):
176        builder = os.path.basename(dirpath)
177        if BaseComparisons._ignore_builder(builder):
178          continue
179        fullpath = os.path.join(dirpath, matching_filename)
180        meta_dict[builder] = gm_json.LoadFromFile(fullpath)
181    return meta_dict
182
183  @staticmethod
184  def _create_relative_url(hashtype_and_digest, test_name):
185    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
186
187    If we don't have a record of this image, returns None.
188
189    Args:
190      hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
191          don't have a record of this image
192      test_name: string; name of the GM test that created this image
193    """
194    if not hashtype_and_digest:
195      return None
196    return gm_json.CreateGmRelativeUrl(
197        test_name=test_name,
198        hash_type=hashtype_and_digest[0],
199        hash_digest=hashtype_and_digest[1])
200
201  @staticmethod
202  def combine_subdicts(input_dict):
203    """ Flatten out a dictionary structure by one level.
204
205    Input:
206      {
207        "failed" : {
208          "changed.png" : [ "bitmap-64bitMD5", 8891695120562235492 ],
209        },
210        "no-comparison" : {
211          "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ],
212        }
213      }
214
215    Output:
216      {
217        "changed.png" : [ "bitmap-64bitMD5", 8891695120562235492 ],
218        "unchanged.png" : [ "bitmap-64bitMD5", 11092453015575919668 ],
219      }
220
221    If this would result in any repeated keys, it will raise an Exception.
222    """
223    output_dict = {}
224    for key, subdict in input_dict.iteritems():
225      for subdict_key, subdict_value in subdict.iteritems():
226        if subdict_key in output_dict:
227          raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
228        output_dict[subdict_key] = subdict_value
229    return output_dict
230