1#!/usr/bin/python
2
3"""
4Copyright 2013 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Repackage expected/actual GM results as needed by our HTML rebaseline viewer.
10"""
11
12# System-level imports
13import fnmatch
14import os
15import re
16
17# Must fix up PYTHONPATH before importing from within Skia
18import rs_fixpypath  # pylint: disable=W0611
19
20# Imports from within Skia
21import gm_json
22import imagepairset
23
24# Keys used to link an image to a particular GM test.
25# NOTE: Keep these in sync with static/constants.js
26VALUE__HEADER__SCHEMA_VERSION = 5
27KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS
28KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE
29KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED
30KEY__EXTRACOLUMNS__BUILDER = 'builder'
31KEY__EXTRACOLUMNS__CONFIG = 'config'
32KEY__EXTRACOLUMNS__RESULT_TYPE = 'resultType'
33KEY__EXTRACOLUMNS__TEST = 'test'
34KEY__HEADER__DATAHASH = 'dataHash'
35KEY__HEADER__IS_EDITABLE = 'isEditable'
36KEY__HEADER__IS_EXPORTED = 'isExported'
37KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading'
38KEY__HEADER__RESULTS_ALL = 'all'
39KEY__HEADER__RESULTS_FAILURES = 'failures'
40KEY__HEADER__SCHEMA_VERSION = 'schemaVersion'
41KEY__HEADER__SET_A_DESCRIPTIONS = 'setA'
42KEY__HEADER__SET_B_DESCRIPTIONS = 'setB'
43KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable'
44KEY__HEADER__TIME_UPDATED = 'timeUpdated'
45KEY__HEADER__TYPE = 'type'
46KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED
47KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED
48KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON
49KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED
50KEY__SET_DESCRIPTIONS__DIR = 'dir'
51KEY__SET_DESCRIPTIONS__REPO_REVISION = 'repoRevision'
52KEY__SET_DESCRIPTIONS__SECTION = 'section'
53
54IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
55IMAGE_FILENAME_FORMATTER = '%s_%s.png'  # pass in (testname, config)
56
57PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
58DEFAULT_ACTUALS_DIR = '.gm-actuals'
59DEFAULT_GENERATED_IMAGES_ROOT = os.path.join(
60    PARENT_DIRECTORY, '.generated-images')
61
62# Define the default set of builders we will process expectations/actuals for.
63# This allows us to ignore builders for which we don't maintain expectations
64# (trybots, Valgrind, ASAN, TSAN), and avoid problems like
65# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server
66# produces error when trying to add baselines for ASAN/TSAN builders')
67DEFAULT_MATCH_BUILDERS_PATTERN_LIST = ['.*']
68DEFAULT_SKIP_BUILDERS_PATTERN_LIST = [
69    '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']
70
71
72class BaseComparisons(object):
73  """Base class for generating summary of comparisons between two image sets.
74  """
75
76  def __init__(self):
77    """Base constructor; most subclasses will override."""
78    self._setA_descriptions = None
79    self._setB_descriptions = None
80
81  def get_results_of_type(self, results_type):
82    """Return results of some/all tests (depending on 'results_type' parameter).
83
84    Args:
85      results_type: string describing which types of results to include; must
86          be one of the RESULTS_* constants
87
88    Results are returned in a dictionary as output by ImagePairSet.as_dict().
89    """
90    return self._results[results_type]
91
92  def get_packaged_results_of_type(self, results_type, reload_seconds=None,
93                                   is_editable=False, is_exported=True):
94    """Package the results of some/all tests as a complete response_dict.
95
96    Args:
97      results_type: string indicating which set of results to return;
98          must be one of the RESULTS_* constants
99      reload_seconds: if specified, note that new results may be available once
100          these results are reload_seconds old
101      is_editable: whether clients are allowed to submit new baselines
102      is_exported: whether these results are being made available to other
103          network hosts
104    """
105    response_dict = self._results[results_type]
106    time_updated = self.get_timestamp()
107    header_dict = {
108        KEY__HEADER__SCHEMA_VERSION: (
109            VALUE__HEADER__SCHEMA_VERSION),
110
111        # Timestamps:
112        # 1. when this data was last updated
113        # 2. when the caller should check back for new data (if ever)
114        KEY__HEADER__TIME_UPDATED: time_updated,
115        KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: (
116            (time_updated+reload_seconds) if reload_seconds else None),
117
118        # The type we passed to get_results_of_type()
119        KEY__HEADER__TYPE: results_type,
120
121        # Hash of dataset, which the client must return with any edits--
122        # this ensures that the edits were made to a particular dataset.
123        KEY__HEADER__DATAHASH: str(hash(repr(
124            response_dict[imagepairset.KEY__ROOT__IMAGEPAIRS]))),
125
126        # Whether the server will accept edits back.
127        KEY__HEADER__IS_EDITABLE: is_editable,
128
129        # Whether the service is accessible from other hosts.
130        KEY__HEADER__IS_EXPORTED: is_exported,
131    }
132    if self._setA_descriptions:
133      header_dict[KEY__HEADER__SET_A_DESCRIPTIONS] = self._setA_descriptions
134    if self._setB_descriptions:
135      header_dict[KEY__HEADER__SET_B_DESCRIPTIONS] = self._setB_descriptions
136    response_dict[imagepairset.KEY__ROOT__HEADER] = header_dict
137    return response_dict
138
139  def get_timestamp(self):
140    """Return the time at which this object was created, in seconds past epoch
141    (UTC).
142    """
143    return self._timestamp
144
145  _match_builders_pattern_list = [
146      re.compile(p) for p in DEFAULT_MATCH_BUILDERS_PATTERN_LIST]
147  _skip_builders_pattern_list = [
148      re.compile(p) for p in DEFAULT_SKIP_BUILDERS_PATTERN_LIST]
149
150  def set_match_builders_pattern_list(self, pattern_list):
151    """Override the default set of builders we should process.
152
153    The default is DEFAULT_MATCH_BUILDERS_PATTERN_LIST .
154
155    Note that skip_builders_pattern_list overrides this; regardless of whether a
156    builder is in the "match" list, if it's in the "skip" list, we will skip it.
157
158    Args:
159      pattern_list: list of regex patterns; process builders that match any
160          entry within this list
161    """
162    if pattern_list == None:
163      pattern_list = []
164    self._match_builders_pattern_list = [re.compile(p) for p in pattern_list]
165
166  def set_skip_builders_pattern_list(self, pattern_list):
167    """Override the default set of builders we should skip while processing.
168
169    The default is DEFAULT_SKIP_BUILDERS_PATTERN_LIST .
170
171    This overrides match_builders_pattern_list; regardless of whether a
172    builder is in the "match" list, if it's in the "skip" list, we will skip it.
173
174    Args:
175      pattern_list: list of regex patterns; skip builders that match any
176          entry within this list
177    """
178    if pattern_list == None:
179      pattern_list = []
180    self._skip_builders_pattern_list = [re.compile(p) for p in pattern_list]
181
182  def _ignore_builder(self, builder):
183    """Returns True if we should skip processing this builder.
184
185    Args:
186      builder: name of this builder, as a string
187
188    Returns:
189      True if we should ignore expectations and actuals for this builder.
190    """
191    for pattern in self._skip_builders_pattern_list:
192      if pattern.match(builder):
193        return True
194    for pattern in self._match_builders_pattern_list:
195      if pattern.match(builder):
196        return False
197    return True
198
199  def _read_builder_dicts_from_root(self, root, pattern='*.json'):
200    """Read all JSON dictionaries within a directory tree.
201
202    Skips any dictionaries belonging to a builder we have chosen to ignore.
203
204    Args:
205      root: path to root of directory tree
206      pattern: which files to read within root (fnmatch-style pattern)
207
208    Returns:
209      A meta-dictionary containing all the JSON dictionaries found within
210      the directory tree, keyed by builder name (the basename of the directory
211      where each JSON dictionary was found).
212
213    Raises:
214      IOError if root does not refer to an existing directory
215    """
216    # I considered making this call read_dicts_from_root(), but I decided
217    # it was better to prune out the ignored builders within the os.walk().
218    if not os.path.isdir(root):
219      raise IOError('no directory found at path %s' % root)
220    meta_dict = {}
221    for dirpath, _, filenames in os.walk(root):
222      for matching_filename in fnmatch.filter(filenames, pattern):
223        builder = os.path.basename(dirpath)
224        if self._ignore_builder(builder):
225          continue
226        full_path = os.path.join(dirpath, matching_filename)
227        meta_dict[builder] = gm_json.LoadFromFile(full_path)
228    return meta_dict
229
230  @staticmethod
231  def read_dicts_from_root(root, pattern='*.json'):
232    """Read all JSON dictionaries within a directory tree.
233
234    TODO(stephana): Factor this out into a utility module, as a standalone
235    function (not part of a class).
236
237    Args:
238      root: path to root of directory tree
239      pattern: which files to read within root (fnmatch-style pattern)
240
241    Returns:
242      A meta-dictionary containing all the JSON dictionaries found within
243      the directory tree, keyed by the pathname (relative to root) of each JSON
244      dictionary.
245
246    Raises:
247      IOError if root does not refer to an existing directory
248    """
249    if not os.path.isdir(root):
250      raise IOError('no directory found at path %s' % root)
251    meta_dict = {}
252    for abs_dirpath, _, filenames in os.walk(root):
253      rel_dirpath = os.path.relpath(abs_dirpath, root)
254      for matching_filename in fnmatch.filter(filenames, pattern):
255        abs_path = os.path.join(abs_dirpath, matching_filename)
256        rel_path = os.path.join(rel_dirpath, matching_filename)
257        meta_dict[rel_path] = gm_json.LoadFromFile(abs_path)
258    return meta_dict
259
260  @staticmethod
261  def _read_noncomment_lines(path):
262    """Return a list of all noncomment lines within a file.
263
264    (A "noncomment" line is one that does not start with a '#'.)
265
266    Args:
267      path: path to file
268    """
269    lines = []
270    with open(path, 'r') as fh:
271      for line in fh:
272        if not line.startswith('#'):
273          lines.append(line.strip())
274    return lines
275
276  @staticmethod
277  def _create_relative_url(hashtype_and_digest, test_name):
278    """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL.
279
280    If we don't have a record of this image, returns None.
281
282    Args:
283      hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we
284          don't have a record of this image
285      test_name: string; name of the GM test that created this image
286    """
287    if not hashtype_and_digest:
288      return None
289    return gm_json.CreateGmRelativeUrl(
290        test_name=test_name,
291        hash_type=hashtype_and_digest[0],
292        hash_digest=hashtype_and_digest[1])
293
294  @staticmethod
295  def combine_subdicts(input_dict):
296    """ Flatten out a dictionary structure by one level.
297
298    Input:
299      {
300        KEY_A1 : {
301          KEY_B1 : VALUE_B1,
302        },
303        KEY_A2 : {
304          KEY_B2 : VALUE_B2,
305        }
306      }
307
308    Output:
309      {
310        KEY_B1 : VALUE_B1,
311        KEY_B2 : VALUE_B2,
312      }
313
314    If this would result in any repeated keys, it will raise an Exception.
315    """
316    output_dict = {}
317    for subdict in input_dict.values():
318      for subdict_key, subdict_value in subdict.iteritems():
319        if subdict_key in output_dict:
320          raise Exception('duplicate key %s in combine_subdicts' % subdict_key)
321        output_dict[subdict_key] = subdict_value
322    return output_dict
323
324  @staticmethod
325  def get_default(input_dict, default_value, *keys):
326    """Returns input_dict[key1][key2][...], or default_value.
327
328    If input_dict is None, or any key is missing along the way, this returns
329    default_value.
330
331    Args:
332      input_dict: dictionary to look within
333      key: key indicating which value to return from input_dict
334      default_value: value to return if input_dict is None or any key cannot
335          be found along the way
336    """
337    if input_dict == None:
338      return default_value
339    for key in keys:
340      input_dict = input_dict.get(key, None)
341      if input_dict == None:
342        return default_value
343    return input_dict
344