results.py revision 31d0b3d806a1aa86b7edaa442b3821f5d548e184
1#!/usr/bin/python 2 3""" 4Copyright 2013 Google Inc. 5 6Use of this source code is governed by a BSD-style license that can be 7found in the LICENSE file. 8 9Repackage expected/actual GM results as needed by our HTML rebaseline viewer. 10""" 11 12# System-level imports 13import fnmatch 14import os 15import re 16import sys 17 18# Imports from within Skia 19# 20# We need to add the 'gm' directory, so that we can import gm_json.py within 21# that directory. That script allows us to parse the actual-results.json file 22# written out by the GM tool. 23# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 24# so any dirs that are already in the PYTHONPATH will be preferred. 25PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) 26GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) 27if GM_DIRECTORY not in sys.path: 28 sys.path.append(GM_DIRECTORY) 29import gm_json 30import imagepairset 31 32# Keys used to link an image to a particular GM test. 33# NOTE: Keep these in sync with static/constants.js 34REBASELINE_SERVER_SCHEMA_VERSION_NUMBER = 2 35KEY__EXPECTATIONS__BUGS = gm_json.JSONKEY_EXPECTEDRESULTS_BUGS 36KEY__EXPECTATIONS__IGNOREFAILURE = gm_json.JSONKEY_EXPECTEDRESULTS_IGNOREFAILURE 37KEY__EXPECTATIONS__REVIEWED = gm_json.JSONKEY_EXPECTEDRESULTS_REVIEWED 38KEY__EXTRACOLUMN__BUILDER = 'builder' 39KEY__EXTRACOLUMN__CONFIG = 'config' 40KEY__EXTRACOLUMN__RESULT_TYPE = 'resultType' 41KEY__EXTRACOLUMN__TEST = 'test' 42KEY__HEADER = 'header' 43KEY__HEADER__DATAHASH = 'dataHash' 44KEY__HEADER__IS_EDITABLE = 'isEditable' 45KEY__HEADER__IS_EXPORTED = 'isExported' 46KEY__HEADER__IS_STILL_LOADING = 'resultsStillLoading' 47KEY__HEADER__RESULTS_ALL = 'all' 48KEY__HEADER__RESULTS_FAILURES = 'failures' 49KEY__HEADER__SCHEMA_VERSION = 'schemaVersion' 50KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE = 'timeNextUpdateAvailable' 51KEY__HEADER__TIME_UPDATED = 'timeUpdated' 52KEY__HEADER__TYPE = 'type' 53KEY__NEW_IMAGE_URL = 'newImageUrl' 54KEY__RESULT_TYPE__FAILED = gm_json.JSONKEY_ACTUALRESULTS_FAILED 55KEY__RESULT_TYPE__FAILUREIGNORED = gm_json.JSONKEY_ACTUALRESULTS_FAILUREIGNORED 56KEY__RESULT_TYPE__NOCOMPARISON = gm_json.JSONKEY_ACTUALRESULTS_NOCOMPARISON 57KEY__RESULT_TYPE__SUCCEEDED = gm_json.JSONKEY_ACTUALRESULTS_SUCCEEDED 58 59IMAGE_FILENAME_RE = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 60IMAGE_FILENAME_FORMATTER = '%s_%s.png' # pass in (testname, config) 61 62# Ignore expectations/actuals for builders matching any of these patterns. 63# This allows us to ignore builders for which we don't maintain expectations 64# (trybots, Valgrind, ASAN, TSAN), and avoid problems like 65# https://code.google.com/p/skia/issues/detail?id=2036 ('rebaseline_server 66# produces error when trying to add baselines for ASAN/TSAN builders') 67SKIP_BUILDERS_PATTERN_LIST = [re.compile(p) for p in [ 68 '.*-Trybot', '.*Valgrind.*', '.*TSAN.*', '.*ASAN.*']] 69 70DEFAULT_ACTUALS_DIR = '.gm-actuals' 71DEFAULT_GENERATED_IMAGES_ROOT = os.path.join( 72 PARENT_DIRECTORY, '.generated-images') 73 74 75class BaseComparisons(object): 76 """Base class for generating summary of comparisons between two image sets. 77 """ 78 79 def __init__(self): 80 raise NotImplementedError('cannot instantiate the abstract base class') 81 82 def get_results_of_type(self, results_type): 83 """Return results of some/all tests (depending on 'results_type' parameter). 84 85 Args: 86 results_type: string describing which types of results to include; must 87 be one of the RESULTS_* constants 88 89 Results are returned in a dictionary as output by ImagePairSet.as_dict(). 90 """ 91 return self._results[results_type] 92 93 def get_packaged_results_of_type(self, results_type, reload_seconds=None, 94 is_editable=False, is_exported=True): 95 """Package the results of some/all tests as a complete response_dict. 96 97 Args: 98 results_type: string indicating which set of results to return; 99 must be one of the RESULTS_* constants 100 reload_seconds: if specified, note that new results may be available once 101 these results are reload_seconds old 102 is_editable: whether clients are allowed to submit new baselines 103 is_exported: whether these results are being made available to other 104 network hosts 105 """ 106 response_dict = self._results[results_type] 107 time_updated = self.get_timestamp() 108 response_dict[KEY__HEADER] = { 109 KEY__HEADER__SCHEMA_VERSION: ( 110 REBASELINE_SERVER_SCHEMA_VERSION_NUMBER), 111 112 # Timestamps: 113 # 1. when this data was last updated 114 # 2. when the caller should check back for new data (if ever) 115 KEY__HEADER__TIME_UPDATED: time_updated, 116 KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( 117 (time_updated+reload_seconds) if reload_seconds else None), 118 119 # The type we passed to get_results_of_type() 120 KEY__HEADER__TYPE: results_type, 121 122 # Hash of dataset, which the client must return with any edits-- 123 # this ensures that the edits were made to a particular dataset. 124 KEY__HEADER__DATAHASH: str(hash(repr( 125 response_dict[imagepairset.KEY__IMAGEPAIRS]))), 126 127 # Whether the server will accept edits back. 128 KEY__HEADER__IS_EDITABLE: is_editable, 129 130 # Whether the service is accessible from other hosts. 131 KEY__HEADER__IS_EXPORTED: is_exported, 132 } 133 return response_dict 134 135 def get_timestamp(self): 136 """Return the time at which this object was created, in seconds past epoch 137 (UTC). 138 """ 139 return self._timestamp 140 141 @staticmethod 142 def _ignore_builder(builder): 143 """Returns True if this builder matches any of SKIP_BUILDERS_PATTERN_LIST. 144 145 Args: 146 builder: name of this builder, as a string 147 148 Returns: 149 True if we should ignore expectations and actuals for this builder. 150 """ 151 for pattern in SKIP_BUILDERS_PATTERN_LIST: 152 if pattern.match(builder): 153 return True 154 return False 155 156 @staticmethod 157 def _read_dicts_from_root(root, pattern='*.json'): 158 """Read all JSON dictionaries within a directory tree. 159 160 Args: 161 root: path to root of directory tree 162 pattern: which files to read within root (fnmatch-style pattern) 163 164 Returns: 165 A meta-dictionary containing all the JSON dictionaries found within 166 the directory tree, keyed by the builder name of each dictionary. 167 168 Raises: 169 IOError if root does not refer to an existing directory 170 """ 171 if not os.path.isdir(root): 172 raise IOError('no directory found at path %s' % root) 173 meta_dict = {} 174 for dirpath, dirnames, filenames in os.walk(root): 175 for matching_filename in fnmatch.filter(filenames, pattern): 176 builder = os.path.basename(dirpath) 177 if BaseComparisons._ignore_builder(builder): 178 continue 179 fullpath = os.path.join(dirpath, matching_filename) 180 meta_dict[builder] = gm_json.LoadFromFile(fullpath) 181 return meta_dict 182 183 @staticmethod 184 def _create_relative_url(hashtype_and_digest, test_name): 185 """Returns the URL for this image, relative to GM_ACTUALS_ROOT_HTTP_URL. 186 187 If we don't have a record of this image, returns None. 188 189 Args: 190 hashtype_and_digest: (hash_type, hash_digest) tuple, or None if we 191 don't have a record of this image 192 test_name: string; name of the GM test that created this image 193 """ 194 if not hashtype_and_digest: 195 return None 196 return gm_json.CreateGmRelativeUrl( 197 test_name=test_name, 198 hash_type=hashtype_and_digest[0], 199 hash_digest=hashtype_and_digest[1]) 200