1#!/usr/bin/python 2 3""" 4Copyright 2013 Google Inc. 5 6Use of this source code is governed by a BSD-style license that can be 7found in the LICENSE file. 8 9Repackage expected/actual GM results as needed by our HTML rebaseline viewer. 10""" 11 12# System-level imports 13import argparse 14import fnmatch 15import json 16import logging 17import os 18import re 19import sys 20import time 21 22# Imports from within Skia 23import fix_pythonpath # must do this first 24from pyutils import url_utils 25import gm_json 26import imagediffdb 27import imagepair 28import imagepairset 29import results 30 31EXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ 32 results.KEY__EXPECTATIONS__BUGS, 33 results.KEY__EXPECTATIONS__IGNOREFAILURE, 34 results.KEY__EXPECTATIONS__REVIEWED, 35] 36TRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 37DEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') 38DEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' 39 40IMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') 41 42 43class ExpectationComparisons(results.BaseComparisons): 44 """Loads actual and expected GM results into an ImagePairSet. 45 46 Loads actual and expected results from all builders, except for those skipped 47 by _ignore_builder(). 48 49 Once this object has been constructed, the results (in self._results[]) 50 are immutable. If you want to update the results based on updated JSON 51 file contents, you will need to create a new ExpectationComparisons object.""" 52 53 def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR, 54 expected_root=DEFAULT_EXPECTATIONS_DIR, 55 ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE, 56 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, 57 diff_base_url=None, builder_regex_list=None): 58 """ 59 Args: 60 actuals_root: root directory containing all actual-results.json files 61 expected_root: root directory containing all expected-results.json files 62 ignore_failures_file: if a file with this name is found within 63 expected_root, ignore failures for any tests listed in the file 64 generated_images_root: directory within which to create all pixel diffs; 65 if this directory does not yet exist, it will be created 66 diff_base_url: base URL within which the client should look for diff 67 images; if not specified, defaults to a "file:///" URL representation 68 of generated_images_root 69 builder_regex_list: List of regular expressions specifying which builders 70 we will process. If None, process all builders. 71 """ 72 time_start = int(time.time()) 73 if builder_regex_list != None: 74 self.set_match_builders_pattern_list(builder_regex_list) 75 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 76 self._diff_base_url = ( 77 diff_base_url or 78 url_utils.create_filepath_url(generated_images_root)) 79 self._actuals_root = actuals_root 80 self._expected_root = expected_root 81 self._ignore_failures_on_these_tests = [] 82 if ignore_failures_file: 83 self._ignore_failures_on_these_tests = ( 84 ExpectationComparisons._read_noncomment_lines( 85 os.path.join(expected_root, ignore_failures_file))) 86 self._load_actual_and_expected() 87 self._timestamp = int(time.time()) 88 logging.info('Results complete; took %d seconds.' % 89 (self._timestamp - time_start)) 90 91 def edit_expectations(self, modifications): 92 """Edit the expectations stored within this object and write them back 93 to disk. 94 95 Note that this will NOT update the results stored in self._results[] ; 96 in order to see those updates, you must instantiate a new 97 ExpectationComparisons object based on the (now updated) files on disk. 98 99 Args: 100 modifications: a list of dictionaries, one for each expectation to update: 101 102 [ 103 { 104 imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: { 105 results.KEY__EXPECTATIONS__BUGS: [123, 456], 106 results.KEY__EXPECTATIONS__IGNOREFAILURE: false, 107 results.KEY__EXPECTATIONS__REVIEWED: true, 108 }, 109 imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: { 110 results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', 111 results.KEY__EXTRACOLUMNS__CONFIG: '8888', 112 results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix', 113 }, 114 results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', 115 }, 116 ... 117 ] 118 119 """ 120 expected_builder_dicts = self._read_builder_dicts_from_root( 121 self._expected_root) 122 for mod in modifications: 123 image_name = results.IMAGE_FILENAME_FORMATTER % ( 124 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 125 [results.KEY__EXTRACOLUMNS__TEST], 126 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 127 [results.KEY__EXTRACOLUMNS__CONFIG]) 128 _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( 129 mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL]) 130 allowed_digests = [[hash_type, int(hash_digest)]] 131 new_expectations = { 132 gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, 133 } 134 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 135 value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field) 136 if value is not None: 137 new_expectations[field] = value 138 builder_dict = expected_builder_dicts[ 139 mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 140 [results.KEY__EXTRACOLUMNS__BUILDER]] 141 builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) 142 if not builder_expectations: 143 builder_expectations = {} 144 builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations 145 builder_expectations[image_name] = new_expectations 146 ExpectationComparisons._write_dicts_to_root( 147 expected_builder_dicts, self._expected_root) 148 149 @staticmethod 150 def _write_dicts_to_root(meta_dict, root, pattern='*.json'): 151 """Write all per-builder dictionaries within meta_dict to files under 152 the root path. 153 154 Security note: this will only write to files that already exist within 155 the root path (as found by os.walk() within root), so we don't need to 156 worry about malformed content writing to disk outside of root. 157 However, the data written to those files is not double-checked, so it 158 could contain poisonous data. 159 160 Args: 161 meta_dict: a builder-keyed meta-dictionary containing all the JSON 162 dictionaries we want to write out 163 root: path to root of directory tree within which to write files 164 pattern: which files to write within root (fnmatch-style pattern) 165 166 Raises: 167 IOError if root does not refer to an existing directory 168 KeyError if the set of per-builder dictionaries written out was 169 different than expected 170 """ 171 if not os.path.isdir(root): 172 raise IOError('no directory found at path %s' % root) 173 actual_builders_written = [] 174 for dirpath, dirnames, filenames in os.walk(root): 175 for matching_filename in fnmatch.filter(filenames, pattern): 176 builder = os.path.basename(dirpath) 177 per_builder_dict = meta_dict.get(builder) 178 if per_builder_dict is not None: 179 fullpath = os.path.join(dirpath, matching_filename) 180 gm_json.WriteToFile(per_builder_dict, fullpath) 181 actual_builders_written.append(builder) 182 183 # Check: did we write out the set of per-builder dictionaries we 184 # expected to? 185 expected_builders_written = sorted(meta_dict.keys()) 186 actual_builders_written.sort() 187 if expected_builders_written != actual_builders_written: 188 raise KeyError( 189 'expected to write dicts for builders %s, but actually wrote them ' 190 'for builders %s' % ( 191 expected_builders_written, actual_builders_written)) 192 193 def _load_actual_and_expected(self): 194 """Loads the results of all tests, across all builders (based on the 195 files within self._actuals_root and self._expected_root), 196 and stores them in self._results. 197 """ 198 logging.info('Reading actual-results JSON files from %s...' % 199 self._actuals_root) 200 actual_builder_dicts = self._read_builder_dicts_from_root( 201 self._actuals_root) 202 logging.info('Reading expected-results JSON files from %s...' % 203 self._expected_root) 204 expected_builder_dicts = self._read_builder_dicts_from_root( 205 self._expected_root) 206 207 all_image_pairs = imagepairset.ImagePairSet( 208 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 209 diff_base_url=self._diff_base_url) 210 failing_image_pairs = imagepairset.ImagePairSet( 211 descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 212 diff_base_url=self._diff_base_url) 213 214 all_image_pairs.ensure_extra_column_values_in_summary( 215 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 216 results.KEY__RESULT_TYPE__FAILED, 217 results.KEY__RESULT_TYPE__FAILUREIGNORED, 218 results.KEY__RESULT_TYPE__NOCOMPARISON, 219 results.KEY__RESULT_TYPE__SUCCEEDED, 220 ]) 221 failing_image_pairs.ensure_extra_column_values_in_summary( 222 column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 223 results.KEY__RESULT_TYPE__FAILED, 224 results.KEY__RESULT_TYPE__FAILUREIGNORED, 225 results.KEY__RESULT_TYPE__NOCOMPARISON, 226 ]) 227 228 # Only consider builders we have both expected and actual results for. 229 # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results 230 # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug 231 # builder') 232 actual_builder_set = set(actual_builder_dicts.keys()) 233 expected_builder_set = set(expected_builder_dicts.keys()) 234 builders = sorted(actual_builder_set.intersection(expected_builder_set)) 235 236 num_builders = len(builders) 237 builder_num = 0 238 for builder in builders: 239 builder_num += 1 240 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 241 (builder_num, num_builders, builder)) 242 actual_results_for_this_builder = ( 243 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 244 for result_type in sorted(actual_results_for_this_builder.keys()): 245 results_of_this_type = actual_results_for_this_builder[result_type] 246 if not results_of_this_type: 247 continue 248 for image_name in sorted(results_of_this_type.keys()): 249 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 250 actual_image_relative_url = ( 251 ExpectationComparisons._create_relative_url( 252 hashtype_and_digest=results_of_this_type[image_name], 253 test_name=test)) 254 255 # Default empty expectations; overwrite these if we find any real ones 256 expectations_per_test = None 257 expected_image_relative_url = None 258 expectations_dict = None 259 try: 260 expectations_per_test = ( 261 expected_builder_dicts 262 [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) 263 # TODO(epoger): assumes a single allowed digest per test, which is 264 # fine; see https://code.google.com/p/skia/issues/detail?id=1787 265 expected_image_hashtype_and_digest = ( 266 expectations_per_test 267 [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) 268 expected_image_relative_url = ( 269 ExpectationComparisons._create_relative_url( 270 hashtype_and_digest=expected_image_hashtype_and_digest, 271 test_name=test)) 272 expectations_dict = {} 273 for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 274 expectations_dict[field] = expectations_per_test.get(field) 275 except (KeyError, TypeError): 276 # There are several cases in which we would expect to find 277 # no expectations for a given test: 278 # 279 # 1. result_type == NOCOMPARISON 280 # There are no expectations for this test yet! 281 # 282 # 2. alternate rendering mode failures (e.g. serialized) 283 # In cases like 284 # https://code.google.com/p/skia/issues/detail?id=1684 285 # ('tileimagefilter GM test failing in serialized render mode'), 286 # the gm-actuals will list a failure for the alternate 287 # rendering mode even though we don't have explicit expectations 288 # for the test (the implicit expectation is that it must 289 # render the same in all rendering modes). 290 # 291 # Don't log type 1, because it is common. 292 # Log other types, because they are rare and we should know about 293 # them, but don't throw an exception, because we need to keep our 294 # tools working in the meanwhile! 295 if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: 296 logging.warning('No expectations found for test: %s' % { 297 results.KEY__EXTRACOLUMNS__BUILDER: builder, 298 results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 299 'image_name': image_name, 300 }) 301 302 # If this test was recently rebaselined, it will remain in 303 # the 'failed' set of actuals until all the bots have 304 # cycled (although the expectations have indeed been set 305 # from the most recent actuals). Treat these as successes 306 # instead of failures. 307 # 308 # TODO(epoger): Do we need to do something similar in 309 # other cases, such as when we have recently marked a test 310 # as ignoreFailure but it still shows up in the 'failed' 311 # category? Maybe we should not rely on the result_type 312 # categories recorded within the gm_actuals AT ALL, and 313 # instead evaluate the result_type ourselves based on what 314 # we see in expectations vs actual checksum? 315 if expected_image_relative_url == actual_image_relative_url: 316 updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED 317 elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and 318 (test in self._ignore_failures_on_these_tests)): 319 updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED 320 else: 321 updated_result_type = result_type 322 extra_columns_dict = { 323 results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type, 324 results.KEY__EXTRACOLUMNS__BUILDER: builder, 325 results.KEY__EXTRACOLUMNS__TEST: test, 326 results.KEY__EXTRACOLUMNS__CONFIG: config, 327 } 328 try: 329 image_pair = imagepair.ImagePair( 330 image_diff_db=self._image_diff_db, 331 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 332 imageA_relative_url=expected_image_relative_url, 333 imageB_relative_url=actual_image_relative_url, 334 expectations=expectations_dict, 335 extra_columns=extra_columns_dict) 336 all_image_pairs.add_image_pair(image_pair) 337 if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 338 failing_image_pairs.add_image_pair(image_pair) 339 except Exception: 340 logging.exception('got exception while creating new ImagePair') 341 342 self._results = { 343 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), 344 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 345 } 346 347 348def main(): 349 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 350 datefmt='%m/%d/%Y %H:%M:%S', 351 level=logging.INFO) 352 parser = argparse.ArgumentParser() 353 parser.add_argument( 354 '--actuals', default=results.DEFAULT_ACTUALS_DIR, 355 help='Directory containing all actual-result JSON files; defaults to ' 356 '\'%(default)s\' .') 357 parser.add_argument( 358 '--expectations', default=DEFAULT_EXPECTATIONS_DIR, 359 help='Directory containing all expected-result JSON files; defaults to ' 360 '\'%(default)s\' .') 361 parser.add_argument( 362 '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE, 363 help='If a file with this name is found within the EXPECTATIONS dir, ' 364 'ignore failures for any tests listed in the file; defaults to ' 365 '\'%(default)s\' .') 366 parser.add_argument( 367 '--outfile', required=True, 368 help='File to write result summary into, in JSON format.') 369 parser.add_argument( 370 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 371 help='Which result types to include. Defaults to \'%(default)s\'; ' 372 'must be one of ' + 373 str([results.KEY__HEADER__RESULTS_FAILURES, 374 results.KEY__HEADER__RESULTS_ALL])) 375 parser.add_argument( 376 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, 377 help='Directory within which to download images and generate diffs; ' 378 'defaults to \'%(default)s\' .') 379 args = parser.parse_args() 380 results_obj = ExpectationComparisons( 381 actuals_root=args.actuals, 382 expected_root=args.expectations, 383 ignore_failures_file=args.ignore_failures_file, 384 generated_images_root=args.workdir) 385 gm_json.WriteToFile( 386 results_obj.get_packaged_results_of_type(results_type=args.results), 387 args.outfile) 388 389 390if __name__ == '__main__': 391 main() 392