1b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org#!/usr/bin/python 2b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 3b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org""" 4b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgCopyright 2013 Google Inc. 5b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 6b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgUse of this source code is governed by a BSD-style license that can be 7b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgfound in the LICENSE file. 8b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 9b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgRepackage expected/actual GM results as needed by our HTML rebaseline viewer. 10b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org""" 11b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 12b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org# System-level imports 13b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport argparse 14b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport fnmatch 15b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport json 16b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport logging 17b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport os 18b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport re 19b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport sys 20b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport time 21b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 22b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org# Imports from within Skia 23b144271179aaf82cb1151e9dfd8e866747402594epogerimport fix_pythonpath # must do this first 24b144271179aaf82cb1151e9dfd8e866747402594epogerfrom pyutils import url_utils 25b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport gm_json 26b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagediffdb 27b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagepair 28b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagepairset 29b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport results 30b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 31b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgEXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ 32b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__BUGS, 33b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__IGNOREFAILURE, 34b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__REVIEWED, 35b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org] 36b144271179aaf82cb1151e9dfd8e866747402594epogerTRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 3731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.orgDEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') 384cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.orgDEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' 39b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 40b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgIMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') 41b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 42b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 4331d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.orgclass ExpectationComparisons(results.BaseComparisons): 4431d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org """Loads actual and expected GM results into an ImagePairSet. 45b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 46b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Loads actual and expected results from all builders, except for those skipped 47b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org by _ignore_builder(). 48b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 49b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Once this object has been constructed, the results (in self._results[]) 50b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org are immutable. If you want to update the results based on updated JSON 5131d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org file contents, you will need to create a new ExpectationComparisons object.""" 52b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 5331d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org def __init__(self, actuals_root=results.DEFAULT_ACTUALS_DIR, 54b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_root=DEFAULT_EXPECTATIONS_DIR, 554cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE, 5631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, 57defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org diff_base_url=None, builder_regex_list=None): 58b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 59b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 60b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actuals_root: root directory containing all actual-results.json files 61b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_root: root directory containing all expected-results.json files 624cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ignore_failures_file: if a file with this name is found within 634cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org expected_root, ignore failures for any tests listed in the file 64b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org generated_images_root: directory within which to create all pixel diffs; 65b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if this directory does not yet exist, it will be created 66b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url: base URL within which the client should look for diff 67b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org images; if not specified, defaults to a "file:///" URL representation 68b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org of generated_images_root 69defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org builder_regex_list: List of regular expressions specifying which builders 70defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org we will process. If None, process all builders. 71b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 72b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org time_start = int(time.time()) 73defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org if builder_regex_list != None: 74defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org self.set_match_builders_pattern_list(builder_regex_list) 75b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 76b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._diff_base_url = ( 77b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url or 78b144271179aaf82cb1151e9dfd8e866747402594epoger url_utils.create_filepath_url(generated_images_root)) 79b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._actuals_root = actuals_root 80b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._expected_root = expected_root 814cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org self._ignore_failures_on_these_tests = [] 824cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org if ignore_failures_file: 834cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org self._ignore_failures_on_these_tests = ( 844cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ExpectationComparisons._read_noncomment_lines( 854cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org os.path.join(expected_root, ignore_failures_file))) 86b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._load_actual_and_expected() 87b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._timestamp = int(time.time()) 88b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Results complete; took %d seconds.' % 89b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (self._timestamp - time_start)) 90b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 91b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def edit_expectations(self, modifications): 92b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Edit the expectations stored within this object and write them back 93b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org to disk. 94b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 95b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Note that this will NOT update the results stored in self._results[] ; 9631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org in order to see those updates, you must instantiate a new 9731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons object based on the (now updated) files on disk. 98b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 99b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 100b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org modifications: a list of dictionaries, one for each expectation to update: 101b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 102b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [ 103b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org { 10468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: { 105b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__BUGS: [123, 456], 106b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__IGNOREFAILURE: false, 107b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__REVIEWED: true, 108b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 10968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: { 11068a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', 11168a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__CONFIG: '8888', 11268a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix', 113b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 11468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', 115b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 116b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ... 117b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ] 118b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 119b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 1207418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org expected_builder_dicts = self._read_builder_dicts_from_root( 1217418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._expected_root) 122b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for mod in modifications: 123b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_name = results.IMAGE_FILENAME_FORMATTER % ( 12468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 12568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__TEST], 12668a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 12768a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__CONFIG]) 128b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( 12968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL]) 130b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org allowed_digests = [[hash_type, int(hash_digest)]] 131b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org new_expectations = { 132b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, 133b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 134b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 13568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field) 136b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if value is not None: 137b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org new_expectations[field] = value 138b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_dict = expected_builder_dicts[ 13968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 14068a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__BUILDER]] 141b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) 142b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not builder_expectations: 143b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations = {} 144b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations 145b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations[image_name] = new_expectations 14631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._write_dicts_to_root( 14731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org expected_builder_dicts, self._expected_root) 148b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 149b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org @staticmethod 150b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def _write_dicts_to_root(meta_dict, root, pattern='*.json'): 151b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Write all per-builder dictionaries within meta_dict to files under 152b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org the root path. 153b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 154b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Security note: this will only write to files that already exist within 155b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org the root path (as found by os.walk() within root), so we don't need to 156b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org worry about malformed content writing to disk outside of root. 157b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org However, the data written to those files is not double-checked, so it 158b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org could contain poisonous data. 159b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 160b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 161b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org meta_dict: a builder-keyed meta-dictionary containing all the JSON 162b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org dictionaries we want to write out 163b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org root: path to root of directory tree within which to write files 164b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org pattern: which files to write within root (fnmatch-style pattern) 165b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 166b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Raises: 167b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org IOError if root does not refer to an existing directory 168b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org KeyError if the set of per-builder dictionaries written out was 169b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org different than expected 170b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 171b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not os.path.isdir(root): 172b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org raise IOError('no directory found at path %s' % root) 173b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written = [] 174b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for dirpath, dirnames, filenames in os.walk(root): 175b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for matching_filename in fnmatch.filter(filenames, pattern): 176b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder = os.path.basename(dirpath) 177b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org per_builder_dict = meta_dict.get(builder) 178b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if per_builder_dict is not None: 179b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org fullpath = os.path.join(dirpath, matching_filename) 180b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.WriteToFile(per_builder_dict, fullpath) 181b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written.append(builder) 182b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 183b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Check: did we write out the set of per-builder dictionaries we 184b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # expected to? 185b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builders_written = sorted(meta_dict.keys()) 186b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written.sort() 187b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if expected_builders_written != actual_builders_written: 188b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org raise KeyError( 189b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'expected to write dicts for builders %s, but actually wrote them ' 190b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'for builders %s' % ( 191b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builders_written, actual_builders_written)) 192b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 193b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def _load_actual_and_expected(self): 194b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Loads the results of all tests, across all builders (based on the 195b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org files within self._actuals_root and self._expected_root), 196b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org and stores them in self._results. 197b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 198b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Reading actual-results JSON files from %s...' % 199b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._actuals_root) 2007418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org actual_builder_dicts = self._read_builder_dicts_from_root( 2017418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._actuals_root) 202b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Reading expected-results JSON files from %s...' % 203b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._expected_root) 2047418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org expected_builder_dicts = self._read_builder_dicts_from_root( 2057418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._expected_root) 206b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 207b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs = imagepairset.ImagePairSet( 208b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 209b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url=self._diff_base_url) 210b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs = imagepairset.ImagePairSet( 211b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 212b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url=self._diff_base_url) 213b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 214b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs.ensure_extra_column_values_in_summary( 21568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 216b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILED, 217b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILUREIGNORED, 218b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__NOCOMPARISON, 219b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__SUCCEEDED, 220b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ]) 221b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs.ensure_extra_column_values_in_summary( 22268a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 223b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILED, 224b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILUREIGNORED, 225b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__NOCOMPARISON, 226b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ]) 227b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 2286323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # Only consider builders we have both expected and actual results for. 2296323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results 2306323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug 2316323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # builder') 2326323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org actual_builder_set = set(actual_builder_dicts.keys()) 2336323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org expected_builder_set = set(expected_builder_dicts.keys()) 2346323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org builders = sorted(actual_builder_set.intersection(expected_builder_set)) 2356323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org 236b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org num_builders = len(builders) 237b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_num = 0 238b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for builder in builders: 239b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_num += 1 240b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 241b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (builder_num, num_builders, builder)) 242b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_results_for_this_builder = ( 243b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 244b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for result_type in sorted(actual_results_for_this_builder.keys()): 245b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results_of_this_type = actual_results_for_this_builder[result_type] 246b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not results_of_this_type: 247b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org continue 248b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for image_name in sorted(results_of_this_type.keys()): 249b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 25031d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org actual_image_relative_url = ( 25131d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._create_relative_url( 25231d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org hashtype_and_digest=results_of_this_type[image_name], 25331d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org test_name=test)) 254b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 255b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Default empty expectations; overwrite these if we find any real ones 256b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test = None 257b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_image_relative_url = None 258b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict = None 259b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org try: 260b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test = ( 261b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builder_dicts 262b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) 263b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # TODO(epoger): assumes a single allowed digest per test, which is 264b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # fine; see https://code.google.com/p/skia/issues/detail?id=1787 265b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_image_hashtype_and_digest = ( 266b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test 267b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) 26831d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org expected_image_relative_url = ( 26931d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._create_relative_url( 27031d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org hashtype_and_digest=expected_image_hashtype_and_digest, 27131d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org test_name=test)) 272b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict = {} 273b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 274b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict[field] = expectations_per_test.get(field) 275b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org except (KeyError, TypeError): 276b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # There are several cases in which we would expect to find 277b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # no expectations for a given test: 278b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 279b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 1. result_type == NOCOMPARISON 280b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # There are no expectations for this test yet! 281b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 282b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 2. alternate rendering mode failures (e.g. serialized) 283b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # In cases like 284b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # https://code.google.com/p/skia/issues/detail?id=1684 285b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # ('tileimagefilter GM test failing in serialized render mode'), 286b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # the gm-actuals will list a failure for the alternate 287b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # rendering mode even though we don't have explicit expectations 288b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # for the test (the implicit expectation is that it must 289b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # render the same in all rendering modes). 290b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 291b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Don't log type 1, because it is common. 292b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Log other types, because they are rare and we should know about 293b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # them, but don't throw an exception, because we need to keep our 294b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # tools working in the meanwhile! 295b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: 296b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.warning('No expectations found for test: %s' % { 29768a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: builder, 29868a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 299b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'image_name': image_name, 300b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }) 301b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 302b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # If this test was recently rebaselined, it will remain in 303b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # the 'failed' set of actuals until all the bots have 304b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # cycled (although the expectations have indeed been set 305b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # from the most recent actuals). Treat these as successes 306b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # instead of failures. 307b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 308b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # TODO(epoger): Do we need to do something similar in 309b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # other cases, such as when we have recently marked a test 310b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # as ignoreFailure but it still shows up in the 'failed' 311b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # category? Maybe we should not rely on the result_type 312b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # categories recorded within the gm_actuals AT ALL, and 313b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # instead evaluate the result_type ourselves based on what 314b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # we see in expectations vs actual checksum? 315b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if expected_image_relative_url == actual_image_relative_url: 316b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED 3174cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and 3184cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org (test in self._ignore_failures_on_these_tests)): 3194cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED 320b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org else: 321b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org updated_result_type = result_type 322b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org extra_columns_dict = { 32368a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type, 32468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: builder, 32568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__TEST: test, 32668a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__CONFIG: config, 327b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 328b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org try: 329b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_pair = imagepair.ImagePair( 330b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_diff_db=self._image_diff_db, 331b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 332b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org imageA_relative_url=expected_image_relative_url, 333b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org imageB_relative_url=actual_image_relative_url, 334b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations=expectations_dict, 335b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org extra_columns=extra_columns_dict) 336b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs.add_image_pair(image_pair) 337b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 338b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs.add_image_pair(image_pair) 339b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org except Exception: 340b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.exception('got exception while creating new ImagePair') 341b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 342b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._results = { 343b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), 344b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 345b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 346b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 347b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 348b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgdef main(): 349b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 350b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org datefmt='%m/%d/%Y %H:%M:%S', 351b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org level=logging.INFO) 352b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser = argparse.ArgumentParser() 353b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 35431d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org '--actuals', default=results.DEFAULT_ACTUALS_DIR, 35541056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org help='Directory containing all actual-result JSON files; defaults to ' 35641056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org '\'%(default)s\' .') 357b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 358b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--expectations', default=DEFAULT_EXPECTATIONS_DIR, 359b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Directory containing all expected-result JSON files; defaults to ' 360b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '\'%(default)s\' .') 361b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 3624cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE, 3634cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org help='If a file with this name is found within the EXPECTATIONS dir, ' 3644cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org 'ignore failures for any tests listed in the file; defaults to ' 3654cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org '\'%(default)s\' .') 3664cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org parser.add_argument( 367b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--outfile', required=True, 368b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='File to write result summary into, in JSON format.') 369b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 370b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 371b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Which result types to include. Defaults to \'%(default)s\'; ' 372b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'must be one of ' + 373b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org str([results.KEY__HEADER__RESULTS_FAILURES, 374b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__HEADER__RESULTS_ALL])) 375b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 37631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, 377b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Directory within which to download images and generate diffs; ' 378b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'defaults to \'%(default)s\' .') 379b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org args = parser.parse_args() 3804cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org results_obj = ExpectationComparisons( 3814cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org actuals_root=args.actuals, 3824cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org expected_root=args.expectations, 3834cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ignore_failures_file=args.ignore_failures_file, 3844cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org generated_images_root=args.workdir) 385b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.WriteToFile( 38641056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org results_obj.get_packaged_results_of_type(results_type=args.results), 387b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org args.outfile) 388b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 389b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 390b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgif __name__ == '__main__': 391b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org main() 392