1b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org#!/usr/bin/python 2b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 3b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org""" 4b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgCopyright 2013 Google Inc. 5b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 6b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgUse of this source code is governed by a BSD-style license that can be 7b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgfound in the LICENSE file. 8b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 9b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgRepackage expected/actual GM results as needed by our HTML rebaseline viewer. 10b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org""" 11b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 12b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org# System-level imports 13b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport argparse 14b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport fnmatch 15b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport logging 16b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport os 17b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport time 18b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 19b4edbffd7c09ca172f95fc30381671962e22dee0epoger# Must fix up PYTHONPATH before importing from within Skia 203b5c86c7a2db25f82a8415b6c79d1ac580fc64aestephanaimport rs_fixpypath # pylint: disable=W0611 21b4edbffd7c09ca172f95fc30381671962e22dee0epoger 22b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org# Imports from within Skia 23133931f4abdafa5bb0bdea3a02af1e5a70d5ac98epogerfrom py.utils import url_utils 24b4edbffd7c09ca172f95fc30381671962e22dee0epogerimport column 25b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport gm_json 26b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagediffdb 27b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagepair 28b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport imagepairset 29b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgimport results 30b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 31b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgEXPECTATION_FIELDS_PASSED_THRU_VERBATIM = [ 32b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__BUGS, 33b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__IGNOREFAILURE, 34b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__REVIEWED, 35b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org] 36b4edbffd7c09ca172f95fc30381671962e22dee0epogerFREEFORM_COLUMN_IDS = [ 37b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__BUILDER, 38b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__TEST, 39b4edbffd7c09ca172f95fc30381671962e22dee0epoger] 40b4edbffd7c09ca172f95fc30381671962e22dee0epogerORDERED_COLUMN_IDS = [ 41b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__RESULT_TYPE, 42b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__BUILDER, 43b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__TEST, 44b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__EXTRACOLUMNS__CONFIG, 45b4edbffd7c09ca172f95fc30381671962e22dee0epoger] 46b4edbffd7c09ca172f95fc30381671962e22dee0epoger 47b144271179aaf82cb1151e9dfd8e866747402594epogerTRUNK_DIRECTORY = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) 4831d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.orgDEFAULT_EXPECTATIONS_DIR = os.path.join(TRUNK_DIRECTORY, 'expectations', 'gm') 494cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.orgDEFAULT_IGNORE_FAILURES_FILE = 'ignored-tests.txt' 50b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 51b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgIMAGEPAIR_SET_DESCRIPTIONS = ('expected image', 'actual image') 52b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 53b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 5431d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.orgclass ExpectationComparisons(results.BaseComparisons): 5531d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org """Loads actual and expected GM results into an ImagePairSet. 56b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 57b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Loads actual and expected results from all builders, except for those skipped 58b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org by _ignore_builder(). 59b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 60b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Once this object has been constructed, the results (in self._results[]) 61b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org are immutable. If you want to update the results based on updated JSON 6231d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org file contents, you will need to create a new ExpectationComparisons object.""" 63b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 646132b436d8723beaf06d1a8a0880f4f1535908a0epoger def __init__(self, image_diff_db, actuals_root=results.DEFAULT_ACTUALS_DIR, 65b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_root=DEFAULT_EXPECTATIONS_DIR, 664cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ignore_failures_file=DEFAULT_IGNORE_FAILURES_FILE, 67defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org diff_base_url=None, builder_regex_list=None): 68b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 69b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 706132b436d8723beaf06d1a8a0880f4f1535908a0epoger image_diff_db: instance of ImageDiffDB we use to cache the image diffs 71b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actuals_root: root directory containing all actual-results.json files 72b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_root: root directory containing all expected-results.json files 734cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ignore_failures_file: if a file with this name is found within 744cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org expected_root, ignore failures for any tests listed in the file 75b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url: base URL within which the client should look for diff 76b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org images; if not specified, defaults to a "file:///" URL representation 776132b436d8723beaf06d1a8a0880f4f1535908a0epoger of image_diff_db's storage_root 78defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org builder_regex_list: List of regular expressions specifying which builders 79defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org we will process. If None, process all builders. 80b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 815c4b137592a7937b4d02d72a976fa783f8d3675aepoger super(ExpectationComparisons, self).__init__() 82b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org time_start = int(time.time()) 83defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org if builder_regex_list != None: 84defe6fdbc8edb2df0887c007450a8d8cc446f420commit-bot@chromium.org self.set_match_builders_pattern_list(builder_regex_list) 856132b436d8723beaf06d1a8a0880f4f1535908a0epoger self._image_diff_db = image_diff_db 86b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._diff_base_url = ( 87b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url or 886132b436d8723beaf06d1a8a0880f4f1535908a0epoger url_utils.create_filepath_url(image_diff_db.storage_root)) 89b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._actuals_root = actuals_root 90b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._expected_root = expected_root 914cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org self._ignore_failures_on_these_tests = [] 924cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org if ignore_failures_file: 934cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org self._ignore_failures_on_these_tests = ( 944cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org ExpectationComparisons._read_noncomment_lines( 954cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org os.path.join(expected_root, ignore_failures_file))) 96b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._load_actual_and_expected() 97b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._timestamp = int(time.time()) 98b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Results complete; took %d seconds.' % 99b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (self._timestamp - time_start)) 100b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 101b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def edit_expectations(self, modifications): 102b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Edit the expectations stored within this object and write them back 103b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org to disk. 104b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 105b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Note that this will NOT update the results stored in self._results[] ; 10631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org in order to see those updates, you must instantiate a new 10731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons object based on the (now updated) files on disk. 108b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 109b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 110b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org modifications: a list of dictionaries, one for each expectation to update: 111b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 112b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [ 113b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org { 11468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org imagepair.KEY__IMAGEPAIRS__EXPECTATIONS: { 115b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__BUGS: [123, 456], 116b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__IGNOREFAILURE: false, 117b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__EXPECTATIONS__REVIEWED: true, 118b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 11968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS: { 12068a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: 'Test-Mac10.6-MacMini4.1-GeForce320M-x86-Debug', 12168a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__CONFIG: '8888', 12268a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__TEST: 'bigmatrix', 123b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 12468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__IMAGEPAIRS__IMAGE_B_URL: 'bitmap-64bitMD5/bigmatrix/10894408024079689926.png', 125b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }, 126b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ... 127b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ] 128b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 129b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 1307418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org expected_builder_dicts = self._read_builder_dicts_from_root( 1317418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._expected_root) 132b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for mod in modifications: 133b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_name = results.IMAGE_FILENAME_FORMATTER % ( 13468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 13568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__TEST], 13668a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 13768a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__CONFIG]) 138b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org _, hash_type, hash_digest = gm_json.SplitGmRelativeUrl( 13968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__IMAGE_B_URL]) 140b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org allowed_digests = [[hash_type, int(hash_digest)]] 141b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org new_expectations = { 142b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS: allowed_digests, 143b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 144b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 14568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org value = mod[imagepair.KEY__IMAGEPAIRS__EXPECTATIONS].get(field) 146b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if value is not None: 147b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org new_expectations[field] = value 148b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_dict = expected_builder_dicts[ 14968a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org mod[imagepair.KEY__IMAGEPAIRS__EXTRACOLUMNS] 15068a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org [results.KEY__EXTRACOLUMNS__BUILDER]] 151b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations = builder_dict.get(gm_json.JSONKEY_EXPECTEDRESULTS) 152b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not builder_expectations: 153b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations = {} 154b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_dict[gm_json.JSONKEY_EXPECTEDRESULTS] = builder_expectations 155b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_expectations[image_name] = new_expectations 15631d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._write_dicts_to_root( 15731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org expected_builder_dicts, self._expected_root) 158b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 159b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org @staticmethod 160b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def _write_dicts_to_root(meta_dict, root, pattern='*.json'): 161b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Write all per-builder dictionaries within meta_dict to files under 162b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org the root path. 163b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 164b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Security note: this will only write to files that already exist within 165b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org the root path (as found by os.walk() within root), so we don't need to 166b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org worry about malformed content writing to disk outside of root. 167b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org However, the data written to those files is not double-checked, so it 168b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org could contain poisonous data. 169b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 170b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Args: 171b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org meta_dict: a builder-keyed meta-dictionary containing all the JSON 172b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org dictionaries we want to write out 173b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org root: path to root of directory tree within which to write files 174b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org pattern: which files to write within root (fnmatch-style pattern) 175b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 176b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org Raises: 177b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org IOError if root does not refer to an existing directory 178b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org KeyError if the set of per-builder dictionaries written out was 179b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org different than expected 180b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 181b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not os.path.isdir(root): 182b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org raise IOError('no directory found at path %s' % root) 183b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written = [] 184b4edbffd7c09ca172f95fc30381671962e22dee0epoger for dirpath, _, filenames in os.walk(root): 185b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for matching_filename in fnmatch.filter(filenames, pattern): 186b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder = os.path.basename(dirpath) 187b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org per_builder_dict = meta_dict.get(builder) 188b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if per_builder_dict is not None: 189b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org fullpath = os.path.join(dirpath, matching_filename) 190b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.WriteToFile(per_builder_dict, fullpath) 191b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written.append(builder) 192b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 193b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Check: did we write out the set of per-builder dictionaries we 194b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # expected to? 195b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builders_written = sorted(meta_dict.keys()) 196b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builders_written.sort() 197b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if expected_builders_written != actual_builders_written: 198b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org raise KeyError( 199b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'expected to write dicts for builders %s, but actually wrote them ' 200b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'for builders %s' % ( 201b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builders_written, actual_builders_written)) 202b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 203b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org def _load_actual_and_expected(self): 204b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """Loads the results of all tests, across all builders (based on the 205b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org files within self._actuals_root and self._expected_root), 206b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org and stores them in self._results. 207b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org """ 208b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Reading actual-results JSON files from %s...' % 209b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._actuals_root) 2107418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org actual_builder_dicts = self._read_builder_dicts_from_root( 2117418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._actuals_root) 212b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Reading expected-results JSON files from %s...' % 213b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._expected_root) 2147418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org expected_builder_dicts = self._read_builder_dicts_from_root( 2157418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org self._expected_root) 216b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 217b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs = imagepairset.ImagePairSet( 218b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 219b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url=self._diff_base_url) 220b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs = imagepairset.ImagePairSet( 221b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org descriptions=IMAGEPAIR_SET_DESCRIPTIONS, 222b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org diff_base_url=self._diff_base_url) 223b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 224b4edbffd7c09ca172f95fc30381671962e22dee0epoger # Override settings for columns that should be filtered using freeform text. 225b4edbffd7c09ca172f95fc30381671962e22dee0epoger for column_id in FREEFORM_COLUMN_IDS: 226b4edbffd7c09ca172f95fc30381671962e22dee0epoger factory = column.ColumnHeaderFactory( 227b4edbffd7c09ca172f95fc30381671962e22dee0epoger header_text=column_id, use_freeform_filter=True) 228b4edbffd7c09ca172f95fc30381671962e22dee0epoger all_image_pairs.set_column_header_factory( 229b4edbffd7c09ca172f95fc30381671962e22dee0epoger column_id=column_id, column_header_factory=factory) 230b4edbffd7c09ca172f95fc30381671962e22dee0epoger failing_image_pairs.set_column_header_factory( 231b4edbffd7c09ca172f95fc30381671962e22dee0epoger column_id=column_id, column_header_factory=factory) 232b4edbffd7c09ca172f95fc30381671962e22dee0epoger 233b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs.ensure_extra_column_values_in_summary( 23468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 235b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILED, 236b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILUREIGNORED, 237b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__NOCOMPARISON, 238b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__SUCCEEDED, 239b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ]) 240b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs.ensure_extra_column_values_in_summary( 24168a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[ 242b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILED, 243b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__FAILUREIGNORED, 244b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__RESULT_TYPE__NOCOMPARISON, 245b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org ]) 246b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 2476323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # Only consider builders we have both expected and actual results for. 2486323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # Fixes http://skbug.com/2486 ('rebaseline_server shows actual results 2496323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # (but not expectations) for Test-Ubuntu12-ShuttleA-NoGPU-x86_64-Debug 2506323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org # builder') 2516323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org actual_builder_set = set(actual_builder_dicts.keys()) 2526323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org expected_builder_set = set(expected_builder_dicts.keys()) 2536323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org builders = sorted(actual_builder_set.intersection(expected_builder_set)) 2546323844cb38ae9c3b677ac230b5cf61c69f57eb3commit-bot@chromium.org 255b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org num_builders = len(builders) 256b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_num = 0 257b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for builder in builders: 258b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org builder_num += 1 259b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 260b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (builder_num, num_builders, builder)) 261b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_results_for_this_builder = ( 262b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 263b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for result_type in sorted(actual_results_for_this_builder.keys()): 264b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results_of_this_type = actual_results_for_this_builder[result_type] 265b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if not results_of_this_type: 266b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org continue 267b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for image_name in sorted(results_of_this_type.keys()): 268b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 26931d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org actual_image_relative_url = ( 27031d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._create_relative_url( 27131d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org hashtype_and_digest=results_of_this_type[image_name], 27231d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org test_name=test)) 273b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 274b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Default empty expectations; overwrite these if we find any real ones 275b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test = None 276b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_image_relative_url = None 277b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict = None 278b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org try: 279b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test = ( 280b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_builder_dicts 281b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [builder][gm_json.JSONKEY_EXPECTEDRESULTS][image_name]) 282b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # TODO(epoger): assumes a single allowed digest per test, which is 283b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # fine; see https://code.google.com/p/skia/issues/detail?id=1787 284b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expected_image_hashtype_and_digest = ( 285b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_per_test 286b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org [gm_json.JSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS][0]) 28731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org expected_image_relative_url = ( 28831d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org ExpectationComparisons._create_relative_url( 28931d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org hashtype_and_digest=expected_image_hashtype_and_digest, 29031d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org test_name=test)) 291b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict = {} 292b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org for field in EXPECTATION_FIELDS_PASSED_THRU_VERBATIM: 293b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations_dict[field] = expectations_per_test.get(field) 294b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org except (KeyError, TypeError): 295b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # There are several cases in which we would expect to find 296b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # no expectations for a given test: 297b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 298b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 1. result_type == NOCOMPARISON 299b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # There are no expectations for this test yet! 300b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 301b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 2. alternate rendering mode failures (e.g. serialized) 302b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # In cases like 303b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # https://code.google.com/p/skia/issues/detail?id=1684 304b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # ('tileimagefilter GM test failing in serialized render mode'), 305b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # the gm-actuals will list a failure for the alternate 306b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # rendering mode even though we don't have explicit expectations 307b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # for the test (the implicit expectation is that it must 308b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # render the same in all rendering modes). 309b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 310b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Don't log type 1, because it is common. 311b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # Log other types, because they are rare and we should know about 312b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # them, but don't throw an exception, because we need to keep our 313b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # tools working in the meanwhile! 314b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if result_type != results.KEY__RESULT_TYPE__NOCOMPARISON: 315b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.warning('No expectations found for test: %s' % { 31668a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: builder, 31768a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type, 318b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'image_name': image_name, 319b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org }) 320b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 321b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # If this test was recently rebaselined, it will remain in 322b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # the 'failed' set of actuals until all the bots have 323b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # cycled (although the expectations have indeed been set 324b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # from the most recent actuals). Treat these as successes 325b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # instead of failures. 326b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # 327b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # TODO(epoger): Do we need to do something similar in 328b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # other cases, such as when we have recently marked a test 329b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # as ignoreFailure but it still shows up in the 'failed' 330b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # category? Maybe we should not rely on the result_type 331b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # categories recorded within the gm_actuals AT ALL, and 332b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # instead evaluate the result_type ourselves based on what 333b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org # we see in expectations vs actual checksum? 334b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if expected_image_relative_url == actual_image_relative_url: 335b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org updated_result_type = results.KEY__RESULT_TYPE__SUCCEEDED 3364cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org elif ((result_type == results.KEY__RESULT_TYPE__FAILED) and 3374cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org (test in self._ignore_failures_on_these_tests)): 3384cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org updated_result_type = results.KEY__RESULT_TYPE__FAILUREIGNORED 339b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org else: 340b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org updated_result_type = result_type 341b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org extra_columns_dict = { 34268a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__RESULT_TYPE: updated_result_type, 34368a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__BUILDER: builder, 34468a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__TEST: test, 34568a3815401f461976f76891d0477cb1440fa0abacommit-bot@chromium.org results.KEY__EXTRACOLUMNS__CONFIG: config, 346b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 347b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org try: 348b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_pair = imagepair.ImagePair( 349b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org image_diff_db=self._image_diff_db, 3502529f2e72cddf87904c8ad4b613942cbef802cfbrmistry imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 3512529f2e72cddf87904c8ad4b613942cbef802cfbrmistry imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 352b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org imageA_relative_url=expected_image_relative_url, 353b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org imageB_relative_url=actual_image_relative_url, 354b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org expectations=expectations_dict, 355b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org extra_columns=extra_columns_dict) 356b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org all_image_pairs.add_image_pair(image_pair) 357b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org if updated_result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 358b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org failing_image_pairs.add_image_pair(image_pair) 359b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org except Exception: 360b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.exception('got exception while creating new ImagePair') 361b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 362b4edbffd7c09ca172f95fc30381671962e22dee0epoger # pylint: disable=W0201 363b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org self._results = { 364b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict( 365b4edbffd7c09ca172f95fc30381671962e22dee0epoger column_ids_in_order=ORDERED_COLUMN_IDS), 366b4edbffd7c09ca172f95fc30381671962e22dee0epoger results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict( 367b4edbffd7c09ca172f95fc30381671962e22dee0epoger column_ids_in_order=ORDERED_COLUMN_IDS), 368b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org } 369b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 370b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 371b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgdef main(): 372b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 373b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org datefmt='%m/%d/%Y %H:%M:%S', 374b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org level=logging.INFO) 375b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser = argparse.ArgumentParser() 376b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 37731d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org '--actuals', default=results.DEFAULT_ACTUALS_DIR, 37841056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org help='Directory containing all actual-result JSON files; defaults to ' 37941056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org '\'%(default)s\' .') 380b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 381b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--expectations', default=DEFAULT_EXPECTATIONS_DIR, 382b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Directory containing all expected-result JSON files; defaults to ' 383b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '\'%(default)s\' .') 384b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 3854cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org '--ignore-failures-file', default=DEFAULT_IGNORE_FAILURES_FILE, 3864cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org help='If a file with this name is found within the EXPECTATIONS dir, ' 3874cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org 'ignore failures for any tests listed in the file; defaults to ' 3884cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org '\'%(default)s\' .') 3894cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org parser.add_argument( 390b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--outfile', required=True, 391b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='File to write result summary into, in JSON format.') 392b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 393b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 394b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Which result types to include. Defaults to \'%(default)s\'; ' 395b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'must be one of ' + 396b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org str([results.KEY__HEADER__RESULTS_FAILURES, 397b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org results.KEY__HEADER__RESULTS_ALL])) 398b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org parser.add_argument( 39931d0b3d806a1aa86b7edaa442b3821f5d548e184commit-bot@chromium.org '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, 400b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org help='Directory within which to download images and generate diffs; ' 401b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 'defaults to \'%(default)s\' .') 402b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org args = parser.parse_args() 4036132b436d8723beaf06d1a8a0880f4f1535908a0epoger image_diff_db = imagediffdb.ImageDiffDB(storage_root=args.workdir) 4044cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org results_obj = ExpectationComparisons( 4056132b436d8723beaf06d1a8a0880f4f1535908a0epoger image_diff_db=image_diff_db, 4064cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org actuals_root=args.actuals, 4074cef1be82193a9d527f7b1f873197c443d0dde2ecommit-bot@chromium.org expected_root=args.expectations, 4086132b436d8723beaf06d1a8a0880f4f1535908a0epoger ignore_failures_file=args.ignore_failures_file) 409b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org gm_json.WriteToFile( 41041056232e471e0f5448e88dabe0ef4c5e50f21e6commit-bot@chromium.org results_obj.get_packaged_results_of_type(results_type=args.results), 411b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org args.outfile) 412b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 413b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org 414b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.orgif __name__ == '__main__': 415b463d5668a498b672b80047f09901981afe513edcommit-bot@chromium.org main() 416