compare_configs.py revision defe6fdbc8edb2df0887c007450a8d8cc446f420
1#!/usr/bin/python 2 3""" 4Copyright 2014 Google Inc. 5 6Use of this source code is governed by a BSD-style license that can be 7found in the LICENSE file. 8 9Compare GM results for two configs, across all builders. 10""" 11 12# System-level imports 13import argparse 14import fnmatch 15import json 16import logging 17import os 18import re 19import sys 20import time 21 22# Imports from within Skia 23# 24# TODO(epoger): Once we move the create_filepath_url() function out of 25# download_actuals into a shared utility module, we won't need to import 26# download_actuals anymore. 27# 28# We need to add the 'gm' directory, so that we can import gm_json.py within 29# that directory. That script allows us to parse the actual-results.json file 30# written out by the GM tool. 31# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end* 32# so any dirs that are already in the PYTHONPATH will be preferred. 33PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__)) 34GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY) 35TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY) 36if GM_DIRECTORY not in sys.path: 37 sys.path.append(GM_DIRECTORY) 38import download_actuals 39import gm_json 40import imagediffdb 41import imagepair 42import imagepairset 43import results 44 45 46class ConfigComparisons(results.BaseComparisons): 47 """Loads results from two different configurations into an ImagePairSet. 48 49 Loads actual and expected results from all builders, except for those skipped 50 by _ignore_builder(). 51 """ 52 53 def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR, 54 generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT, 55 diff_base_url=None, builder_regex_list=None): 56 """ 57 Args: 58 configs: (string, string) tuple; pair of configs to compare 59 actuals_root: root directory containing all actual-results.json files 60 generated_images_root: directory within which to create all pixel diffs; 61 if this directory does not yet exist, it will be created 62 diff_base_url: base URL within which the client should look for diff 63 images; if not specified, defaults to a "file:///" URL representation 64 of generated_images_root 65 builder_regex_list: List of regular expressions specifying which builders 66 we will process. If None, process all builders. 67 """ 68 time_start = int(time.time()) 69 if builder_regex_list != None: 70 self.set_match_builders_pattern_list(builder_regex_list) 71 self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root) 72 self._diff_base_url = ( 73 diff_base_url or 74 download_actuals.create_filepath_url(generated_images_root)) 75 self._actuals_root = actuals_root 76 self._load_config_pairs(configs) 77 self._timestamp = int(time.time()) 78 logging.info('Results complete; took %d seconds.' % 79 (self._timestamp - time_start)) 80 81 def _load_config_pairs(self, configs): 82 """Loads the results of all tests, across all builders (based on the 83 files within self._actuals_root), compares them across two configs, 84 and stores the summary in self._results. 85 86 Args: 87 configs: tuple of strings; pair of configs to compare 88 """ 89 logging.info('Reading actual-results JSON files from %s...' % 90 self._actuals_root) 91 actual_builder_dicts = self._read_dicts_from_root(self._actuals_root) 92 configA, configB = configs 93 logging.info('Comparing configs %s and %s...' % (configA, configB)) 94 95 all_image_pairs = imagepairset.ImagePairSet( 96 descriptions=configs, 97 diff_base_url=self._diff_base_url) 98 failing_image_pairs = imagepairset.ImagePairSet( 99 descriptions=configs, 100 diff_base_url=self._diff_base_url) 101 102 all_image_pairs.ensure_extra_column_values_in_summary( 103 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ 104 results.KEY__RESULT_TYPE__FAILED, 105 results.KEY__RESULT_TYPE__NOCOMPARISON, 106 results.KEY__RESULT_TYPE__SUCCEEDED, 107 ]) 108 failing_image_pairs.ensure_extra_column_values_in_summary( 109 column_id=results.KEY__EXTRACOLUMN__RESULT_TYPE, values=[ 110 results.KEY__RESULT_TYPE__FAILED, 111 results.KEY__RESULT_TYPE__NOCOMPARISON, 112 ]) 113 114 builders = sorted(actual_builder_dicts.keys()) 115 num_builders = len(builders) 116 builder_num = 0 117 for builder in builders: 118 builder_num += 1 119 logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' % 120 (builder_num, num_builders, builder)) 121 actual_results_for_this_builder = ( 122 actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS]) 123 for result_type in sorted(actual_results_for_this_builder.keys()): 124 results_of_this_type = actual_results_for_this_builder[result_type] 125 if not results_of_this_type: 126 continue 127 128 tests_found = set() 129 for image_name in sorted(results_of_this_type.keys()): 130 (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups() 131 tests_found.add(test) 132 133 for test in tests_found: 134 # Get image_relative_url (or None) for each of configA, configB 135 image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA) 136 configA_image_relative_url = ConfigComparisons._create_relative_url( 137 hashtype_and_digest=results_of_this_type.get(image_name_A), 138 test_name=test) 139 image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB) 140 configB_image_relative_url = ConfigComparisons._create_relative_url( 141 hashtype_and_digest=results_of_this_type.get(image_name_B), 142 test_name=test) 143 144 # If we have images for at least one of these two configs, 145 # add them to our list. 146 if configA_image_relative_url or configB_image_relative_url: 147 if configA_image_relative_url == configB_image_relative_url: 148 result_type = results.KEY__RESULT_TYPE__SUCCEEDED 149 elif not configA_image_relative_url: 150 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 151 elif not configB_image_relative_url: 152 result_type = results.KEY__RESULT_TYPE__NOCOMPARISON 153 else: 154 result_type = results.KEY__RESULT_TYPE__FAILED 155 156 extra_columns_dict = { 157 results.KEY__EXTRACOLUMN__RESULT_TYPE: result_type, 158 results.KEY__EXTRACOLUMN__BUILDER: builder, 159 results.KEY__EXTRACOLUMN__TEST: test, 160 # TODO(epoger): Right now, the client UI crashes if it receives 161 # results that do not include a 'config' column. 162 # Until we fix that, keep the client happy. 163 results.KEY__EXTRACOLUMN__CONFIG: 'TODO', 164 } 165 166 try: 167 image_pair = imagepair.ImagePair( 168 image_diff_db=self._image_diff_db, 169 base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL, 170 imageA_relative_url=configA_image_relative_url, 171 imageB_relative_url=configB_image_relative_url, 172 extra_columns=extra_columns_dict) 173 all_image_pairs.add_image_pair(image_pair) 174 if result_type != results.KEY__RESULT_TYPE__SUCCEEDED: 175 failing_image_pairs.add_image_pair(image_pair) 176 except (KeyError, TypeError): 177 logging.exception( 178 'got exception while creating ImagePair for image_name ' 179 '"%s", builder "%s"' % (image_name, builder)) 180 181 self._results = { 182 results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(), 183 results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(), 184 } 185 186 187def main(): 188 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', 189 datefmt='%m/%d/%Y %H:%M:%S', 190 level=logging.INFO) 191 parser = argparse.ArgumentParser() 192 parser.add_argument( 193 '--actuals', default=results.DEFAULT_ACTUALS_DIR, 194 help='Directory containing all actual-result JSON files; defaults to ' 195 '\'%(default)s\' .') 196 parser.add_argument( 197 'config', nargs=2, 198 help='Two configurations to compare (8888, gpu, etc.).') 199 parser.add_argument( 200 '--outfile', required=True, 201 help='File to write result summary into, in JSON format.') 202 parser.add_argument( 203 '--results', default=results.KEY__HEADER__RESULTS_FAILURES, 204 help='Which result types to include. Defaults to \'%(default)s\'; ' 205 'must be one of ' + 206 str([results.KEY__HEADER__RESULTS_FAILURES, 207 results.KEY__HEADER__RESULTS_ALL])) 208 parser.add_argument( 209 '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT, 210 help='Directory within which to download images and generate diffs; ' 211 'defaults to \'%(default)s\' .') 212 args = parser.parse_args() 213 results_obj = ConfigComparisons(configs=args.config, 214 actuals_root=args.actuals, 215 generated_images_root=args.workdir) 216 gm_json.WriteToFile( 217 results_obj.get_packaged_results_of_type(results_type=args.results), 218 args.outfile) 219 220 221if __name__ == '__main__': 222 main() 223