compare_configs.py revision 68a3815401f461976f76891d0477cb1440fa0aba 
1#!/usr/bin/python
2
3"""
4Copyright 2014 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Compare GM results for two configs, across all builders.
10"""
11
12# System-level imports
13import argparse
14import fnmatch
15import json
16import logging
17import os
18import re
19import sys
20import time
21
22# Imports from within Skia
23#
24# TODO(epoger): Once we move the create_filepath_url() function out of
25# download_actuals into a shared utility module, we won't need to import
26# download_actuals anymore.
27#
28# We need to add the 'gm' directory, so that we can import gm_json.py within
29# that directory.  That script allows us to parse the actual-results.json file
30# written out by the GM tool.
31# Make sure that the 'gm' dir is in the PYTHONPATH, but add it at the *end*
32# so any dirs that are already in the PYTHONPATH will be preferred.
33PARENT_DIRECTORY = os.path.dirname(os.path.realpath(__file__))
34GM_DIRECTORY = os.path.dirname(PARENT_DIRECTORY)
35TRUNK_DIRECTORY = os.path.dirname(GM_DIRECTORY)
36if GM_DIRECTORY not in sys.path:
37  sys.path.append(GM_DIRECTORY)
38import download_actuals
39import gm_json
40import imagediffdb
41import imagepair
42import imagepairset
43import results
44
45
46class ConfigComparisons(results.BaseComparisons):
47  """Loads results from two different configurations into an ImagePairSet.
48
49  Loads actual and expected results from all builders, except for those skipped
50  by _ignore_builder().
51  """
52
53  def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
54               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
55               diff_base_url=None, builder_regex_list=None):
56    """
57    Args:
58      configs: (string, string) tuple; pair of configs to compare
59      actuals_root: root directory containing all actual-results.json files
60      generated_images_root: directory within which to create all pixel diffs;
61          if this directory does not yet exist, it will be created
62      diff_base_url: base URL within which the client should look for diff
63          images; if not specified, defaults to a "file:///" URL representation
64          of generated_images_root
65      builder_regex_list: List of regular expressions specifying which builders
66          we will process. If None, process all builders.
67    """
68    time_start = int(time.time())
69    if builder_regex_list != None:
70      self.set_match_builders_pattern_list(builder_regex_list)
71    self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
72    self._diff_base_url = (
73        diff_base_url or
74        download_actuals.create_filepath_url(generated_images_root))
75    self._actuals_root = actuals_root
76    self._load_config_pairs(configs)
77    self._timestamp = int(time.time())
78    logging.info('Results complete; took %d seconds.' %
79                 (self._timestamp - time_start))
80
81  def _load_config_pairs(self, configs):
82    """Loads the results of all tests, across all builders (based on the
83    files within self._actuals_root), compares them across two configs,
84    and stores the summary in self._results.
85
86    Args:
87      configs: tuple of strings; pair of configs to compare
88    """
89    logging.info('Reading actual-results JSON files from %s...' %
90                 self._actuals_root)
91    actual_builder_dicts = self._read_builder_dicts_from_root(
92        self._actuals_root)
93    configA, configB = configs
94    logging.info('Comparing configs %s and %s...' % (configA, configB))
95
96    all_image_pairs = imagepairset.ImagePairSet(
97        descriptions=configs,
98        diff_base_url=self._diff_base_url)
99    failing_image_pairs = imagepairset.ImagePairSet(
100        descriptions=configs,
101        diff_base_url=self._diff_base_url)
102
103    all_image_pairs.ensure_extra_column_values_in_summary(
104        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
105            results.KEY__RESULT_TYPE__FAILED,
106            results.KEY__RESULT_TYPE__NOCOMPARISON,
107            results.KEY__RESULT_TYPE__SUCCEEDED,
108        ])
109    failing_image_pairs.ensure_extra_column_values_in_summary(
110        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
111            results.KEY__RESULT_TYPE__FAILED,
112            results.KEY__RESULT_TYPE__NOCOMPARISON,
113        ])
114
115    builders = sorted(actual_builder_dicts.keys())
116    num_builders = len(builders)
117    builder_num = 0
118    for builder in builders:
119      builder_num += 1
120      logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
121                   (builder_num, num_builders, builder))
122      actual_results_for_this_builder = (
123          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
124      for result_type in sorted(actual_results_for_this_builder.keys()):
125        results_of_this_type = actual_results_for_this_builder[result_type]
126        if not results_of_this_type:
127          continue
128
129        tests_found = set()
130        for image_name in sorted(results_of_this_type.keys()):
131          (test, config) = results.IMAGE_FILENAME_RE.match(image_name).groups()
132          tests_found.add(test)
133
134        for test in tests_found:
135          # Get image_relative_url (or None) for each of configA, configB
136          image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
137          configA_image_relative_url = ConfigComparisons._create_relative_url(
138              hashtype_and_digest=results_of_this_type.get(image_name_A),
139              test_name=test)
140          image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
141          configB_image_relative_url = ConfigComparisons._create_relative_url(
142              hashtype_and_digest=results_of_this_type.get(image_name_B),
143              test_name=test)
144
145          # If we have images for at least one of these two configs,
146          # add them to our list.
147          if configA_image_relative_url or configB_image_relative_url:
148            if configA_image_relative_url == configB_image_relative_url:
149              result_type = results.KEY__RESULT_TYPE__SUCCEEDED
150            elif not configA_image_relative_url:
151              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
152            elif not configB_image_relative_url:
153              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
154            else:
155              result_type = results.KEY__RESULT_TYPE__FAILED
156
157            extra_columns_dict = {
158                results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
159                results.KEY__EXTRACOLUMNS__BUILDER: builder,
160                results.KEY__EXTRACOLUMNS__TEST: test,
161                # TODO(epoger): Right now, the client UI crashes if it receives
162                # results that do not include a 'config' column.
163                # Until we fix that, keep the client happy.
164                results.KEY__EXTRACOLUMNS__CONFIG: 'TODO',
165            }
166
167            try:
168              image_pair = imagepair.ImagePair(
169                  image_diff_db=self._image_diff_db,
170                  base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
171                  imageA_relative_url=configA_image_relative_url,
172                  imageB_relative_url=configB_image_relative_url,
173                  extra_columns=extra_columns_dict)
174              all_image_pairs.add_image_pair(image_pair)
175              if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
176                failing_image_pairs.add_image_pair(image_pair)
177            except (KeyError, TypeError):
178              logging.exception(
179                  'got exception while creating ImagePair for image_name '
180                  '"%s", builder "%s"' % (image_name, builder))
181
182    self._results = {
183      results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
184      results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
185    }
186
187
188def main():
189  logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
190                      datefmt='%m/%d/%Y %H:%M:%S',
191                      level=logging.INFO)
192  parser = argparse.ArgumentParser()
193  parser.add_argument(
194      '--actuals', default=results.DEFAULT_ACTUALS_DIR,
195      help='Directory containing all actual-result JSON files; defaults to '
196      '\'%(default)s\' .')
197  parser.add_argument(
198      'config', nargs=2,
199      help='Two configurations to compare (8888, gpu, etc.).')
200  parser.add_argument(
201      '--outfile', required=True,
202      help='File to write result summary into, in JSON format.')
203  parser.add_argument(
204      '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
205      help='Which result types to include. Defaults to \'%(default)s\'; '
206      'must be one of ' +
207      str([results.KEY__HEADER__RESULTS_FAILURES,
208           results.KEY__HEADER__RESULTS_ALL]))
209  parser.add_argument(
210      '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
211      help='Directory within which to download images and generate diffs; '
212      'defaults to \'%(default)s\' .')
213  args = parser.parse_args()
214  results_obj = ConfigComparisons(configs=args.config,
215                                  actuals_root=args.actuals,
216                                  generated_images_root=args.workdir)
217  gm_json.WriteToFile(
218      results_obj.get_packaged_results_of_type(results_type=args.results),
219      args.outfile)
220
221
222if __name__ == '__main__':
223  main()
224