1#!/usr/bin/python
2
3"""
4Copyright 2014 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Compare GM results for two configs, across all builders.
10"""
11
12# System-level imports
13import argparse
14import logging
15import time
16
17# Must fix up PYTHONPATH before importing from within Skia
18import rs_fixpypath  # pylint: disable=W0611
19
20# Imports from within Skia
21from py.utils import url_utils
22import gm_json
23import imagediffdb
24import imagepair
25import imagepairset
26import results
27
28
29class ConfigComparisons(results.BaseComparisons):
30  """Loads results from two different configurations into an ImagePairSet.
31
32  Loads actual and expected results from all builders, except for those skipped
33  by _ignore_builder().
34  """
35
36  def __init__(self, configs, actuals_root=results.DEFAULT_ACTUALS_DIR,
37               generated_images_root=results.DEFAULT_GENERATED_IMAGES_ROOT,
38               diff_base_url=None, builder_regex_list=None):
39    """
40    Args:
41      configs: (string, string) tuple; pair of configs to compare
42      actuals_root: root directory containing all actual-results.json files
43      generated_images_root: directory within which to create all pixel diffs;
44          if this directory does not yet exist, it will be created
45      diff_base_url: base URL within which the client should look for diff
46          images; if not specified, defaults to a "file:///" URL representation
47          of generated_images_root
48      builder_regex_list: List of regular expressions specifying which builders
49          we will process. If None, process all builders.
50    """
51    super(ConfigComparisons, self).__init__()
52    time_start = int(time.time())
53    if builder_regex_list != None:
54      self.set_match_builders_pattern_list(builder_regex_list)
55    self._image_diff_db = imagediffdb.ImageDiffDB(generated_images_root)
56    self._diff_base_url = (
57        diff_base_url or
58        url_utils.create_filepath_url(generated_images_root))
59    self._actuals_root = actuals_root
60    self._load_config_pairs(configs)
61    self._timestamp = int(time.time())
62    logging.info('Results complete; took %d seconds.' %
63                 (self._timestamp - time_start))
64
65  def _load_config_pairs(self, configs):
66    """Loads the results of all tests, across all builders (based on the
67    files within self._actuals_root), compares them across two configs,
68    and stores the summary in self._results.
69
70    Args:
71      configs: tuple of strings; pair of configs to compare
72    """
73    logging.info('Reading actual-results JSON files from %s...' %
74                 self._actuals_root)
75    actual_builder_dicts = self._read_builder_dicts_from_root(
76        self._actuals_root)
77    configA, configB = configs
78    logging.info('Comparing configs %s and %s...' % (configA, configB))
79
80    all_image_pairs = imagepairset.ImagePairSet(
81        descriptions=configs,
82        diff_base_url=self._diff_base_url)
83    failing_image_pairs = imagepairset.ImagePairSet(
84        descriptions=configs,
85        diff_base_url=self._diff_base_url)
86
87    all_image_pairs.ensure_extra_column_values_in_summary(
88        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
89            results.KEY__RESULT_TYPE__FAILED,
90            results.KEY__RESULT_TYPE__NOCOMPARISON,
91            results.KEY__RESULT_TYPE__SUCCEEDED,
92        ])
93    failing_image_pairs.ensure_extra_column_values_in_summary(
94        column_id=results.KEY__EXTRACOLUMNS__RESULT_TYPE, values=[
95            results.KEY__RESULT_TYPE__FAILED,
96            results.KEY__RESULT_TYPE__NOCOMPARISON,
97        ])
98
99    builders = sorted(actual_builder_dicts.keys())
100    num_builders = len(builders)
101    builder_num = 0
102    for builder in builders:
103      builder_num += 1
104      logging.info('Generating pixel diffs for builder #%d of %d, "%s"...' %
105                   (builder_num, num_builders, builder))
106      actual_results_for_this_builder = (
107          actual_builder_dicts[builder][gm_json.JSONKEY_ACTUALRESULTS])
108      for result_type in sorted(actual_results_for_this_builder.keys()):
109        results_of_this_type = actual_results_for_this_builder[result_type]
110        if not results_of_this_type:
111          continue
112
113        tests_found = set()
114        for image_name in sorted(results_of_this_type.keys()):
115          (test, _) = results.IMAGE_FILENAME_RE.match(image_name).groups()
116          tests_found.add(test)
117
118        for test in tests_found:
119          # Get image_relative_url (or None) for each of configA, configB
120          image_name_A = results.IMAGE_FILENAME_FORMATTER % (test, configA)
121          configA_image_relative_url = ConfigComparisons._create_relative_url(
122              hashtype_and_digest=results_of_this_type.get(image_name_A),
123              test_name=test)
124          image_name_B = results.IMAGE_FILENAME_FORMATTER % (test, configB)
125          configB_image_relative_url = ConfigComparisons._create_relative_url(
126              hashtype_and_digest=results_of_this_type.get(image_name_B),
127              test_name=test)
128
129          # If we have images for at least one of these two configs,
130          # add them to our list.
131          if configA_image_relative_url or configB_image_relative_url:
132            if configA_image_relative_url == configB_image_relative_url:
133              result_type = results.KEY__RESULT_TYPE__SUCCEEDED
134            elif not configA_image_relative_url:
135              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
136            elif not configB_image_relative_url:
137              result_type = results.KEY__RESULT_TYPE__NOCOMPARISON
138            else:
139              result_type = results.KEY__RESULT_TYPE__FAILED
140
141            extra_columns_dict = {
142                results.KEY__EXTRACOLUMNS__RESULT_TYPE: result_type,
143                results.KEY__EXTRACOLUMNS__BUILDER: builder,
144                results.KEY__EXTRACOLUMNS__TEST: test,
145                # TODO(epoger): Right now, the client UI crashes if it receives
146                # results that do not include a 'config' column.
147                # Until we fix that, keep the client happy.
148                results.KEY__EXTRACOLUMNS__CONFIG: 'TODO',
149            }
150
151            try:
152              image_pair = imagepair.ImagePair(
153                  image_diff_db=self._image_diff_db,
154                  imageA_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
155                  imageB_base_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL,
156                  imageA_relative_url=configA_image_relative_url,
157                  imageB_relative_url=configB_image_relative_url,
158                  extra_columns=extra_columns_dict)
159              all_image_pairs.add_image_pair(image_pair)
160              if result_type != results.KEY__RESULT_TYPE__SUCCEEDED:
161                failing_image_pairs.add_image_pair(image_pair)
162            except (KeyError, TypeError):
163              logging.exception(
164                  'got exception while creating ImagePair for test '
165                  '"%s", builder "%s"' % (test, builder))
166
167    # pylint: disable=W0201
168    self._results = {
169      results.KEY__HEADER__RESULTS_ALL: all_image_pairs.as_dict(),
170      results.KEY__HEADER__RESULTS_FAILURES: failing_image_pairs.as_dict(),
171    }
172
173
174def main():
175  logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
176                      datefmt='%m/%d/%Y %H:%M:%S',
177                      level=logging.INFO)
178  parser = argparse.ArgumentParser()
179  parser.add_argument(
180      '--actuals', default=results.DEFAULT_ACTUALS_DIR,
181      help='Directory containing all actual-result JSON files; defaults to '
182      '\'%(default)s\' .')
183  parser.add_argument(
184      'config', nargs=2,
185      help='Two configurations to compare (8888, gpu, etc.).')
186  parser.add_argument(
187      '--outfile', required=True,
188      help='File to write result summary into, in JSON format.')
189  parser.add_argument(
190      '--results', default=results.KEY__HEADER__RESULTS_FAILURES,
191      help='Which result types to include. Defaults to \'%(default)s\'; '
192      'must be one of ' +
193      str([results.KEY__HEADER__RESULTS_FAILURES,
194           results.KEY__HEADER__RESULTS_ALL]))
195  parser.add_argument(
196      '--workdir', default=results.DEFAULT_GENERATED_IMAGES_ROOT,
197      help='Directory within which to download images and generate diffs; '
198      'defaults to \'%(default)s\' .')
199  args = parser.parse_args()
200  results_obj = ConfigComparisons(configs=args.config,
201                                  actuals_root=args.actuals,
202                                  generated_images_root=args.workdir)
203  gm_json.WriteToFile(
204      results_obj.get_packaged_results_of_type(results_type=args.results),
205      args.outfile)
206
207
208if __name__ == '__main__':
209  main()
210