1#!/usr/bin/python
2
3"""
4Copyright 2014 Google Inc.
5
6Use of this source code is governed by a BSD-style license that can be
7found in the LICENSE file.
8
9Download actual GM results for a particular builder.
10"""
11
12# System-level imports
13import optparse
14import os
15import posixpath
16import re
17import urllib2
18
19# Must fix up PYTHONPATH before importing from within Skia
20import rs_fixpypath  # pylint: disable=W0611
21
22# Imports from within Skia
23from py.utils import gs_utils
24from py.utils import url_utils
25import buildbot_globals
26import gm_json
27
28
29GM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket')
30DEFAULT_ACTUALS_BASE_URL = (
31    'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET)
32DEFAULT_JSON_FILENAME = 'actual-results.json'
33
34
35class Download(object):
36
37  def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL,
38               json_filename=DEFAULT_JSON_FILENAME,
39               gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL):
40    """
41    Args:
42      actuals_base_url: URL pointing at the root directory
43          containing all actual-results.json files, e.g.,
44          http://domain.name/path/to/dir  OR
45          file:///absolute/path/to/localdir
46      json_filename: The JSON filename to read from within each directory.
47      gm_actuals_root_url: Base URL under which the actually-generated-by-bots
48          GM images are stored.
49    """
50    self._actuals_base_url = actuals_base_url
51    self._json_filename = json_filename
52    self._gm_actuals_root_url = gm_actuals_root_url
53    self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN)
54
55  def fetch(self, builder_name, dest_dir):
56    """ Downloads actual GM results for a particular builder.
57
58    Args:
59      builder_name: which builder to download results of
60      dest_dir: path to directory where the image files will be written;
61                if the directory does not exist yet, it will be created
62
63    TODO(epoger): Display progress info.  Right now, it can take a long time
64    to download all of the results, and there is no indication of progress.
65
66    TODO(epoger): Download multiple images in parallel to speed things up.
67    """
68    json_url = posixpath.join(self._actuals_base_url, builder_name,
69                              self._json_filename)
70    json_contents = urllib2.urlopen(json_url).read()
71    results_dict = gm_json.LoadFromString(json_contents)
72
73    actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS]
74    for result_type in sorted(actual_results_dict.keys()):
75      results_of_this_type = actual_results_dict[result_type]
76      if not results_of_this_type:
77        continue
78      for image_name in sorted(results_of_this_type.keys()):
79        (test, config) = self._image_filename_re.match(image_name).groups()
80        (hash_type, hash_digest) = results_of_this_type[image_name]
81        source_url = gm_json.CreateGmActualUrl(
82            test_name=test, hash_type=hash_type, hash_digest=hash_digest,
83            gm_actuals_root_url=self._gm_actuals_root_url)
84        dest_path = os.path.join(dest_dir, config, test + '.png')
85        url_utils.copy_contents(source_url=source_url, dest_path=dest_path,
86                                create_subdirs_if_needed=True)
87
88
89def get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET):
90  """ Returns the list of builders we have actual results for.
91
92  Args:
93    summaries_bucket: Google Cloud Storage bucket containing the summary
94        JSON files
95  """
96  dirs, _ = gs_utils.GSUtils().list_bucket_contents(bucket=GM_SUMMARIES_BUCKET)
97  return dirs
98
99
100def main():
101  parser = optparse.OptionParser()
102  required_params = []
103  parser.add_option('--actuals-base-url',
104                    action='store', type='string',
105                    default=DEFAULT_ACTUALS_BASE_URL,
106                    help=('Base URL from which to read files containing JSON '
107                          'summaries of actual GM results; defaults to '
108                          '"%default".'))
109  required_params.append('builder')
110  # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this
111  # tool downloaded the JSON summaries from skia-autogen, it had the ability
112  # to get results as of a specific revision number.  We should add similar
113  # functionality when retrieving the summaries from Google Storage.
114  parser.add_option('--builder',
115                    action='store', type='string',
116                    help=('REQUIRED: Which builder to download results for. '
117                          'To see a list of builders, run with the '
118                          '--list-builders option set.'))
119  required_params.append('dest_dir')
120  parser.add_option('--dest-dir',
121                    action='store', type='string',
122                    help=('REQUIRED: Directory where all images should be '
123                          'written. If this directory does not exist yet, it '
124                          'will be created.'))
125  parser.add_option('--json-filename',
126                    action='store', type='string',
127                    default=DEFAULT_JSON_FILENAME,
128                    help=('JSON summary filename to read for each builder; '
129                          'defaults to "%default".'))
130  parser.add_option('--list-builders', action='store_true',
131                    help=('List all available builders.'))
132  (params, remaining_args) = parser.parse_args()
133
134  if params.list_builders:
135    print '\n'.join(get_builders_list())
136    return
137
138  # Make sure all required options were set,
139  # and that there were no items left over in the command line.
140  for required_param in required_params:
141    if not getattr(params, required_param):
142      raise Exception('required option \'%s\' was not set' % required_param)
143  if len(remaining_args) is not 0:
144    raise Exception('extra items specified in the command line: %s' %
145                    remaining_args)
146
147  downloader = Download(actuals_base_url=params.actuals_base_url)
148  downloader.fetch(builder_name=params.builder,
149                   dest_dir=params.dest_dir)
150
151
152
153if __name__ == '__main__':
154  main()
155