1d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org#!/usr/bin/python 2d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 3d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org""" 4d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgCopyright 2014 Google Inc. 5d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 6d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgUse of this source code is governed by a BSD-style license that can be 7d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgfound in the LICENSE file. 8d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 9d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgDownload actual GM results for a particular builder. 10d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org""" 11d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 12d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org# System-level imports 13d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport optparse 14d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport os 15d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport posixpath 16d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport re 17d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport urllib2 18d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 19d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org# Imports from within Skia 20b144271179aaf82cb1151e9dfd8e866747402594epogerimport fix_pythonpath # must do this first 21b144271179aaf82cb1151e9dfd8e866747402594epogerfrom pyutils import gs_utils 22b144271179aaf82cb1151e9dfd8e866747402594epogerfrom pyutils import url_utils 23d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport buildbot_globals 24d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgimport gm_json 25d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 2655ada0630e1ce1bfd6e6e60b70370384530622ccepoger 2755ada0630e1ce1bfd6e6e60b70370384530622ccepogerGM_SUMMARIES_BUCKET = buildbot_globals.Get('gm_summaries_bucket') 2855ada0630e1ce1bfd6e6e60b70370384530622ccepogerDEFAULT_ACTUALS_BASE_URL = ( 2955ada0630e1ce1bfd6e6e60b70370384530622ccepoger 'http://storage.googleapis.com/%s' % GM_SUMMARIES_BUCKET) 30d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgDEFAULT_JSON_FILENAME = 'actual-results.json' 31d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 32d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 33d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgclass Download(object): 34d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 35d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org def __init__(self, actuals_base_url=DEFAULT_ACTUALS_BASE_URL, 36d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org json_filename=DEFAULT_JSON_FILENAME, 37d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org gm_actuals_root_url=gm_json.GM_ACTUALS_ROOT_HTTP_URL): 38d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org """ 39d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org Args: 40d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org actuals_base_url: URL pointing at the root directory 41d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org containing all actual-results.json files, e.g., 42d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org http://domain.name/path/to/dir OR 43d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org file:///absolute/path/to/localdir 44d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org json_filename: The JSON filename to read from within each directory. 45d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org gm_actuals_root_url: Base URL under which the actually-generated-by-bots 46d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org GM images are stored. 47d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org """ 48d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org self._actuals_base_url = actuals_base_url 49d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org self._json_filename = json_filename 50d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org self._gm_actuals_root_url = gm_actuals_root_url 51d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org self._image_filename_re = re.compile(gm_json.IMAGE_FILENAME_PATTERN) 52d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 53d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org def fetch(self, builder_name, dest_dir): 54d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org """ Downloads actual GM results for a particular builder. 55d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 56d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org Args: 57d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org builder_name: which builder to download results of 58d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org dest_dir: path to directory where the image files will be written; 59d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org if the directory does not exist yet, it will be created 60d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 61d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org TODO(epoger): Display progress info. Right now, it can take a long time 62d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org to download all of the results, and there is no indication of progress. 63d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 64d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org TODO(epoger): Download multiple images in parallel to speed things up. 65d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org """ 66d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org json_url = posixpath.join(self._actuals_base_url, builder_name, 67d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org self._json_filename) 68d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org json_contents = urllib2.urlopen(json_url).read() 69d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org results_dict = gm_json.LoadFromString(json_contents) 70d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 71d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org actual_results_dict = results_dict[gm_json.JSONKEY_ACTUALRESULTS] 72d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org for result_type in sorted(actual_results_dict.keys()): 73d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org results_of_this_type = actual_results_dict[result_type] 74d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org if not results_of_this_type: 75d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org continue 76d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org for image_name in sorted(results_of_this_type.keys()): 77d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org (test, config) = self._image_filename_re.match(image_name).groups() 78d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org (hash_type, hash_digest) = results_of_this_type[image_name] 79d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org source_url = gm_json.CreateGmActualUrl( 80d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org test_name=test, hash_type=hash_type, hash_digest=hash_digest, 81d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org gm_actuals_root_url=self._gm_actuals_root_url) 82d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org dest_path = os.path.join(dest_dir, config, test + '.png') 83b144271179aaf82cb1151e9dfd8e866747402594epoger url_utils.copy_contents(source_url=source_url, dest_path=dest_path, 84b144271179aaf82cb1151e9dfd8e866747402594epoger create_subdirs_if_needed=True) 85d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 86d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 87b144271179aaf82cb1151e9dfd8e866747402594epogerdef get_builders_list(summaries_bucket=GM_SUMMARIES_BUCKET): 88b144271179aaf82cb1151e9dfd8e866747402594epoger """ Returns the list of builders we have actual results for. 89d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 90d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org Args: 91b144271179aaf82cb1151e9dfd8e866747402594epoger summaries_bucket: Google Cloud Storage bucket containing the summary 92b144271179aaf82cb1151e9dfd8e866747402594epoger JSON files 93d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org """ 94b144271179aaf82cb1151e9dfd8e866747402594epoger dirs, _ = gs_utils.list_bucket_contents(bucket=GM_SUMMARIES_BUCKET) 95b144271179aaf82cb1151e9dfd8e866747402594epoger return dirs 9655ada0630e1ce1bfd6e6e60b70370384530622ccepoger 9755ada0630e1ce1bfd6e6e60b70370384530622ccepoger 98d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgdef main(): 99d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org parser = optparse.OptionParser() 100d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org required_params = [] 101d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org parser.add_option('--actuals-base-url', 102d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org action='store', type='string', 103d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org default=DEFAULT_ACTUALS_BASE_URL, 104d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org help=('Base URL from which to read files containing JSON ' 105d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 'summaries of actual GM results; defaults to ' 10655ada0630e1ce1bfd6e6e60b70370384530622ccepoger '"%default".')) 107d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org required_params.append('builder') 10855ada0630e1ce1bfd6e6e60b70370384530622ccepoger # TODO(epoger): Before https://codereview.chromium.org/309653005 , when this 10955ada0630e1ce1bfd6e6e60b70370384530622ccepoger # tool downloaded the JSON summaries from skia-autogen, it had the ability 11055ada0630e1ce1bfd6e6e60b70370384530622ccepoger # to get results as of a specific revision number. We should add similar 11155ada0630e1ce1bfd6e6e60b70370384530622ccepoger # functionality when retrieving the summaries from Google Storage. 112d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org parser.add_option('--builder', 113d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org action='store', type='string', 114d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org help=('REQUIRED: Which builder to download results for. ' 11555ada0630e1ce1bfd6e6e60b70370384530622ccepoger 'To see a list of builders, run with the ' 11655ada0630e1ce1bfd6e6e60b70370384530622ccepoger '--list-builders option set.')) 117d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org required_params.append('dest_dir') 118d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org parser.add_option('--dest-dir', 119d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org action='store', type='string', 120d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org help=('REQUIRED: Directory where all images should be ' 121d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 'written. If this directory does not exist yet, it ' 122d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 'will be created.')) 123d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org parser.add_option('--json-filename', 124d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org action='store', type='string', 125d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org default=DEFAULT_JSON_FILENAME, 126d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org help=('JSON summary filename to read for each builder; ' 127d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 'defaults to "%default".')) 12855ada0630e1ce1bfd6e6e60b70370384530622ccepoger parser.add_option('--list-builders', action='store_true', 12955ada0630e1ce1bfd6e6e60b70370384530622ccepoger help=('List all available builders.')) 130d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org (params, remaining_args) = parser.parse_args() 131d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 13255ada0630e1ce1bfd6e6e60b70370384530622ccepoger if params.list_builders: 133b144271179aaf82cb1151e9dfd8e866747402594epoger print '\n'.join(get_builders_list()) 13455ada0630e1ce1bfd6e6e60b70370384530622ccepoger return 13555ada0630e1ce1bfd6e6e60b70370384530622ccepoger 136d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org # Make sure all required options were set, 137d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org # and that there were no items left over in the command line. 138d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org for required_param in required_params: 139d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org if not getattr(params, required_param): 140d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org raise Exception('required option \'%s\' was not set' % required_param) 141d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org if len(remaining_args) is not 0: 142d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org raise Exception('extra items specified in the command line: %s' % 143d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org remaining_args) 144d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 145d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org downloader = Download(actuals_base_url=params.actuals_base_url) 146d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org downloader.fetch(builder_name=params.builder, 147d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org dest_dir=params.dest_dir) 148d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 149d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 150d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org 151d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.orgif __name__ == '__main__': 152d1630e8fc0eb27dbe75ed0fed6d2b1c7b359bf76commit-bot@chromium.org main() 153