1#!/usr/bin/env python 2# 3# Copyright (c) 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9Before running this script, you should first verify that you are authenticated 10for SVN. You can do this by running: 11 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds 12You may need to get your SVN password from https://chromium-access.appspot.com/. 13 14Usage: 15 $ cd /tmp 16 $ /path/to/update_reference_build.py VERSION # e.g. 37.0.2062.94 17 $ cd reference_builds/reference_builds 18 $ gcl change 19 $ gcl upload <change> 20 $ gcl commit <change> 21""" 22 23import logging 24import optparse 25import os 26import shutil 27import subprocess 28import sys 29import time 30import urllib 31import urllib2 32import zipfile 33 34 35# Google storage location (no public web URL's), example: 36# gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip 37CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s') 38 39 40class BuildUpdater(object): 41 _CHROME_PLATFORM_FILES_MAP = { 42 'Win': [ 43 'chrome-win.zip', 44 ], 45 'Mac': [ 46 'chrome-mac.zip', 47 ], 48 'Linux': [ 49 'chrome-precise32.zip', 50 ], 51 'Linux_x64': [ 52 'chrome-precise64.zip', 53 ], 54 } 55 56 # Map of platform names to gs:// Chrome build names. 57 _BUILD_PLATFORM_MAP = { 58 'Linux': 'precise32', 59 'Linux_x64': 'precise64', 60 'Win': 'win', 61 'Mac': 'mac', 62 } 63 64 _PLATFORM_DEST_MAP = { 65 'Linux': 'chrome_linux', 66 'Linux_x64': 'chrome_linux64', 67 'Win': 'chrome_win', 68 'Mac': 'chrome_mac', 69 } 70 71 def __init__(self, version, options): 72 self._version = version 73 self._platforms = options.platforms.split(',') 74 75 @staticmethod 76 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 77 """Executes a subprocess and returns its exit code and output. 78 79 Args: 80 args: A string or a sequence of program arguments. 81 cwd: If not None, the subprocess's current directory will be changed to 82 |cwd| before it's executed. 83 shell: Whether to execute args as a shell command. 84 85 Returns: 86 The tuple (exit code, output). 87 """ 88 logging.info(str(args) + ' ' + (cwd or '')) 89 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 90 stderr=subprocess.PIPE, shell=shell) 91 stdout, stderr = p.communicate() 92 exit_code = p.returncode 93 if stderr: 94 logging.critical(stderr) 95 logging.info(stdout) 96 return (exit_code, stdout) 97 98 def _GetBuildUrl(self, platform, version, filename): 99 """Returns the URL for fetching one file. 100 101 Args: 102 platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|. 103 version: A Chrome version number, e.g. 30.0.1600.1. 104 filename: Name of the file to fetch. 105 106 Returns: 107 The URL for fetching a file. This may be a GS or HTTP URL. 108 """ 109 return CHROME_GS_URL_FMT % ( 110 version, self._BUILD_PLATFORM_MAP[platform], filename) 111 112 def _FindBuildVersion(self, platform, version, filename): 113 """Searches for a version where a filename can be found. 114 115 Args: 116 platform: Platform name. 117 version: A Chrome version number, e.g. 30.0.1600.1. 118 filename: Filename to look for. 119 120 Returns: 121 A version where the file could be found, or None. 122 """ 123 # TODO(shadi): Iterate over official versions to find a valid one. 124 return (version 125 if self._DoesBuildExist(platform, version, filename) else None) 126 127 def _DoesBuildExist(self, platform, version, filename): 128 """Checks whether a file can be found for the given Chrome version. 129 130 Args: 131 platform: Platform name. 132 version: Chrome version number, e.g. 30.0.1600.1. 133 filename: Filename to look for. 134 135 Returns: 136 True if the file could be found, False otherwise. 137 """ 138 url = self._GetBuildUrl(platform, version, filename) 139 return self._DoesGSFileExist(url) 140 141 def _DoesGSFileExist(self, gs_file_name): 142 """Returns True if the GS file can be found, False otherwise.""" 143 exit_code = BuildUpdater._GetCmdStatusAndOutput( 144 ['gsutil', 'ls', gs_file_name])[0] 145 return not exit_code 146 147 def _GetPlatformFiles(self, platform): 148 """Returns a list of filenames to fetch for the given platform.""" 149 return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform] 150 151 def _DownloadBuilds(self): 152 for platform in self._platforms: 153 for filename in self._GetPlatformFiles(platform): 154 output = os.path.join('dl', platform, 155 '%s_%s_%s' % (platform, 156 self._version, 157 filename)) 158 if os.path.exists(output): 159 logging.info('%s alread exists, skipping download', output) 160 continue 161 version = self._FindBuildVersion(platform, self._version, filename) 162 if not version: 163 logging.critical('Failed to find %s build for r%s\n', platform, 164 self._version) 165 sys.exit(1) 166 dirname = os.path.dirname(output) 167 if dirname and not os.path.exists(dirname): 168 os.makedirs(dirname) 169 url = self._GetBuildUrl(platform, version, filename) 170 self._DownloadFile(url, output) 171 172 def _DownloadFile(self, url, output): 173 logging.info('Downloading %s, saving to %s', url, output) 174 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output]) 175 176 def _FetchSvnRepos(self): 177 if not os.path.exists('reference_builds'): 178 os.makedirs('reference_builds') 179 BuildUpdater._GetCmdStatusAndOutput( 180 ['gclient', 'config', 181 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 182 'reference_builds') 183 BuildUpdater._GetCmdStatusAndOutput( 184 ['gclient', 'sync'], 'reference_builds') 185 186 def _UnzipFile(self, dl_file, dest_dir): 187 """Unzips a file if it is a zip file. 188 189 Args: 190 dl_file: The downloaded file to unzip. 191 dest_dir: The destination directory to unzip to. 192 193 Returns: 194 True if the file was unzipped. False if it wasn't a zip file. 195 """ 196 if not zipfile.is_zipfile(dl_file): 197 return False 198 logging.info('Opening %s', dl_file) 199 with zipfile.ZipFile(dl_file, 'r') as z: 200 for content in z.namelist(): 201 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 202 # Create dest parent dir if it does not exist. 203 if not os.path.isdir(os.path.dirname(dest)): 204 os.makedirs(os.path.dirname(dest)) 205 # If dest is just a dir listing, do nothing. 206 if not os.path.basename(dest): 207 continue 208 if not os.path.isdir(os.path.dirname(dest)): 209 os.makedirs(os.path.dirname(dest)) 210 with z.open(content) as unzipped_content: 211 logging.info('Extracting %s to %s (%s)', content, dest, dl_file) 212 with file(dest, 'wb') as dest_file: 213 dest_file.write(unzipped_content.read()) 214 permissions = z.getinfo(content).external_attr >> 16 215 if permissions: 216 os.chmod(dest, permissions) 217 return True 218 219 def _ClearDir(self, dir): 220 """Clears all files in |dir| except for hidden files and folders.""" 221 for root, dirs, files in os.walk(dir): 222 # Skip hidden files and folders (like .svn and .git). 223 files = [f for f in files if f[0] != '.'] 224 dirs[:] = [d for d in dirs if d[0] != '.'] 225 226 for f in files: 227 os.remove(os.path.join(root, f)) 228 229 def _ExtractBuilds(self): 230 for platform in self._platforms: 231 if os.path.exists('tmp_unzip'): 232 os.path.unlink('tmp_unzip') 233 dest_dir = os.path.join('reference_builds', 'reference_builds', 234 BuildUpdater._PLATFORM_DEST_MAP[platform]) 235 self._ClearDir(dest_dir) 236 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 237 for dl_file in dl_files: 238 dl_file = os.path.join(root, dl_file) 239 if not self._UnzipFile(dl_file, dest_dir): 240 logging.info('Copying %s to %s', dl_file, dest_dir) 241 shutil.copy(dl_file, dest_dir) 242 243 def _SvnAddAndRemove(self): 244 svn_dir = os.path.join('reference_builds', 'reference_builds') 245 # List all changes without ignoring any files. 246 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'], 247 svn_dir)[1] 248 for line in stat.splitlines(): 249 action, filename = line.split(None, 1) 250 # Add new and ignored files. 251 if action == '?' or action == 'I': 252 BuildUpdater._GetCmdStatusAndOutput( 253 ['svn', 'add', filename], svn_dir) 254 elif action == '!': 255 BuildUpdater._GetCmdStatusAndOutput( 256 ['svn', 'delete', filename], svn_dir) 257 filepath = os.path.join(svn_dir, filename) 258 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 259 BuildUpdater._GetCmdStatusAndOutput( 260 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 261 262 def DownloadAndUpdateBuilds(self): 263 self._DownloadBuilds() 264 self._FetchSvnRepos() 265 self._ExtractBuilds() 266 self._SvnAddAndRemove() 267 268 269def ParseOptions(argv): 270 parser = optparse.OptionParser() 271 parser.set_usage('Usage: %prog VERSION [-p PLATFORMS]') 272 parser.add_option('-p', dest='platforms', 273 default='Win,Mac,Linux,Linux_x64', 274 help='Comma separated list of platforms to download ' 275 '(as defined by the chromium builders).') 276 277 options, args = parser.parse_args(argv) 278 if len(args) != 2: 279 parser.print_help() 280 sys.exit(1) 281 version = args[1] 282 283 return version, options 284 285 286def main(argv): 287 logging.getLogger().setLevel(logging.DEBUG) 288 version, options = ParseOptions(argv) 289 b = BuildUpdater(version, options) 290 b.DownloadAndUpdateBuilds() 291 logging.info('Successfully updated reference builds. Move to ' 292 'reference_builds/reference_builds and make a change with gcl.') 293 294if __name__ == '__main__': 295 sys.exit(main(sys.argv)) 296