1#!/usr/bin/env python 2# 3# Copyright (c) 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9To update Chromium (unofficial) reference build, use the -r option and give a 10Chromium SVN revision number, like 228977. To update a Chrome (official) build, 11use the -v option and give a Chrome version number like 30.0.1595.0. 12 13If you're getting a Chrome build, you can give the flag --gs to fetch from 14Google Storage. Otherwise, it will be fetched from go/chrome_official_builds. 15 16Before running this script, you should first verify that you are authenticated 17for SVN. You can do this by running: 18 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds 19You may need to get your SVN password from https://chromium-access.appspot.com/. 20 21Usage: 22 $ cd /tmp 23 $ /path/to/update_reference_build.py --gs -v <version> 24 $ cd reference_builds/reference_builds 25 $ gcl change 26 $ gcl upload <change> 27 $ gcl commit <change> 28""" 29 30import logging 31import optparse 32import os 33import shutil 34import subprocess 35import sys 36import time 37import urllib 38import urllib2 39import zipfile 40 41# Example chromium build location: 42# gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip 43CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/' 44 'chromium-browser-snapshots/%s/%s/%s') 45 46# Chrome official build storage 47# https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds 48 49# Internal Google archive of official Chrome builds, example: 50# https://goto.google.com/chrome_official_builds/ 51# 32.0.1677.0/precise32bit/chrome-precise32bit.zip 52CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/' 53 'official_builds/%s/%s/%s') 54 55# Google storage location (no public web URL's), example: 56# gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip 57CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s') 58 59 60class BuildUpdater(object): 61 _PLATFORM_FILES_MAP = { 62 'Win': [ 63 'chrome-win32.zip', 64 'chrome-win32-syms.zip', 65 ], 66 'Mac': [ 67 'chrome-mac.zip', 68 ], 69 'Linux': [ 70 'chrome-linux.zip', 71 ], 72 'Linux_x64': [ 73 'chrome-linux.zip', 74 ], 75 } 76 77 _CHROME_PLATFORM_FILES_MAP = { 78 'Win': [ 79 'chrome-win32.zip', 80 'chrome-win32-syms.zip', 81 ], 82 'Mac': [ 83 'chrome-mac.zip', 84 ], 85 'Linux': [ 86 'chrome-precise32bit.zip', 87 ], 88 'Linux_x64': [ 89 'chrome-precise64bit.zip', 90 ], 91 } 92 93 # Map of platform names to gs:// Chrome build names. 94 _BUILD_PLATFORM_MAP = { 95 'Linux': 'precise32bit', 96 'Linux_x64': 'precise64bit', 97 'Win': 'win', 98 'Mac': 'mac', 99 } 100 101 _PLATFORM_DEST_MAP = { 102 'Linux': 'chrome_linux', 103 'Linux_x64': 'chrome_linux64', 104 'Win': 'chrome_win', 105 'Mac': 'chrome_mac', 106 } 107 108 def __init__(self, options): 109 self._platforms = options.platforms.split(',') 110 self._version_or_revision = options.version or int(options.revision) 111 self._use_official_version = bool(options.version) 112 self._use_gs = options.use_gs 113 114 @staticmethod 115 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 116 """Executes a subprocess and returns its exit code and output. 117 118 Args: 119 args: A string or a sequence of program arguments. 120 cwd: If not None, the subprocess's current directory will be changed to 121 |cwd| before it's executed. 122 shell: Whether to execute args as a shell command. 123 124 Returns: 125 The tuple (exit code, output). 126 """ 127 logging.info(str(args) + ' ' + (cwd or '')) 128 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 129 stderr=subprocess.PIPE, shell=shell) 130 stdout, stderr = p.communicate() 131 exit_code = p.returncode 132 if stderr: 133 logging.critical(stderr) 134 logging.info(stdout) 135 return (exit_code, stdout) 136 137 def _GetBuildUrl(self, platform, version_or_revision, filename): 138 """Returns the URL for fetching one file. 139 140 Args: 141 platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|. 142 version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome 143 version number, e.g. 30.0.1600.1. 144 filename: Name of the file to fetch. 145 146 Returns: 147 The URL for fetching a file. This may be a GS or HTTP URL. 148 """ 149 if self._use_official_version: 150 # Chrome Google storage bucket. 151 version = version_or_revision 152 if self._use_gs: 153 release = version[:version.find('.')] 154 return (CHROME_GS_URL_FMT % ( 155 release, 156 version, 157 self._BUILD_PLATFORM_MAP[platform], 158 filename)) 159 # Chrome internal archive. 160 return (CHROME_INTERNAL_URL_FMT % ( 161 version, 162 self._BUILD_PLATFORM_MAP[platform], 163 filename)) 164 # Chromium archive. 165 revision = version_or_revision 166 return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename) 167 168 def _FindBuildVersionOrRevision( 169 self, platform, version_or_revision, filename): 170 """Searches for a version or revision where a filename can be found. 171 172 Args: 173 platform: Platform name. 174 version_or_revision: Either Chrome version or Chromium revision. 175 filename: Filename to look for. 176 177 Returns: 178 A version or revision where the file could be found, or None. 179 """ 180 # TODO(shadi): Iterate over official versions to find a valid one. 181 if self._use_official_version: 182 version = version_or_revision 183 return (version 184 if self._DoesBuildExist(platform, version, filename) else None) 185 186 revision = version_or_revision 187 MAX_REVISIONS_PER_BUILD = 100 188 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): 189 if self._DoesBuildExist(platform, revision_guess, filename): 190 return revision_guess 191 else: 192 time.sleep(.1) 193 return None 194 195 def _DoesBuildExist(self, platform, version, filename): 196 """Checks whether a file can be found for the given Chrome version. 197 198 Args: 199 platform: Platform name. 200 version: Chrome version number, e.g. 30.0.1600.1. 201 filename: Filename to look for. 202 203 Returns: 204 True if the file could be found, False otherwise. 205 """ 206 url = self._GetBuildUrl(platform, version, filename) 207 if self._use_gs: 208 return self._DoesGSFileExist(url) 209 210 request = urllib2.Request(url) 211 request.get_method = lambda: 'HEAD' 212 try: 213 urllib2.urlopen(request) 214 return True 215 except urllib2.HTTPError, err: 216 if err.code == 404: 217 return False 218 219 def _DoesGSFileExist(self, gs_file_name): 220 """Returns True if the GS file can be found, False otherwise.""" 221 exit_code = BuildUpdater._GetCmdStatusAndOutput( 222 ['gsutil', 'ls', gs_file_name])[0] 223 return not exit_code 224 225 def _GetPlatformFiles(self, platform): 226 """Returns a list of filenames to fetch for the given platform.""" 227 if self._use_official_version: 228 return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform] 229 return BuildUpdater._PLATFORM_FILES_MAP[platform] 230 231 def _DownloadBuilds(self): 232 for platform in self._platforms: 233 for filename in self._GetPlatformFiles(platform): 234 output = os.path.join('dl', platform, 235 '%s_%s_%s' % (platform, 236 self._version_or_revision, 237 filename)) 238 if os.path.exists(output): 239 logging.info('%s alread exists, skipping download', output) 240 continue 241 version_or_revision = self._FindBuildVersionOrRevision( 242 platform, self._version_or_revision, filename) 243 if not version_or_revision: 244 logging.critical('Failed to find %s build for r%s\n', platform, 245 self._version_or_revision) 246 sys.exit(1) 247 dirname = os.path.dirname(output) 248 if dirname and not os.path.exists(dirname): 249 os.makedirs(dirname) 250 url = self._GetBuildUrl(platform, version_or_revision, filename) 251 self._DownloadFile(url, output) 252 253 def _DownloadFile(self, url, output): 254 logging.info('Downloading %s, saving to %s', url, output) 255 if self._use_official_version and self._use_gs: 256 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output]) 257 else: 258 response = urllib2.urlopen(url) 259 with file(output, 'wb') as f: 260 f.write(response.read()) 261 262 def _FetchSvnRepos(self): 263 if not os.path.exists('reference_builds'): 264 os.makedirs('reference_builds') 265 BuildUpdater._GetCmdStatusAndOutput( 266 ['gclient', 'config', 267 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 268 'reference_builds') 269 BuildUpdater._GetCmdStatusAndOutput( 270 ['gclient', 'sync'], 'reference_builds') 271 272 def _UnzipFile(self, dl_file, dest_dir): 273 """Unzips a file if it is a zip file. 274 275 Args: 276 dl_file: The downloaded file to unzip. 277 dest_dir: The destination directory to unzip to. 278 279 Returns: 280 True if the file was unzipped. False if it wasn't a zip file. 281 """ 282 if not zipfile.is_zipfile(dl_file): 283 return False 284 logging.info('Opening %s', dl_file) 285 with zipfile.ZipFile(dl_file, 'r') as z: 286 for content in z.namelist(): 287 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 288 # Create dest parent dir if it does not exist. 289 if not os.path.isdir(os.path.dirname(dest)): 290 os.makedirs(os.path.dirname(dest)) 291 # If dest is just a dir listing, do nothing. 292 if not os.path.basename(dest): 293 continue 294 if not os.path.isdir(os.path.dirname(dest)): 295 os.makedirs(os.path.dirname(dest)) 296 with z.open(content) as unzipped_content: 297 logging.info('Extracting %s to %s (%s)', content, dest, dl_file) 298 with file(dest, 'wb') as dest_file: 299 dest_file.write(unzipped_content.read()) 300 permissions = z.getinfo(content).external_attr >> 16 301 if permissions: 302 os.chmod(dest, permissions) 303 return True 304 305 def _ClearDir(self, dir): 306 """Clears all files in |dir| except for hidden files and folders.""" 307 for root, dirs, files in os.walk(dir): 308 # Skip hidden files and folders (like .svn and .git). 309 files = [f for f in files if f[0] != '.'] 310 dirs[:] = [d for d in dirs if d[0] != '.'] 311 312 for f in files: 313 os.remove(os.path.join(root, f)) 314 315 def _ExtractBuilds(self): 316 for platform in self._platforms: 317 if os.path.exists('tmp_unzip'): 318 os.path.unlink('tmp_unzip') 319 dest_dir = os.path.join('reference_builds', 'reference_builds', 320 BuildUpdater._PLATFORM_DEST_MAP[platform]) 321 self._ClearDir(dest_dir) 322 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 323 for dl_file in dl_files: 324 dl_file = os.path.join(root, dl_file) 325 if not self._UnzipFile(dl_file, dest_dir): 326 logging.info('Copying %s to %s', dl_file, dest_dir) 327 shutil.copy(dl_file, dest_dir) 328 329 def _SvnAddAndRemove(self): 330 svn_dir = os.path.join('reference_builds', 'reference_builds') 331 # List all changes without ignoring any files. 332 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'], 333 svn_dir)[1] 334 for line in stat.splitlines(): 335 action, filename = line.split(None, 1) 336 # Add new and ignored files. 337 if action == '?' or action == 'I': 338 BuildUpdater._GetCmdStatusAndOutput( 339 ['svn', 'add', filename], svn_dir) 340 elif action == '!': 341 BuildUpdater._GetCmdStatusAndOutput( 342 ['svn', 'delete', filename], svn_dir) 343 filepath = os.path.join(svn_dir, filename) 344 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 345 BuildUpdater._GetCmdStatusAndOutput( 346 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 347 348 def DownloadAndUpdateBuilds(self): 349 self._DownloadBuilds() 350 self._FetchSvnRepos() 351 self._ExtractBuilds() 352 self._SvnAddAndRemove() 353 354 355def ParseOptions(argv): 356 parser = optparse.OptionParser() 357 usage = 'usage: %prog <options>' 358 parser.set_usage(usage) 359 parser.add_option('-v', dest='version', 360 help='Chrome official version to pick up ' 361 '(e.g. 30.0.1600.1).') 362 parser.add_option('--gs', dest='use_gs', action='store_true', default=False, 363 help='Use Google storage for official builds. Used with -b ' 364 'option. Default is false (i.e. use internal storage.') 365 parser.add_option('-p', dest='platforms', 366 default='Win,Mac,Linux,Linux_x64', 367 help='Comma separated list of platforms to download ' 368 '(as defined by the chromium builders).') 369 parser.add_option('-r', dest='revision', 370 help='Chromium revision to pick up (e.g. 234567).') 371 372 (options, _) = parser.parse_args(argv) 373 if not options.revision and not options.version: 374 logging.critical('Must specify either -r or -v.\n') 375 sys.exit(1) 376 if options.revision and options.version: 377 logging.critical('Must specify either -r or -v but not both.\n') 378 sys.exit(1) 379 if options.use_gs and not options.version: 380 logging.critical('Can only use --gs with -v option.\n') 381 sys.exit(1) 382 383 return options 384 385 386def main(argv): 387 logging.getLogger().setLevel(logging.DEBUG) 388 options = ParseOptions(argv) 389 b = BuildUpdater(options) 390 b.DownloadAndUpdateBuilds() 391 logging.info('Successfully updated reference builds. Move to ' 392 'reference_builds/reference_builds and make a change with gcl.') 393 394if __name__ == '__main__': 395 sys.exit(main(sys.argv)) 396