1#!/usr/bin/env python 2# 3# Copyright (c) 2013 The Chromium Authors. All rights reserved. 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Updates the Chrome reference builds. 8 9Usage: 10 $ cd /tmp 11 $ /path/to/update_reference_build.py -r <revision> 12 $ cd reference_builds/reference_builds 13 $ gcl change 14 $ gcl upload <change> 15 $ gcl commit <change> 16""" 17 18import errno 19import logging 20import optparse 21import os 22import shutil 23import subprocess 24import sys 25import time 26import urllib 27import urllib2 28import zipfile 29 30 31class BuildUpdater(object): 32 _PLATFORM_FILES_MAP = { 33 'Win': [ 34 'chrome-win32.zip', 35 'chrome-win32-syms.zip', 36 'chrome-win32.test/_pyautolib.pyd', 37 'chrome-win32.test/pyautolib.py', 38 ], 39 'Mac': [ 40 'chrome-mac.zip', 41 'chrome-mac.test/_pyautolib.so', 42 'chrome-mac.test/pyautolib.py', 43 ], 44 'Linux': [ 45 'chrome-linux.zip', 46 ], 47 'Linux_x64': [ 48 'chrome-linux.zip', 49 ], 50 } 51 52 _PLATFORM_DEST_MAP = { 53 'Linux': 'chrome_linux', 54 'Linux_x64': 'chrome_linux64', 55 'Win': 'chrome_win', 56 'Mac': 'chrome_mac', 57 } 58 59 def __init__(self, options): 60 self._platforms = options.platforms.split(',') 61 self._revision = int(options.revision) 62 63 @staticmethod 64 def _GetCmdStatusAndOutput(args, cwd=None, shell=False): 65 """Executes a subprocess and returns its exit code and output. 66 67 Args: 68 args: A string or a sequence of program arguments. 69 cwd: If not None, the subprocess's current directory will be changed to 70 |cwd| before it's executed. 71 shell: Whether to execute args as a shell command. 72 73 Returns: 74 The tuple (exit code, output). 75 """ 76 logging.info(str(args) + ' ' + (cwd or '')) 77 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE, 78 stderr=subprocess.PIPE, shell=shell) 79 stdout, stderr = p.communicate() 80 exit_code = p.returncode 81 if stderr: 82 logging.critical(stderr) 83 logging.info(stdout) 84 return (exit_code, stdout) 85 86 def _GetBuildUrl(self, platform, revision, filename): 87 URL_FMT = ('http://commondatastorage.googleapis.com/' 88 'chromium-browser-snapshots/%s/%s/%s') 89 return URL_FMT % (urllib.quote_plus(platform), revision, filename) 90 91 def _FindBuildRevision(self, platform, revision, filename): 92 MAX_REVISIONS_PER_BUILD = 100 93 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD): 94 r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename)) 95 r.get_method = lambda: 'HEAD' 96 try: 97 response = urllib2.urlopen(r) 98 return revision_guess 99 except urllib2.HTTPError, err: 100 if err.code == 404: 101 time.sleep(.1) 102 continue 103 return None 104 105 def _DownloadBuilds(self): 106 for platform in self._platforms: 107 for f in BuildUpdater._PLATFORM_FILES_MAP[platform]: 108 output = os.path.join('dl', platform, 109 '%s_%s_%s' % (platform, self._revision, f)) 110 if os.path.exists(output): 111 logging.info('%s alread exists, skipping download' % output) 112 continue 113 build_revision = self._FindBuildRevision(platform, self._revision, f) 114 if not build_revision: 115 logging.critical('Failed to find %s build for r%s\n' % ( 116 platform, self._revision)) 117 sys.exit(1) 118 dirname = os.path.dirname(output) 119 if dirname and not os.path.exists(dirname): 120 os.makedirs(dirname) 121 url = self._GetBuildUrl(platform, build_revision, f) 122 logging.info('Downloading %s, saving to %s' % (url, output)) 123 r = urllib2.urlopen(url) 124 with file(output, 'wb') as f: 125 f.write(r.read()) 126 127 def _FetchSvnRepos(self): 128 if not os.path.exists('reference_builds'): 129 os.makedirs('reference_builds') 130 BuildUpdater._GetCmdStatusAndOutput( 131 ['gclient', 'config', 132 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'], 133 'reference_builds') 134 BuildUpdater._GetCmdStatusAndOutput( 135 ['gclient', 'sync'], 'reference_builds') 136 137 def _UnzipFile(self, dl_file, dest_dir): 138 if not zipfile.is_zipfile(dl_file): 139 return False 140 logging.info('Opening %s' % dl_file) 141 with zipfile.ZipFile(dl_file, 'r') as z: 142 for content in z.namelist(): 143 dest = os.path.join(dest_dir, content[content.find('/')+1:]) 144 if not os.path.basename(dest): 145 if not os.path.isdir(dest): 146 os.makedirs(dest) 147 continue 148 with z.open(content) as unzipped_content: 149 logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file)) 150 with file(dest, 'wb') as dest_file: 151 dest_file.write(unzipped_content.read()) 152 permissions = z.getinfo(content).external_attr >> 16 153 if permissions: 154 os.chmod(dest, permissions) 155 return True 156 157 def _ClearDir(self, dir): 158 """Clears all files in |dir| except for hidden files and folders.""" 159 for root, dirs, files in os.walk(dir): 160 # Skip hidden files and folders (like .svn and .git). 161 files = [f for f in files if f[0] != '.'] 162 dirs[:] = [d for d in dirs if d[0] != '.'] 163 164 for f in files: 165 os.remove(os.path.join(root, f)) 166 167 def _ExtractBuilds(self): 168 for platform in self._platforms: 169 if os.path.exists('tmp_unzip'): 170 os.path.unlink('tmp_unzip') 171 dest_dir = os.path.join('reference_builds', 'reference_builds', 172 BuildUpdater._PLATFORM_DEST_MAP[platform]) 173 self._ClearDir(dest_dir) 174 for root, _, dl_files in os.walk(os.path.join('dl', platform)): 175 for dl_file in dl_files: 176 dl_file = os.path.join(root, dl_file) 177 if not self._UnzipFile(dl_file, dest_dir): 178 logging.info('Copying %s to %s' % (dl_file, dest_dir)) 179 shutil.copy(dl_file, dest_dir) 180 181 def _SvnAddAndRemove(self): 182 svn_dir = os.path.join('reference_builds', 'reference_builds') 183 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1] 184 for line in stat.splitlines(): 185 action, filename = line.split(None, 1) 186 if action == '?': 187 BuildUpdater._GetCmdStatusAndOutput( 188 ['svn', 'add', filename], svn_dir) 189 elif action == '!': 190 BuildUpdater._GetCmdStatusAndOutput( 191 ['svn', 'delete', filename], svn_dir) 192 filepath = os.path.join(svn_dir, filename) 193 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK): 194 BuildUpdater._GetCmdStatusAndOutput( 195 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir) 196 197 def DownloadAndUpdateBuilds(self): 198 self._DownloadBuilds() 199 self._FetchSvnRepos() 200 self._ExtractBuilds() 201 self._SvnAddAndRemove() 202 203 204def ParseOptions(argv): 205 parser = optparse.OptionParser() 206 usage = 'usage: %prog <options>' 207 parser.set_usage(usage) 208 parser.add_option('-r', dest='revision', 209 help='Revision to pickup') 210 parser.add_option('-p', dest='platforms', 211 default='Win,Mac,Linux,Linux_x64', 212 help='Comma separated list of platforms to download ' 213 '(as defined by the chromium builders).') 214 (options, _) = parser.parse_args(argv) 215 if not options.revision: 216 logging.critical('Must specify -r\n') 217 sys.exit(1) 218 219 return options 220 221 222def main(argv): 223 logging.getLogger().setLevel(logging.DEBUG) 224 options = ParseOptions(argv) 225 b = BuildUpdater(options) 226 b.DownloadAndUpdateBuilds() 227 logging.info('Successfully updated reference builds. Move to ' 228 'reference_builds/reference_builds and make a change with gcl.') 229 230if __name__ == '__main__': 231 sys.exit(main(sys.argv)) 232