1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Updates the Chrome reference builds.
8
9To update Chromium (unofficial) reference build, use the -r option and give a
10Chromium SVN revision number, like 228977. To update a Chrome (official) build,
11use the -v option and give a Chrome version number like 30.0.1595.0.
12
13If you're getting a Chrome build, you can give the flag --gs to fetch from
14Google Storage. Otherwise, it will be fetched from go/chrome_official_builds.
15
16Before running this script, you should first verify that you are authenticated
17for SVN. You can do this by running:
18  $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
19You may need to get your SVN password from https://chromium-access.appspot.com/.
20
21Usage:
22  $ cd /tmp
23  $ /path/to/update_reference_build.py --gs -v <version>
24  $ cd reference_builds/reference_builds
25  $ gcl change
26  $ gcl upload <change>
27  $ gcl commit <change>
28"""
29
30import logging
31import optparse
32import os
33import shutil
34import subprocess
35import sys
36import time
37import urllib
38import urllib2
39import zipfile
40
41# Example chromium build location:
42# gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
43CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
44                    'chromium-browser-snapshots/%s/%s/%s')
45
46# Chrome official build storage
47# https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
48
49# Internal Google archive of official Chrome builds, example:
50# https://goto.google.com/chrome_official_builds/
51# 32.0.1677.0/precise32bit/chrome-precise32bit.zip
52CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/'
53                           'official_builds/%s/%s/%s')
54
55# Google storage location (no public web URL's), example:
56# gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
57CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s')
58
59
60class BuildUpdater(object):
61  _PLATFORM_FILES_MAP = {
62      'Win': [
63          'chrome-win32.zip',
64          'chrome-win32-syms.zip',
65      ],
66      'Mac': [
67          'chrome-mac.zip',
68      ],
69      'Linux': [
70          'chrome-linux.zip',
71      ],
72      'Linux_x64': [
73          'chrome-linux.zip',
74      ],
75  }
76
77  _CHROME_PLATFORM_FILES_MAP = {
78      'Win': [
79          'chrome-win32.zip',
80          'chrome-win32-syms.zip',
81      ],
82      'Mac': [
83          'chrome-mac.zip',
84      ],
85      'Linux': [
86          'chrome-precise32bit.zip',
87      ],
88      'Linux_x64': [
89          'chrome-precise64bit.zip',
90      ],
91  }
92
93  # Map of platform names to gs:// Chrome build names.
94  _BUILD_PLATFORM_MAP = {
95      'Linux': 'precise32bit',
96      'Linux_x64': 'precise64bit',
97      'Win': 'win',
98      'Mac': 'mac',
99  }
100
101  _PLATFORM_DEST_MAP = {
102      'Linux': 'chrome_linux',
103      'Linux_x64': 'chrome_linux64',
104      'Win': 'chrome_win',
105      'Mac': 'chrome_mac',
106  }
107
108  def __init__(self, options):
109    self._platforms = options.platforms.split(',')
110    self._version_or_revision = options.version or int(options.revision)
111    self._use_official_version = bool(options.version)
112    self._use_gs = options.use_gs
113
114  @staticmethod
115  def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
116    """Executes a subprocess and returns its exit code and output.
117
118    Args:
119      args: A string or a sequence of program arguments.
120      cwd: If not None, the subprocess's current directory will be changed to
121        |cwd| before it's executed.
122      shell: Whether to execute args as a shell command.
123
124    Returns:
125      The tuple (exit code, output).
126    """
127    logging.info(str(args) + ' ' + (cwd or ''))
128    p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
129                         stderr=subprocess.PIPE, shell=shell)
130    stdout, stderr = p.communicate()
131    exit_code = p.returncode
132    if stderr:
133      logging.critical(stderr)
134    logging.info(stdout)
135    return (exit_code, stdout)
136
137  def _GetBuildUrl(self, platform, version_or_revision, filename):
138    """Returns the URL for fetching one file.
139
140    Args:
141      platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
142      version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome
143          version number, e.g. 30.0.1600.1.
144      filename: Name of the file to fetch.
145
146    Returns:
147      The URL for fetching a file. This may be a GS or HTTP URL.
148    """
149    if self._use_official_version:
150      # Chrome Google storage bucket.
151      version = version_or_revision
152      if self._use_gs:
153        release = version[:version.find('.')]
154        return (CHROME_GS_URL_FMT % (
155            release,
156            version,
157            self._BUILD_PLATFORM_MAP[platform],
158            filename))
159      # Chrome internal archive.
160      return (CHROME_INTERNAL_URL_FMT % (
161          version,
162          self._BUILD_PLATFORM_MAP[platform],
163          filename))
164    # Chromium archive.
165    revision = version_or_revision
166    return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename)
167
168  def _FindBuildVersionOrRevision(
169      self, platform, version_or_revision, filename):
170    """Searches for a version or revision where a filename can be found.
171
172    Args:
173      platform: Platform name.
174      version_or_revision: Either Chrome version or Chromium revision.
175      filename: Filename to look for.
176
177    Returns:
178      A version or revision where the file could be found, or None.
179    """
180    # TODO(shadi): Iterate over official versions to find a valid one.
181    if self._use_official_version:
182      version = version_or_revision
183      return (version
184              if self._DoesBuildExist(platform, version, filename) else None)
185
186    revision = version_or_revision
187    MAX_REVISIONS_PER_BUILD = 100
188    for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
189      if self._DoesBuildExist(platform, revision_guess, filename):
190        return revision_guess
191      else:
192        time.sleep(.1)
193    return None
194
195  def _DoesBuildExist(self, platform, version, filename):
196    """Checks whether a file can be found for the given Chrome version.
197
198    Args:
199      platform: Platform name.
200      version: Chrome version number, e.g. 30.0.1600.1.
201      filename: Filename to look for.
202
203    Returns:
204      True if the file could be found, False otherwise.
205    """
206    url = self._GetBuildUrl(platform, version, filename)
207    if self._use_gs:
208      return self._DoesGSFileExist(url)
209
210    request = urllib2.Request(url)
211    request.get_method = lambda: 'HEAD'
212    try:
213      urllib2.urlopen(request)
214      return True
215    except urllib2.HTTPError, err:
216      if err.code == 404:
217        return False
218
219  def _DoesGSFileExist(self, gs_file_name):
220    """Returns True if the GS file can be found, False otherwise."""
221    exit_code = BuildUpdater._GetCmdStatusAndOutput(
222        ['gsutil', 'ls', gs_file_name])[0]
223    return not exit_code
224
225  def _GetPlatformFiles(self, platform):
226    """Returns a list of filenames to fetch for the given platform."""
227    if self._use_official_version:
228      return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
229    return BuildUpdater._PLATFORM_FILES_MAP[platform]
230
231  def _DownloadBuilds(self):
232    for platform in self._platforms:
233      for filename in self._GetPlatformFiles(platform):
234        output = os.path.join('dl', platform,
235                              '%s_%s_%s' % (platform,
236                                            self._version_or_revision,
237                                            filename))
238        if os.path.exists(output):
239          logging.info('%s alread exists, skipping download', output)
240          continue
241        version_or_revision = self._FindBuildVersionOrRevision(
242            platform, self._version_or_revision, filename)
243        if not version_or_revision:
244          logging.critical('Failed to find %s build for r%s\n', platform,
245                           self._version_or_revision)
246          sys.exit(1)
247        dirname = os.path.dirname(output)
248        if dirname and not os.path.exists(dirname):
249          os.makedirs(dirname)
250        url = self._GetBuildUrl(platform, version_or_revision, filename)
251        self._DownloadFile(url, output)
252
253  def _DownloadFile(self, url, output):
254    logging.info('Downloading %s, saving to %s', url, output)
255    if self._use_official_version and self._use_gs:
256      BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
257    else:
258      response = urllib2.urlopen(url)
259      with file(output, 'wb') as f:
260        f.write(response.read())
261
262  def _FetchSvnRepos(self):
263    if not os.path.exists('reference_builds'):
264      os.makedirs('reference_builds')
265    BuildUpdater._GetCmdStatusAndOutput(
266        ['gclient', 'config',
267         'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
268        'reference_builds')
269    BuildUpdater._GetCmdStatusAndOutput(
270        ['gclient', 'sync'], 'reference_builds')
271
272  def _UnzipFile(self, dl_file, dest_dir):
273    """Unzips a file if it is a zip file.
274
275    Args:
276      dl_file: The downloaded file to unzip.
277      dest_dir: The destination directory to unzip to.
278
279    Returns:
280      True if the file was unzipped. False if it wasn't a zip file.
281    """
282    if not zipfile.is_zipfile(dl_file):
283      return False
284    logging.info('Opening %s', dl_file)
285    with zipfile.ZipFile(dl_file, 'r') as z:
286      for content in z.namelist():
287        dest = os.path.join(dest_dir, content[content.find('/')+1:])
288        # Create dest parent dir if it does not exist.
289        if not os.path.isdir(os.path.dirname(dest)):
290          os.makedirs(os.path.dirname(dest))
291        # If dest is just a dir listing, do nothing.
292        if not os.path.basename(dest):
293          continue
294        if not os.path.isdir(os.path.dirname(dest)):
295          os.makedirs(os.path.dirname(dest))
296        with z.open(content) as unzipped_content:
297          logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
298          with file(dest, 'wb') as dest_file:
299            dest_file.write(unzipped_content.read())
300          permissions = z.getinfo(content).external_attr >> 16
301          if permissions:
302            os.chmod(dest, permissions)
303    return True
304
305  def _ClearDir(self, dir):
306    """Clears all files in |dir| except for hidden files and folders."""
307    for root, dirs, files in os.walk(dir):
308      # Skip hidden files and folders (like .svn and .git).
309      files = [f for f in files if f[0] != '.']
310      dirs[:] = [d for d in dirs if d[0] != '.']
311
312      for f in files:
313        os.remove(os.path.join(root, f))
314
315  def _ExtractBuilds(self):
316    for platform in self._platforms:
317      if os.path.exists('tmp_unzip'):
318        os.path.unlink('tmp_unzip')
319      dest_dir = os.path.join('reference_builds', 'reference_builds',
320                              BuildUpdater._PLATFORM_DEST_MAP[platform])
321      self._ClearDir(dest_dir)
322      for root, _, dl_files in os.walk(os.path.join('dl', platform)):
323        for dl_file in dl_files:
324          dl_file = os.path.join(root, dl_file)
325          if not self._UnzipFile(dl_file, dest_dir):
326            logging.info('Copying %s to %s', dl_file, dest_dir)
327            shutil.copy(dl_file, dest_dir)
328
329  def _SvnAddAndRemove(self):
330    svn_dir = os.path.join('reference_builds', 'reference_builds')
331    # List all changes without ignoring any files.
332    stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
333                                               svn_dir)[1]
334    for line in stat.splitlines():
335      action, filename = line.split(None, 1)
336      # Add new and ignored files.
337      if action == '?' or action == 'I':
338        BuildUpdater._GetCmdStatusAndOutput(
339            ['svn', 'add', filename], svn_dir)
340      elif action == '!':
341        BuildUpdater._GetCmdStatusAndOutput(
342            ['svn', 'delete', filename], svn_dir)
343      filepath = os.path.join(svn_dir, filename)
344      if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
345        BuildUpdater._GetCmdStatusAndOutput(
346            ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
347
348  def DownloadAndUpdateBuilds(self):
349    self._DownloadBuilds()
350    self._FetchSvnRepos()
351    self._ExtractBuilds()
352    self._SvnAddAndRemove()
353
354
355def ParseOptions(argv):
356  parser = optparse.OptionParser()
357  usage = 'usage: %prog <options>'
358  parser.set_usage(usage)
359  parser.add_option('-v', dest='version',
360                    help='Chrome official version to pick up '
361                         '(e.g. 30.0.1600.1).')
362  parser.add_option('--gs', dest='use_gs', action='store_true', default=False,
363                    help='Use Google storage for official builds. Used with -b '
364                         'option. Default is false (i.e. use internal storage.')
365  parser.add_option('-p', dest='platforms',
366                    default='Win,Mac,Linux,Linux_x64',
367                    help='Comma separated list of platforms to download '
368                         '(as defined by the chromium builders).')
369  parser.add_option('-r', dest='revision',
370                    help='Chromium revision to pick up (e.g. 234567).')
371
372  (options, _) = parser.parse_args(argv)
373  if not options.revision and not options.version:
374    logging.critical('Must specify either -r or -v.\n')
375    sys.exit(1)
376  if options.revision and options.version:
377    logging.critical('Must specify either -r or -v but not both.\n')
378    sys.exit(1)
379  if options.use_gs and not options.version:
380    logging.critical('Can only use --gs with -v option.\n')
381    sys.exit(1)
382
383  return options
384
385
386def main(argv):
387  logging.getLogger().setLevel(logging.DEBUG)
388  options = ParseOptions(argv)
389  b = BuildUpdater(options)
390  b.DownloadAndUpdateBuilds()
391  logging.info('Successfully updated reference builds. Move to '
392               'reference_builds/reference_builds and make a change with gcl.')
393
394if __name__ == '__main__':
395  sys.exit(main(sys.argv))
396