1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Updates the Chrome reference builds.
8
9Before running this script, you should first verify that you are authenticated
10for SVN. You can do this by running:
11  $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
12You may need to get your SVN password from https://chromium-access.appspot.com/.
13
14Usage:
15  $ cd /tmp
16  $ /path/to/update_reference_build.py VERSION  # e.g. 37.0.2062.94
17  $ cd reference_builds/reference_builds
18  $ gcl change
19  $ gcl upload <change>
20  $ gcl commit <change>
21"""
22
23import logging
24import optparse
25import os
26import shutil
27import subprocess
28import sys
29import time
30import urllib
31import urllib2
32import zipfile
33
34
35# Google storage location (no public web URL's), example:
36# gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
37CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s')
38
39
40class BuildUpdater(object):
41  _CHROME_PLATFORM_FILES_MAP = {
42      'Win': [
43          'chrome-win.zip',
44      ],
45      'Mac': [
46          'chrome-mac.zip',
47      ],
48      'Linux': [
49          'chrome-precise32.zip',
50      ],
51      'Linux_x64': [
52          'chrome-precise64.zip',
53      ],
54  }
55
56  # Map of platform names to gs:// Chrome build names.
57  _BUILD_PLATFORM_MAP = {
58      'Linux': 'precise32',
59      'Linux_x64': 'precise64',
60      'Win': 'win',
61      'Mac': 'mac',
62  }
63
64  _PLATFORM_DEST_MAP = {
65      'Linux': 'chrome_linux',
66      'Linux_x64': 'chrome_linux64',
67      'Win': 'chrome_win',
68      'Mac': 'chrome_mac',
69  }
70
71  def __init__(self, version, options):
72    self._version = version
73    self._platforms = options.platforms.split(',')
74
75  @staticmethod
76  def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
77    """Executes a subprocess and returns its exit code and output.
78
79    Args:
80      args: A string or a sequence of program arguments.
81      cwd: If not None, the subprocess's current directory will be changed to
82        |cwd| before it's executed.
83      shell: Whether to execute args as a shell command.
84
85    Returns:
86      The tuple (exit code, output).
87    """
88    logging.info(str(args) + ' ' + (cwd or ''))
89    p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
90                         stderr=subprocess.PIPE, shell=shell)
91    stdout, stderr = p.communicate()
92    exit_code = p.returncode
93    if stderr:
94      logging.critical(stderr)
95    logging.info(stdout)
96    return (exit_code, stdout)
97
98  def _GetBuildUrl(self, platform, version, filename):
99    """Returns the URL for fetching one file.
100
101    Args:
102      platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
103      version: A Chrome version number, e.g. 30.0.1600.1.
104      filename: Name of the file to fetch.
105
106    Returns:
107      The URL for fetching a file. This may be a GS or HTTP URL.
108    """
109    return CHROME_GS_URL_FMT % (
110        version, self._BUILD_PLATFORM_MAP[platform], filename)
111
112  def _FindBuildVersion(self, platform, version, filename):
113    """Searches for a version where a filename can be found.
114
115    Args:
116      platform: Platform name.
117      version: A Chrome version number, e.g. 30.0.1600.1.
118      filename: Filename to look for.
119
120    Returns:
121      A version where the file could be found, or None.
122    """
123    # TODO(shadi): Iterate over official versions to find a valid one.
124    return (version
125            if self._DoesBuildExist(platform, version, filename) else None)
126
127  def _DoesBuildExist(self, platform, version, filename):
128    """Checks whether a file can be found for the given Chrome version.
129
130    Args:
131      platform: Platform name.
132      version: Chrome version number, e.g. 30.0.1600.1.
133      filename: Filename to look for.
134
135    Returns:
136      True if the file could be found, False otherwise.
137    """
138    url = self._GetBuildUrl(platform, version, filename)
139    return self._DoesGSFileExist(url)
140
141  def _DoesGSFileExist(self, gs_file_name):
142    """Returns True if the GS file can be found, False otherwise."""
143    exit_code = BuildUpdater._GetCmdStatusAndOutput(
144        ['gsutil', 'ls', gs_file_name])[0]
145    return not exit_code
146
147  def _GetPlatformFiles(self, platform):
148    """Returns a list of filenames to fetch for the given platform."""
149    return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
150
151  def _DownloadBuilds(self):
152    for platform in self._platforms:
153      for filename in self._GetPlatformFiles(platform):
154        output = os.path.join('dl', platform,
155                              '%s_%s_%s' % (platform,
156                                            self._version,
157                                            filename))
158        if os.path.exists(output):
159          logging.info('%s alread exists, skipping download', output)
160          continue
161        version = self._FindBuildVersion(platform, self._version, filename)
162        if not version:
163          logging.critical('Failed to find %s build for r%s\n', platform,
164                           self._version)
165          sys.exit(1)
166        dirname = os.path.dirname(output)
167        if dirname and not os.path.exists(dirname):
168          os.makedirs(dirname)
169        url = self._GetBuildUrl(platform, version, filename)
170        self._DownloadFile(url, output)
171
172  def _DownloadFile(self, url, output):
173    logging.info('Downloading %s, saving to %s', url, output)
174    BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
175
176  def _FetchSvnRepos(self):
177    if not os.path.exists('reference_builds'):
178      os.makedirs('reference_builds')
179    BuildUpdater._GetCmdStatusAndOutput(
180        ['gclient', 'config',
181         'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
182        'reference_builds')
183    BuildUpdater._GetCmdStatusAndOutput(
184        ['gclient', 'sync'], 'reference_builds')
185
186  def _UnzipFile(self, dl_file, dest_dir):
187    """Unzips a file if it is a zip file.
188
189    Args:
190      dl_file: The downloaded file to unzip.
191      dest_dir: The destination directory to unzip to.
192
193    Returns:
194      True if the file was unzipped. False if it wasn't a zip file.
195    """
196    if not zipfile.is_zipfile(dl_file):
197      return False
198    logging.info('Opening %s', dl_file)
199    with zipfile.ZipFile(dl_file, 'r') as z:
200      for content in z.namelist():
201        dest = os.path.join(dest_dir, content[content.find('/')+1:])
202        # Create dest parent dir if it does not exist.
203        if not os.path.isdir(os.path.dirname(dest)):
204          os.makedirs(os.path.dirname(dest))
205        # If dest is just a dir listing, do nothing.
206        if not os.path.basename(dest):
207          continue
208        if not os.path.isdir(os.path.dirname(dest)):
209          os.makedirs(os.path.dirname(dest))
210        with z.open(content) as unzipped_content:
211          logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
212          with file(dest, 'wb') as dest_file:
213            dest_file.write(unzipped_content.read())
214          permissions = z.getinfo(content).external_attr >> 16
215          if permissions:
216            os.chmod(dest, permissions)
217    return True
218
219  def _ClearDir(self, dir):
220    """Clears all files in |dir| except for hidden files and folders."""
221    for root, dirs, files in os.walk(dir):
222      # Skip hidden files and folders (like .svn and .git).
223      files = [f for f in files if f[0] != '.']
224      dirs[:] = [d for d in dirs if d[0] != '.']
225
226      for f in files:
227        os.remove(os.path.join(root, f))
228
229  def _ExtractBuilds(self):
230    for platform in self._platforms:
231      if os.path.exists('tmp_unzip'):
232        os.path.unlink('tmp_unzip')
233      dest_dir = os.path.join('reference_builds', 'reference_builds',
234                              BuildUpdater._PLATFORM_DEST_MAP[platform])
235      self._ClearDir(dest_dir)
236      for root, _, dl_files in os.walk(os.path.join('dl', platform)):
237        for dl_file in dl_files:
238          dl_file = os.path.join(root, dl_file)
239          if not self._UnzipFile(dl_file, dest_dir):
240            logging.info('Copying %s to %s', dl_file, dest_dir)
241            shutil.copy(dl_file, dest_dir)
242
243  def _SvnAddAndRemove(self):
244    svn_dir = os.path.join('reference_builds', 'reference_builds')
245    # List all changes without ignoring any files.
246    stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
247                                               svn_dir)[1]
248    for line in stat.splitlines():
249      action, filename = line.split(None, 1)
250      # Add new and ignored files.
251      if action == '?' or action == 'I':
252        BuildUpdater._GetCmdStatusAndOutput(
253            ['svn', 'add', filename], svn_dir)
254      elif action == '!':
255        BuildUpdater._GetCmdStatusAndOutput(
256            ['svn', 'delete', filename], svn_dir)
257      filepath = os.path.join(svn_dir, filename)
258      if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
259        BuildUpdater._GetCmdStatusAndOutput(
260            ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
261
262  def DownloadAndUpdateBuilds(self):
263    self._DownloadBuilds()
264    self._FetchSvnRepos()
265    self._ExtractBuilds()
266    self._SvnAddAndRemove()
267
268
269def ParseOptions(argv):
270  parser = optparse.OptionParser()
271  parser.set_usage('Usage: %prog VERSION [-p PLATFORMS]')
272  parser.add_option('-p', dest='platforms',
273                    default='Win,Mac,Linux,Linux_x64',
274                    help='Comma separated list of platforms to download '
275                         '(as defined by the chromium builders).')
276
277  options, args = parser.parse_args(argv)
278  if len(args) != 2:
279    parser.print_help()
280    sys.exit(1)
281  version = args[1]
282
283  return version, options
284
285
286def main(argv):
287  logging.getLogger().setLevel(logging.DEBUG)
288  version, options = ParseOptions(argv)
289  b = BuildUpdater(version, options)
290  b.DownloadAndUpdateBuilds()
291  logging.info('Successfully updated reference builds. Move to '
292               'reference_builds/reference_builds and make a change with gcl.')
293
294if __name__ == '__main__':
295  sys.exit(main(sys.argv))
296