1#!/usr/bin/env python
2#
3# Copyright (c) 2013 The Chromium Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Updates the Chrome reference builds.
8
9Usage:
10  $ cd /tmp
11  $ /path/to/update_reference_build.py -r <revision>
12  $ cd reference_builds/reference_builds
13  $ gcl change
14  $ gcl upload <change>
15  $ gcl commit <change>
16"""
17
18import errno
19import logging
20import optparse
21import os
22import shutil
23import subprocess
24import sys
25import time
26import urllib
27import urllib2
28import zipfile
29
30
31class BuildUpdater(object):
32  _PLATFORM_FILES_MAP = {
33    'Win': [
34        'chrome-win32.zip',
35        'chrome-win32-syms.zip',
36        'chrome-win32.test/_pyautolib.pyd',
37        'chrome-win32.test/pyautolib.py',
38    ],
39    'Mac': [
40      'chrome-mac.zip',
41      'chrome-mac.test/_pyautolib.so',
42      'chrome-mac.test/pyautolib.py',
43    ],
44    'Linux': [
45        'chrome-linux.zip',
46    ],
47    'Linux_x64': [
48        'chrome-linux.zip',
49    ],
50  }
51
52  _PLATFORM_DEST_MAP = {
53    'Linux': 'chrome_linux',
54    'Linux_x64': 'chrome_linux64',
55    'Win': 'chrome_win',
56    'Mac': 'chrome_mac',
57   }
58
59  def __init__(self, options):
60    self._platforms = options.platforms.split(',')
61    self._revision = int(options.revision)
62
63  @staticmethod
64  def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
65    """Executes a subprocess and returns its exit code and output.
66
67    Args:
68      args: A string or a sequence of program arguments.
69      cwd: If not None, the subprocess's current directory will be changed to
70        |cwd| before it's executed.
71      shell: Whether to execute args as a shell command.
72
73    Returns:
74      The tuple (exit code, output).
75    """
76    logging.info(str(args) + ' ' + (cwd or ''))
77    p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
78                         stderr=subprocess.PIPE, shell=shell)
79    stdout, stderr = p.communicate()
80    exit_code = p.returncode
81    if stderr:
82      logging.critical(stderr)
83    logging.info(stdout)
84    return (exit_code, stdout)
85
86  def _GetBuildUrl(self, platform, revision, filename):
87    URL_FMT = ('http://commondatastorage.googleapis.com/'
88               'chromium-browser-snapshots/%s/%s/%s')
89    return URL_FMT % (urllib.quote_plus(platform), revision, filename)
90
91  def _FindBuildRevision(self, platform, revision, filename):
92    MAX_REVISIONS_PER_BUILD = 100
93    for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
94      r = urllib2.Request(self._GetBuildUrl(platform, revision_guess, filename))
95      r.get_method = lambda: 'HEAD'
96      try:
97        response = urllib2.urlopen(r)
98        return revision_guess
99      except urllib2.HTTPError, err:
100        if err.code == 404:
101          time.sleep(.1)
102          continue
103    return None
104
105  def _DownloadBuilds(self):
106    for platform in self._platforms:
107      for f in BuildUpdater._PLATFORM_FILES_MAP[platform]:
108        output = os.path.join('dl', platform,
109                              '%s_%s_%s' % (platform, self._revision, f))
110        if os.path.exists(output):
111          logging.info('%s alread exists, skipping download' % output)
112          continue
113        build_revision = self._FindBuildRevision(platform, self._revision, f)
114        if not build_revision:
115          logging.critical('Failed to find %s build for r%s\n' % (
116              platform, self._revision))
117          sys.exit(1)
118        dirname = os.path.dirname(output)
119        if dirname and not os.path.exists(dirname):
120          os.makedirs(dirname)
121        url = self._GetBuildUrl(platform, build_revision, f)
122        logging.info('Downloading %s, saving to %s' % (url, output))
123        r = urllib2.urlopen(url)
124        with file(output, 'wb') as f:
125          f.write(r.read())
126
127  def _FetchSvnRepos(self):
128    if not os.path.exists('reference_builds'):
129      os.makedirs('reference_builds')
130    BuildUpdater._GetCmdStatusAndOutput(
131        ['gclient', 'config',
132         'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
133        'reference_builds')
134    BuildUpdater._GetCmdStatusAndOutput(
135        ['gclient', 'sync'], 'reference_builds')
136
137  def _UnzipFile(self, dl_file, dest_dir):
138    if not zipfile.is_zipfile(dl_file):
139      return False
140    logging.info('Opening %s' % dl_file)
141    with zipfile.ZipFile(dl_file, 'r') as z:
142      for content in z.namelist():
143        dest = os.path.join(dest_dir, content[content.find('/')+1:])
144        if not os.path.basename(dest):
145          if not os.path.isdir(dest):
146            os.makedirs(dest)
147          continue
148        with z.open(content) as unzipped_content:
149          logging.info('Extracting %s to %s (%s)' % (content, dest, dl_file))
150          with file(dest, 'wb') as dest_file:
151            dest_file.write(unzipped_content.read())
152          permissions = z.getinfo(content).external_attr >> 16
153          if permissions:
154            os.chmod(dest, permissions)
155    return True
156
157  def _ClearDir(self, dir):
158    """Clears all files in |dir| except for hidden files and folders."""
159    for root, dirs, files in os.walk(dir):
160      # Skip hidden files and folders (like .svn and .git).
161      files = [f for f in files if f[0] != '.']
162      dirs[:] = [d for d in dirs if d[0] != '.']
163
164      for f in files:
165        os.remove(os.path.join(root, f))
166
167  def _ExtractBuilds(self):
168    for platform in self._platforms:
169      if os.path.exists('tmp_unzip'):
170        os.path.unlink('tmp_unzip')
171      dest_dir = os.path.join('reference_builds', 'reference_builds',
172                              BuildUpdater._PLATFORM_DEST_MAP[platform])
173      self._ClearDir(dest_dir)
174      for root, _, dl_files in os.walk(os.path.join('dl', platform)):
175        for dl_file in dl_files:
176          dl_file = os.path.join(root, dl_file)
177          if not self._UnzipFile(dl_file, dest_dir):
178            logging.info('Copying %s to %s' % (dl_file, dest_dir))
179            shutil.copy(dl_file, dest_dir)
180
181  def _SvnAddAndRemove(self):
182    svn_dir = os.path.join('reference_builds', 'reference_builds')
183    stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat'], svn_dir)[1]
184    for line in stat.splitlines():
185      action, filename = line.split(None, 1)
186      if action == '?':
187        BuildUpdater._GetCmdStatusAndOutput(
188            ['svn', 'add', filename], svn_dir)
189      elif action == '!':
190        BuildUpdater._GetCmdStatusAndOutput(
191            ['svn', 'delete', filename], svn_dir)
192      filepath = os.path.join(svn_dir, filename)
193      if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
194        BuildUpdater._GetCmdStatusAndOutput(
195            ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
196
197  def DownloadAndUpdateBuilds(self):
198    self._DownloadBuilds()
199    self._FetchSvnRepos()
200    self._ExtractBuilds()
201    self._SvnAddAndRemove()
202
203
204def ParseOptions(argv):
205  parser = optparse.OptionParser()
206  usage = 'usage: %prog <options>'
207  parser.set_usage(usage)
208  parser.add_option('-r', dest='revision',
209                    help='Revision to pickup')
210  parser.add_option('-p', dest='platforms',
211                    default='Win,Mac,Linux,Linux_x64',
212                    help='Comma separated list of platforms to download '
213                         '(as defined by the chromium builders).')
214  (options, _) = parser.parse_args(argv)
215  if not options.revision:
216    logging.critical('Must specify -r\n')
217    sys.exit(1)
218
219  return options
220
221
222def main(argv):
223  logging.getLogger().setLevel(logging.DEBUG)
224  options = ParseOptions(argv)
225  b = BuildUpdater(options)
226  b.DownloadAndUpdateBuilds()
227  logging.info('Successfully updated reference builds. Move to '
228               'reference_builds/reference_builds and make a change with gcl.')
229
230if __name__ == '__main__':
231  sys.exit(main(sys.argv))
232