1#!/usr/bin/env python
2# Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS.  All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10"""Script to roll chromium_revision in the WebRTC DEPS file."""
11
12import argparse
13import base64
14import collections
15import logging
16import os
17import re
18import subprocess
19import sys
20import urllib
21
22
23CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
24CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
25CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
26CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
27
28COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
29CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION=(\d+)$')
30ROLL_BRANCH_NAME = 'roll_chromium_revision'
31
32SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
33CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
34                                                  os.pardir))
35sys.path.append(CHECKOUT_ROOT_DIR)
36import setup_links
37
38sys.path.append(os.path.join(CHECKOUT_ROOT_DIR, 'build'))
39import find_depot_tools
40find_depot_tools.add_depot_tools_to_path()
41from gclient import GClientKeywords
42
43CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
44CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join('tools', 'clang', 'scripts',
45                                              'update.py')
46
47DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
48ChangedDep = collections.namedtuple('ChangedDep',
49                                    'path url current_rev new_rev')
50
51
52def ParseDepsDict(deps_content):
53  local_scope = {}
54  var = GClientKeywords.VarImpl({}, local_scope)
55  global_scope = {
56    'File': GClientKeywords.FileImpl,
57    'From': GClientKeywords.FromImpl,
58    'Var': var.Lookup,
59    'deps_os': {},
60  }
61  exec(deps_content, global_scope, local_scope)
62  return local_scope
63
64
65def ParseLocalDepsFile(filename):
66  with open(filename, 'rb') as f:
67    deps_content = f.read()
68  return ParseDepsDict(deps_content)
69
70
71def ParseRemoteCrDepsFile(revision):
72  deps_content = ReadRemoteCrFile('DEPS', revision)
73  return ParseDepsDict(deps_content)
74
75
76def ParseCommitPosition(commit_message):
77  for line in reversed(commit_message.splitlines()):
78    m = COMMIT_POSITION_RE.match(line.strip())
79    if m:
80      return m.group(1)
81  logging.error('Failed to parse commit position id from:\n%s\n',
82                commit_message)
83  sys.exit(-1)
84
85
86def _RunCommand(command, working_dir=None, ignore_exit_code=False,
87                extra_env=None):
88  """Runs a command and returns the output from that command.
89
90  If the command fails (exit code != 0), the function will exit the process.
91
92  Returns:
93    A tuple containing the stdout and stderr outputs as strings.
94  """
95  working_dir = working_dir or CHECKOUT_ROOT_DIR
96  logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
97  env = os.environ.copy()
98  if extra_env:
99    assert all(type(value) == str for value in extra_env.values())
100    logging.debug('extra env: %s', extra_env)
101    env.update(extra_env)
102  p = subprocess.Popen(command, stdout=subprocess.PIPE,
103                       stderr=subprocess.PIPE, env=env,
104                       cwd=working_dir, universal_newlines=True)
105  std_output = p.stdout.read()
106  err_output = p.stderr.read()
107  p.wait()
108  p.stdout.close()
109  p.stderr.close()
110  if not ignore_exit_code and p.returncode != 0:
111    logging.error('Command failed: %s\n'
112                  'stdout:\n%s\n'
113                  'stderr:\n%s\n', ' '.join(command), std_output, err_output)
114    sys.exit(p.returncode)
115  return std_output, err_output
116
117
118def _GetBranches():
119  """Returns a tuple of active,branches.
120
121  The 'active' is the name of the currently active branch and 'branches' is a
122  list of all branches.
123  """
124  lines = _RunCommand(['git', 'branch'])[0].split('\n')
125  branches = []
126  active = ''
127  for line in lines:
128    if '*' in line:
129      # The assumption is that the first char will always be the '*'.
130      active = line[1:].strip()
131      branches.append(active)
132    else:
133      branch = line.strip()
134      if branch:
135        branches.append(branch)
136  return active, branches
137
138
139def _ReadGitilesContent(url):
140  # Download and decode BASE64 content until
141  # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
142  base64_content = ReadUrlContent(url + '?format=TEXT')
143  return base64.b64decode(base64_content[0])
144
145
146def ReadRemoteCrFile(path_below_src, revision):
147  """Reads a remote Chromium file of a specific revision. Returns a string."""
148  return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
149                                                       path_below_src))
150
151
152def ReadRemoteCrCommit(revision):
153  """Reads a remote Chromium commit message. Returns a string."""
154  return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
155
156
157def ReadUrlContent(url):
158  """Connect to a remote host and read the contents. Returns a list of lines."""
159  conn = urllib.urlopen(url)
160  try:
161    return conn.readlines()
162  except IOError as e:
163    logging.exception('Error connecting to %s. Error: %s', url, e)
164    raise
165  finally:
166    conn.close()
167
168
169def GetMatchingDepsEntries(depsentry_dict, dir_path):
170  """Gets all deps entries matching the provided path.
171
172  This list may contain more than one DepsEntry object.
173  Example: dir_path='src/testing' would give results containing both
174  'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
175  Example 2: dir_path='src/build' should return 'src/build' but not
176  'src/buildtools'.
177
178  Returns:
179    A list of DepsEntry objects.
180  """
181  result = []
182  for path, depsentry in depsentry_dict.iteritems():
183    if path == dir_path:
184      result.append(depsentry)
185    else:
186      parts = path.split('/')
187      if all(part == parts[i]
188             for i, part in enumerate(dir_path.split('/'))):
189        result.append(depsentry)
190  return result
191
192
193def BuildDepsentryDict(deps_dict):
194  """Builds a dict of DepsEntry object from a raw parsed deps dict."""
195  result = {}
196  def AddDepsEntries(deps_subdict):
197    for path, deps_url in deps_subdict.iteritems():
198      if not result.has_key(path):
199        url, revision = deps_url.split('@') if deps_url else (None, None)
200        result[path] = DepsEntry(path, url, revision)
201
202  AddDepsEntries(deps_dict['deps'])
203  for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
204    AddDepsEntries(deps_dict['deps_os'].get(deps_os, {}))
205  return result
206
207
208def CalculateChangedDeps(current_deps, new_deps):
209  result = []
210  current_entries = BuildDepsentryDict(current_deps)
211  new_entries = BuildDepsentryDict(new_deps)
212
213  all_deps_dirs = setup_links.DIRECTORIES
214  for deps_dir in all_deps_dirs:
215    # All deps have 'src' prepended to the path in the Chromium DEPS file.
216    dir_path = 'src/%s' % deps_dir
217
218    for entry in GetMatchingDepsEntries(current_entries, dir_path):
219      new_matching_entries = GetMatchingDepsEntries(new_entries, entry.path)
220      assert len(new_matching_entries) <= 1, (
221          'Should never find more than one entry matching %s in %s, found %d' %
222          (entry.path, new_entries, len(new_matching_entries)))
223      if not new_matching_entries:
224        result.append(ChangedDep(entry.path, entry.url, entry.revision, 'None'))
225      elif entry != new_matching_entries[0]:
226        result.append(ChangedDep(entry.path, entry.url, entry.revision,
227                                 new_matching_entries[0].revision))
228  return result
229
230
231def CalculateChangedClang(new_cr_rev):
232  def GetClangRev(lines):
233    for line in lines:
234      match = CLANG_REVISION_RE.match(line)
235      if match:
236        return match.group(1)
237    return None
238
239  chromium_src_path = os.path.join(CHECKOUT_ROOT_DIR, 'chromium', 'src',
240                                   CLANG_UPDATE_SCRIPT_LOCAL_PATH)
241  with open(chromium_src_path, 'rb') as f:
242    current_lines = f.readlines()
243  current_rev = GetClangRev(current_lines)
244
245  new_clang_update_sh = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
246                                         new_cr_rev).splitlines()
247  new_rev = GetClangRev(new_clang_update_sh)
248  return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
249
250
251def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
252                          new_commit_pos, changed_deps_list, clang_change):
253  current_cr_rev = current_cr_rev[0:7]
254  new_cr_rev = new_cr_rev[0:7]
255  rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
256  git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
257
258  commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
259                                                    git_number_interval)]
260  commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
261  commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
262                                         rev_interval))
263  # TBR field will be empty unless in some custom cases, where some engineers
264  # are added.
265  tbr_authors = ''
266  if changed_deps_list:
267    commit_msg.append('Changed dependencies:')
268
269    for c in changed_deps_list:
270      commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
271                                                  c.current_rev[0:7],
272                                                  c.new_rev[0:7]))
273      if 'libvpx' in c.path:
274        tbr_authors += 'marpan@webrtc.org, stefan@webrtc.org, '
275
276    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
277    commit_msg.append('DEPS diff: %s\n' % change_url)
278  else:
279    commit_msg.append('No dependencies changed.')
280
281  if clang_change.current_rev != clang_change.new_rev:
282    commit_msg.append('Clang version changed %s:%s' %
283                      (clang_change.current_rev, clang_change.new_rev))
284    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
285                                           CLANG_UPDATE_SCRIPT_URL_PATH)
286    commit_msg.append('Details: %s\n' % change_url)
287    tbr_authors += 'pbos@webrtc.org'
288  else:
289    commit_msg.append('No update to Clang.\n')
290
291  commit_msg.append('TBR=%s' % tbr_authors)
292  return '\n'.join(commit_msg)
293
294
295def UpdateDeps(deps_filename, old_cr_revision, new_cr_revision):
296  """Update the DEPS file with the new revision."""
297  with open(deps_filename, 'rb') as deps_file:
298    deps_content = deps_file.read()
299  deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
300  with open(deps_filename, 'wb') as deps_file:
301    deps_file.write(deps_content)
302
303def _IsTreeClean():
304  stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
305  if len(stdout) == 0:
306    return True
307
308  logging.error('Dirty/unversioned files:\n%s', stdout)
309  return False
310
311
312def _EnsureUpdatedMasterBranch(dry_run):
313  current_branch = _RunCommand(
314      ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
315  if current_branch != 'master':
316    logging.error('Please checkout the master branch and re-run this script.')
317    if not dry_run:
318      sys.exit(-1)
319
320  logging.info('Updating master branch...')
321  _RunCommand(['git', 'pull'])
322
323
324def _CreateRollBranch(dry_run):
325  logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
326  if not dry_run:
327    _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
328
329
330def _RemovePreviousRollBranch(dry_run):
331  active_branch, branches = _GetBranches()
332  if active_branch == ROLL_BRANCH_NAME:
333    active_branch = 'master'
334  if ROLL_BRANCH_NAME in branches:
335    logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
336    if not dry_run:
337      _RunCommand(['git', 'checkout', active_branch])
338      _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
339
340
341def _LocalCommit(commit_msg, dry_run):
342  logging.info('Committing changes locally.')
343  if not dry_run:
344    _RunCommand(['git', 'add', '--update', '.'])
345    _RunCommand(['git', 'commit', '-m', commit_msg])
346
347
348def _UploadCL(dry_run, rietveld_email=None):
349  logging.info('Uploading CL...')
350  if not dry_run:
351    cmd = ['git', 'cl', 'upload', '-f']
352    if rietveld_email:
353      cmd.append('--email=%s' % rietveld_email)
354    _RunCommand(cmd, extra_env={'EDITOR': 'true'})
355
356
357def _LaunchTrybots(dry_run, skip_try):
358  logging.info('Sending tryjobs...')
359  if not dry_run and not skip_try:
360    _RunCommand(['git', 'cl', 'try'])
361
362
363def _SendToCQ(dry_run, skip_cq):
364  logging.info('Sending the CL to the CQ...')
365  if not dry_run and not skip_cq:
366    _RunCommand(['git', 'cl', 'set_commit'])
367    logging.info('Sent the CL to the CQ.')
368
369
370def main():
371  p = argparse.ArgumentParser()
372  p.add_argument('--clean', action='store_true', default=False,
373                 help='Removes any previous local roll branch.')
374  p.add_argument('-r', '--revision',
375                 help=('Chromium Git revision to roll to. Defaults to the '
376                       'Chromium HEAD revision if omitted.'))
377  p.add_argument('-u', '--rietveld-email',
378                 help=('E-mail address to use for creating the CL at Rietveld'
379                       'If omitted a previously cached one will be used or an '
380                       'error will be thrown during upload.'))
381  p.add_argument('--dry-run', action='store_true', default=False,
382                 help=('Calculate changes and modify DEPS, but don\'t create '
383                       'any local branch, commit, upload CL or send any '
384                       'tryjobs.'))
385  p.add_argument('--allow-reverse', action='store_true', default=False,
386                 help=('Allow rolling back in time (disabled by default but '
387                       'may be useful to be able do to manually).'))
388  p.add_argument('-s', '--skip-try', action='store_true', default=False,
389                 help='Skip sending tryjobs (default: %(default)s)')
390  p.add_argument('--skip-cq', action='store_true', default=False,
391                 help='Skip sending the CL to the CQ (default: %(default)s)')
392  p.add_argument('-v', '--verbose', action='store_true', default=False,
393                 help='Be extra verbose in printing of log messages.')
394  opts = p.parse_args()
395
396  if opts.verbose:
397    logging.basicConfig(level=logging.DEBUG)
398  else:
399    logging.basicConfig(level=logging.INFO)
400
401  if not _IsTreeClean():
402    logging.error('Please clean your local checkout first.')
403    return 1
404
405  if opts.clean:
406    _RemovePreviousRollBranch(opts.dry_run)
407
408  _EnsureUpdatedMasterBranch(opts.dry_run)
409
410  new_cr_rev = opts.revision
411  if not new_cr_rev:
412    stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
413    head_rev = stdout.strip().split('\t')[0]
414    logging.info('No revision specified. Using HEAD: %s', head_rev)
415    new_cr_rev = head_rev
416
417  deps_filename = os.path.join(CHECKOUT_ROOT_DIR, 'DEPS')
418  local_deps = ParseLocalDepsFile(deps_filename)
419  current_cr_rev = local_deps['vars']['chromium_revision']
420
421  current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
422  new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
423
424  current_cr_deps = ParseRemoteCrDepsFile(current_cr_rev)
425  new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
426
427  if new_commit_pos > current_commit_pos or opts.allow_reverse:
428    changed_deps = sorted(CalculateChangedDeps(current_cr_deps, new_cr_deps))
429    clang_change = CalculateChangedClang(new_cr_rev)
430    commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
431                                       current_commit_pos, new_commit_pos,
432                                       changed_deps, clang_change)
433    logging.debug('Commit message:\n%s', commit_msg)
434  else:
435    logging.info('Currently pinned chromium_revision: %s (#%s) is newer than '
436                 '%s (#%s). To roll to older revisions, you must pass the '
437                 '--allow-reverse flag.\n'
438                 'Aborting without action.', current_cr_rev, current_commit_pos,
439                 new_cr_rev, new_commit_pos)
440    return 0
441
442  _CreateRollBranch(opts.dry_run)
443  UpdateDeps(deps_filename, current_cr_rev, new_cr_rev)
444  _LocalCommit(commit_msg, opts.dry_run)
445  _UploadCL(opts.dry_run, opts.rietveld_email)
446  _LaunchTrybots(opts.dry_run, opts.skip_try)
447  _SendToCQ(opts.dry_run, opts.skip_cq)
448  return 0
449
450
451if __name__ == '__main__':
452  sys.exit(main())
453