1#!/usr/bin/env python
2# Copyright 2017 The LibYuv Project Authors. All rights reserved.
3#
4# Use of this source code is governed by a BSD-style license
5# that can be found in the LICENSE file in the root of the source
6# tree. An additional intellectual property rights grant can be found
7# in the file PATENTS. All contributing project authors may
8# be found in the AUTHORS file in the root of the source tree.
9
10# This is a modified copy of the script in
11# https://chromium.googlesource.com/external/webrtc/+/master/tools-webrtc/autoroller/roll_deps.py
12# customized for libyuv.
13
14
15"""Script to automatically roll dependencies in the libyuv DEPS file."""
16
17import argparse
18import base64
19import collections
20import logging
21import os
22import re
23import subprocess
24import sys
25import urllib
26
27
28# Skip these dependencies (list without solution name prefix).
29DONT_AUTOROLL_THESE = [
30  'src/third_party/gflags/src',
31]
32
33LIBYUV_URL = 'https://chromium.googlesource.com/libyuv/libyuv'
34CHROMIUM_SRC_URL = 'https://chromium.googlesource.com/chromium/src'
35CHROMIUM_COMMIT_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s'
36CHROMIUM_LOG_TEMPLATE = CHROMIUM_SRC_URL + '/+log/%s'
37CHROMIUM_FILE_TEMPLATE = CHROMIUM_SRC_URL + '/+/%s/%s'
38
39COMMIT_POSITION_RE = re.compile('^Cr-Commit-Position: .*#([0-9]+).*$')
40CLANG_REVISION_RE = re.compile(r'^CLANG_REVISION = \'(\d+)\'$')
41ROLL_BRANCH_NAME = 'roll_chromium_revision'
42
43SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
44CHECKOUT_SRC_DIR = os.path.realpath(os.path.join(SCRIPT_DIR, os.pardir,
45                                                 os.pardir))
46CHECKOUT_ROOT_DIR = os.path.realpath(os.path.join(CHECKOUT_SRC_DIR, os.pardir))
47
48sys.path.append(os.path.join(CHECKOUT_SRC_DIR, 'build'))
49import find_depot_tools
50find_depot_tools.add_depot_tools_to_path()
51from gclient import GClientKeywords
52
53CLANG_UPDATE_SCRIPT_URL_PATH = 'tools/clang/scripts/update.py'
54CLANG_UPDATE_SCRIPT_LOCAL_PATH = os.path.join(CHECKOUT_SRC_DIR, 'tools',
55                                              'clang', 'scripts', 'update.py')
56
57DepsEntry = collections.namedtuple('DepsEntry', 'path url revision')
58ChangedDep = collections.namedtuple('ChangedDep',
59                                    'path url current_rev new_rev')
60
61class RollError(Exception):
62  pass
63
64
65def ParseDepsDict(deps_content):
66  local_scope = {}
67  var = GClientKeywords.VarImpl({}, local_scope)
68  global_scope = {
69    'From': GClientKeywords.FromImpl,
70    'Var': var.Lookup,
71    'deps_os': {},
72  }
73  exec(deps_content, global_scope, local_scope)
74  return local_scope
75
76
77def ParseLocalDepsFile(filename):
78  with open(filename, 'rb') as f:
79    deps_content = f.read()
80  return ParseDepsDict(deps_content)
81
82
83def ParseRemoteCrDepsFile(revision):
84  deps_content = ReadRemoteCrFile('DEPS', revision)
85  return ParseDepsDict(deps_content)
86
87
88def ParseCommitPosition(commit_message):
89  for line in reversed(commit_message.splitlines()):
90    m = COMMIT_POSITION_RE.match(line.strip())
91    if m:
92      return m.group(1)
93  logging.error('Failed to parse commit position id from:\n%s\n',
94                commit_message)
95  sys.exit(-1)
96
97
98def _RunCommand(command, working_dir=None, ignore_exit_code=False,
99                extra_env=None):
100  """Runs a command and returns the output from that command.
101
102  If the command fails (exit code != 0), the function will exit the process.
103
104  Returns:
105    A tuple containing the stdout and stderr outputs as strings.
106  """
107  working_dir = working_dir or CHECKOUT_SRC_DIR
108  logging.debug('CMD: %s CWD: %s', ' '.join(command), working_dir)
109  env = os.environ.copy()
110  if extra_env:
111    assert all(type(value) == str for value in extra_env.values())
112    logging.debug('extra env: %s', extra_env)
113    env.update(extra_env)
114  p = subprocess.Popen(command, stdout=subprocess.PIPE,
115                       stderr=subprocess.PIPE, env=env,
116                       cwd=working_dir, universal_newlines=True)
117  std_output = p.stdout.read()
118  err_output = p.stderr.read()
119  p.wait()
120  p.stdout.close()
121  p.stderr.close()
122  if not ignore_exit_code and p.returncode != 0:
123    logging.error('Command failed: %s\n'
124                  'stdout:\n%s\n'
125                  'stderr:\n%s\n', ' '.join(command), std_output, err_output)
126    sys.exit(p.returncode)
127  return std_output, err_output
128
129
130def _GetBranches():
131  """Returns a tuple of active,branches.
132
133  The 'active' is the name of the currently active branch and 'branches' is a
134  list of all branches.
135  """
136  lines = _RunCommand(['git', 'branch'])[0].split('\n')
137  branches = []
138  active = ''
139  for line in lines:
140    if '*' in line:
141      # The assumption is that the first char will always be the '*'.
142      active = line[1:].strip()
143      branches.append(active)
144    else:
145      branch = line.strip()
146      if branch:
147        branches.append(branch)
148  return active, branches
149
150
151def _ReadGitilesContent(url):
152  # Download and decode BASE64 content until
153  # https://code.google.com/p/gitiles/issues/detail?id=7 is fixed.
154  base64_content = ReadUrlContent(url + '?format=TEXT')
155  return base64.b64decode(base64_content[0])
156
157
158def ReadRemoteCrFile(path_below_src, revision):
159  """Reads a remote Chromium file of a specific revision. Returns a string."""
160  return _ReadGitilesContent(CHROMIUM_FILE_TEMPLATE % (revision,
161                                                       path_below_src))
162
163
164def ReadRemoteCrCommit(revision):
165  """Reads a remote Chromium commit message. Returns a string."""
166  return _ReadGitilesContent(CHROMIUM_COMMIT_TEMPLATE % revision)
167
168
169def ReadUrlContent(url):
170  """Connect to a remote host and read the contents. Returns a list of lines."""
171  conn = urllib.urlopen(url)
172  try:
173    return conn.readlines()
174  except IOError as e:
175    logging.exception('Error connecting to %s. Error: %s', url, e)
176    raise
177  finally:
178    conn.close()
179
180
181def GetMatchingDepsEntries(depsentry_dict, dir_path):
182  """Gets all deps entries matching the provided path.
183
184  This list may contain more than one DepsEntry object.
185  Example: dir_path='src/testing' would give results containing both
186  'src/testing/gtest' and 'src/testing/gmock' deps entries for Chromium's DEPS.
187  Example 2: dir_path='src/build' should return 'src/build' but not
188  'src/buildtools'.
189
190  Returns:
191    A list of DepsEntry objects.
192  """
193  result = []
194  for path, depsentry in depsentry_dict.iteritems():
195    if path == dir_path:
196      result.append(depsentry)
197    else:
198      parts = path.split('/')
199      if all(part == parts[i]
200             for i, part in enumerate(dir_path.split('/'))):
201        result.append(depsentry)
202  return result
203
204
205def BuildDepsentryDict(deps_dict):
206  """Builds a dict of paths to DepsEntry objects from a raw parsed deps dict."""
207  result = {}
208  def AddDepsEntries(deps_subdict):
209    for path, deps_url in deps_subdict.iteritems():
210      if not result.has_key(path):
211        url, revision = deps_url.split('@') if deps_url else (None, None)
212        result[path] = DepsEntry(path, url, revision)
213
214  AddDepsEntries(deps_dict['deps'])
215  for deps_os in ['win', 'mac', 'unix', 'android', 'ios', 'unix']:
216    AddDepsEntries(deps_dict.get('deps_os', {}).get(deps_os, {}))
217  return result
218
219
220def CalculateChangedDeps(libyuv_deps, new_cr_deps):
221  """
222  Calculate changed deps entries based on entries defined in the libyuv DEPS
223  file:
224     - If a shared dependency with the Chromium DEPS file: roll it to the same
225       revision as Chromium (i.e. entry in the new_cr_deps dict)
226     - If it's a Chromium sub-directory, roll it to the HEAD revision (notice
227       this means it may be ahead of the chromium_revision, but generally these
228       should be close).
229     - If it's another DEPS entry (not shared with Chromium), roll it to HEAD
230       unless it's configured to be skipped.
231
232  Returns:
233    A list of ChangedDep objects representing the changed deps.
234  """
235  result = []
236  libyuv_entries = BuildDepsentryDict(libyuv_deps)
237  new_cr_entries = BuildDepsentryDict(new_cr_deps)
238  for path, libyuv_deps_entry in libyuv_entries.iteritems():
239    if path in DONT_AUTOROLL_THESE:
240      continue
241    cr_deps_entry = new_cr_entries.get(path)
242    if cr_deps_entry:
243      # Use the revision from Chromium's DEPS file.
244      new_rev = cr_deps_entry.revision
245      assert libyuv_deps_entry.url == cr_deps_entry.url, (
246        'Libyuv DEPS entry %s has a different URL (%s) than Chromium (%s).' %
247        (path, libyuv_deps_entry.url, cr_deps_entry.url))
248    else:
249      # Use the HEAD of the deps repo.
250      stdout, _ = _RunCommand(['git', 'ls-remote', libyuv_deps_entry.url,
251                               'HEAD'])
252      new_rev = stdout.strip().split('\t')[0]
253
254    # Check if an update is necessary.
255    if libyuv_deps_entry.revision != new_rev:
256      logging.debug('Roll dependency %s to %s', path, new_rev)
257      result.append(ChangedDep(path, libyuv_deps_entry.url,
258                               libyuv_deps_entry.revision, new_rev))
259  return sorted(result)
260
261
262def CalculateChangedClang(new_cr_rev):
263  def GetClangRev(lines):
264    for line in lines:
265      match = CLANG_REVISION_RE.match(line)
266      if match:
267        return match.group(1)
268    raise RollError('Could not parse Clang revision!')
269
270  with open(CLANG_UPDATE_SCRIPT_LOCAL_PATH, 'rb') as f:
271    current_lines = f.readlines()
272  current_rev = GetClangRev(current_lines)
273
274  new_clang_update_py = ReadRemoteCrFile(CLANG_UPDATE_SCRIPT_URL_PATH,
275                                             new_cr_rev).splitlines()
276  new_rev = GetClangRev(new_clang_update_py)
277  return ChangedDep(CLANG_UPDATE_SCRIPT_LOCAL_PATH, None, current_rev, new_rev)
278
279
280def GenerateCommitMessage(current_cr_rev, new_cr_rev, current_commit_pos,
281                          new_commit_pos, changed_deps_list, clang_change):
282  current_cr_rev = current_cr_rev[0:10]
283  new_cr_rev = new_cr_rev[0:10]
284  rev_interval = '%s..%s' % (current_cr_rev, new_cr_rev)
285  git_number_interval = '%s:%s' % (current_commit_pos, new_commit_pos)
286
287  commit_msg = ['Roll chromium_revision %s (%s)\n' % (rev_interval,
288                                                    git_number_interval)]
289  commit_msg.append('Change log: %s' % (CHROMIUM_LOG_TEMPLATE % rev_interval))
290  commit_msg.append('Full diff: %s\n' % (CHROMIUM_COMMIT_TEMPLATE %
291                                         rev_interval))
292  # TBR field will be empty unless in some custom cases, where some engineers
293  # are added.
294  tbr_authors = ''
295  if changed_deps_list:
296    commit_msg.append('Changed dependencies:')
297
298    for c in changed_deps_list:
299      commit_msg.append('* %s: %s/+log/%s..%s' % (c.path, c.url,
300                                                  c.current_rev[0:10],
301                                                  c.new_rev[0:10]))
302    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval, 'DEPS')
303    commit_msg.append('DEPS diff: %s\n' % change_url)
304  else:
305    commit_msg.append('No dependencies changed.')
306
307  if clang_change.current_rev != clang_change.new_rev:
308    commit_msg.append('Clang version changed %s:%s' %
309                      (clang_change.current_rev, clang_change.new_rev))
310    change_url = CHROMIUM_FILE_TEMPLATE % (rev_interval,
311                                           CLANG_UPDATE_SCRIPT_URL_PATH)
312    commit_msg.append('Details: %s\n' % change_url)
313  else:
314    commit_msg.append('No update to Clang.\n')
315
316  commit_msg.append('TBR=%s' % tbr_authors)
317  commit_msg.append('BUG=None')
318  return '\n'.join(commit_msg)
319
320
321def UpdateDepsFile(deps_filename, old_cr_revision, new_cr_revision,
322                   changed_deps):
323  """Update the DEPS file with the new revision."""
324
325  # Update the chromium_revision variable.
326  with open(deps_filename, 'rb') as deps_file:
327    deps_content = deps_file.read()
328  deps_content = deps_content.replace(old_cr_revision, new_cr_revision)
329  with open(deps_filename, 'wb') as deps_file:
330    deps_file.write(deps_content)
331
332  # Update each individual DEPS entry.
333  for dep in changed_deps:
334    local_dep_dir = os.path.join(CHECKOUT_ROOT_DIR, dep.path)
335    if not os.path.isdir(local_dep_dir):
336      raise RollError(
337          'Cannot find local directory %s. Either run\n'
338          'gclient sync --deps=all\n'
339          'or make sure the .gclient file for your solution contains all '
340          'platforms in the target_os list, i.e.\n'
341          'target_os = ["android", "unix", "mac", "ios", "win"];\n'
342          'Then run "gclient sync" again.' % local_dep_dir)
343    _, stderr = _RunCommand(
344      ['roll-dep-svn', '--no-verify-revision', dep.path, dep.new_rev],
345      working_dir=CHECKOUT_SRC_DIR, ignore_exit_code=True)
346    if stderr:
347      logging.warning('roll-dep-svn: %s', stderr)
348
349
350def _IsTreeClean():
351  stdout, _ = _RunCommand(['git', 'status', '--porcelain'])
352  if len(stdout) == 0:
353    return True
354
355  logging.error('Dirty/unversioned files:\n%s', stdout)
356  return False
357
358
359def _EnsureUpdatedMasterBranch(dry_run):
360  current_branch = _RunCommand(
361      ['git', 'rev-parse', '--abbrev-ref', 'HEAD'])[0].splitlines()[0]
362  if current_branch != 'master':
363    logging.error('Please checkout the master branch and re-run this script.')
364    if not dry_run:
365      sys.exit(-1)
366
367  logging.info('Updating master branch...')
368  _RunCommand(['git', 'pull'])
369
370
371def _CreateRollBranch(dry_run):
372  logging.info('Creating roll branch: %s', ROLL_BRANCH_NAME)
373  if not dry_run:
374    _RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
375
376
377def _RemovePreviousRollBranch(dry_run):
378  active_branch, branches = _GetBranches()
379  if active_branch == ROLL_BRANCH_NAME:
380    active_branch = 'master'
381  if ROLL_BRANCH_NAME in branches:
382    logging.info('Removing previous roll branch (%s)', ROLL_BRANCH_NAME)
383    if not dry_run:
384      _RunCommand(['git', 'checkout', active_branch])
385      _RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
386
387
388def _LocalCommit(commit_msg, dry_run):
389  logging.info('Committing changes locally.')
390  if not dry_run:
391    _RunCommand(['git', 'add', '--update', '.'])
392    _RunCommand(['git', 'commit', '-m', commit_msg])
393
394
395def _UploadCL(dry_run, rietveld_email=None):
396  logging.info('Uploading CL...')
397  if not dry_run:
398    cmd = ['git', 'cl', 'upload', '-f']
399    if rietveld_email:
400      cmd.append('--email=%s' % rietveld_email)
401    _RunCommand(cmd, extra_env={'EDITOR': 'true'})
402
403
404def _SendToCQ(dry_run, skip_cq):
405  logging.info('Sending the CL to the CQ...')
406  if not dry_run and not skip_cq:
407    _RunCommand(['git', 'cl', 'set_commit'])
408    logging.info('Sent the CL to the CQ.')
409
410
411def main():
412  p = argparse.ArgumentParser()
413  p.add_argument('--clean', action='store_true', default=False,
414                 help='Removes any previous local roll branch.')
415  p.add_argument('-r', '--revision',
416                 help=('Chromium Git revision to roll to. Defaults to the '
417                       'Chromium HEAD revision if omitted.'))
418  p.add_argument('-u', '--rietveld-email',
419                 help=('E-mail address to use for creating the CL at Rietveld'
420                       'If omitted a previously cached one will be used or an '
421                       'error will be thrown during upload.'))
422  p.add_argument('--dry-run', action='store_true', default=False,
423                 help=('Calculate changes and modify DEPS, but don\'t create '
424                       'any local branch, commit, upload CL or send any '
425                       'tryjobs.'))
426  p.add_argument('-i', '--ignore-unclean-workdir', action='store_true',
427                 default=False,
428                 help=('Ignore if the current branch is not master or if there '
429                       'are uncommitted changes (default: %(default)s).'))
430  p.add_argument('--skip-cq', action='store_true', default=False,
431                 help='Skip sending the CL to the CQ (default: %(default)s)')
432  p.add_argument('-v', '--verbose', action='store_true', default=False,
433                 help='Be extra verbose in printing of log messages.')
434  opts = p.parse_args()
435
436  if opts.verbose:
437    logging.basicConfig(level=logging.DEBUG)
438  else:
439    logging.basicConfig(level=logging.INFO)
440
441  if not opts.ignore_unclean_workdir and not _IsTreeClean():
442    logging.error('Please clean your local checkout first.')
443    return 1
444
445  if opts.clean:
446    _RemovePreviousRollBranch(opts.dry_run)
447
448  if not opts.ignore_unclean_workdir:
449    _EnsureUpdatedMasterBranch(opts.dry_run)
450
451  new_cr_rev = opts.revision
452  if not new_cr_rev:
453    stdout, _ = _RunCommand(['git', 'ls-remote', CHROMIUM_SRC_URL, 'HEAD'])
454    head_rev = stdout.strip().split('\t')[0]
455    logging.info('No revision specified. Using HEAD: %s', head_rev)
456    new_cr_rev = head_rev
457
458  deps_filename = os.path.join(CHECKOUT_SRC_DIR, 'DEPS')
459  libyuv_deps = ParseLocalDepsFile(deps_filename)
460  current_cr_rev = libyuv_deps['vars']['chromium_revision']
461
462  current_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(current_cr_rev))
463  new_commit_pos = ParseCommitPosition(ReadRemoteCrCommit(new_cr_rev))
464
465  new_cr_deps = ParseRemoteCrDepsFile(new_cr_rev)
466  changed_deps = CalculateChangedDeps(libyuv_deps, new_cr_deps)
467  clang_change = CalculateChangedClang(new_cr_rev)
468  commit_msg = GenerateCommitMessage(current_cr_rev, new_cr_rev,
469                                     current_commit_pos, new_commit_pos,
470                                     changed_deps, clang_change)
471  logging.debug('Commit message:\n%s', commit_msg)
472
473  _CreateRollBranch(opts.dry_run)
474  UpdateDepsFile(deps_filename, current_cr_rev, new_cr_rev, changed_deps)
475  _LocalCommit(commit_msg, opts.dry_run)
476  _UploadCL(opts.dry_run, opts.rietveld_email)
477  _SendToCQ(opts.dry_run, opts.skip_cq)
478  return 0
479
480
481if __name__ == '__main__':
482  sys.exit(main())
483