merge_from_chromium.py revision a611c7e365068e2130b6de87b2b14e3b75ae1d4e 
1#!/usr/bin/python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Merge Chromium into the Android tree."""
18
19import contextlib
20import logging
21import optparse
22import os
23import re
24import sys
25import urllib2
26
27import merge_common
28
29
30# We need to import this *after* merging from upstream to get the latest
31# version. Set it to none here to catch uses before it's imported.
32webview_licenses = None
33
34
35AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.'
36SRC_GIT_URL = 'http://chromium.googlesource.com/chromium/src.git'
37SRC_GIT_BRANCH = 'git-svn'
38
39
40def _ReadGitFile(git_url, git_branch, sha1, path):
41  """Reads a file from a remote git project at a specific revision.
42
43  Args:
44    git_url: The URL of the git server.
45    git_branch: The branch to fetch.
46    sha1: The SHA1 at which to read.
47    path: The relative path of the file to read.
48  Returns:
49    The contents of the specified file.
50  """
51  # We fetch the branch to a temporary head so that we don't download the same
52  # commits multiple times.
53  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
54                                 git_branch + ':cached_upstream'])
55  return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)])
56
57
58def _ParseDEPS(git_url, git_branch, sha1):
59  """Parses the .DEPS.git file from Chromium and returns its contents.
60
61  Args:
62    git_url: The URL of the git server.
63    git_branch: The branch to read.
64    sha1: The SHA1 at which to read.
65  Returns:
66    A dictionary of the contents of .DEPS.git at the specified revision
67  """
68
69  class FromImpl(object):
70    """Used to implement the From syntax."""
71
72    def __init__(self, module_name):
73      self.module_name = module_name
74
75    def __str__(self):
76      return 'From("%s")' % self.module_name
77
78  class _VarImpl(object):
79    def __init__(self, custom_vars, local_scope):
80      self._custom_vars = custom_vars
81      self._local_scope = local_scope
82
83    def Lookup(self, var_name):
84      """Implements the Var syntax."""
85      if var_name in self._custom_vars:
86        return self._custom_vars[var_name]
87      elif var_name in self._local_scope.get('vars', {}):
88        return self._local_scope['vars'][var_name]
89      raise Exception('Var is not defined: %s' % var_name)
90
91  tmp_locals = {}
92  var = _VarImpl({}, tmp_locals)
93  tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
94  deps_content = _ReadGitFile(git_url, git_branch, sha1, '.DEPS.git')
95  exec(deps_content) in tmp_globals, tmp_locals
96  return tmp_locals
97
98
99def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars):
100  """Gets the git URL and SHA1 for each project based on .DEPS.git.
101
102  Args:
103    third_party_projects: The list of projects to consider.
104    deps_vars: The dictionary of dependencies from .DEPS.git.
105  Returns:
106    A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
107  Raises:
108    TemporaryMergeError: if a project to be merged is not found in .DEPS.git.
109  """
110  deps_fallback_order = [
111      deps_vars['deps'],
112      deps_vars['deps_os']['unix'],
113      deps_vars['deps_os']['android'],
114  ]
115  result = {}
116  for path in third_party_projects:
117    for deps in deps_fallback_order:
118      url_plus_sha1 = deps.get(os.path.join('src', path))
119      if url_plus_sha1:
120        break
121    else:
122      raise merge_common.TemporaryMergeError(
123          'Could not find .DEPS.git entry for project %s. This probably '
124          'means that the project list in merge_from_chromium.py needs to be '
125          'updated.' % path)
126    match = re.match('(.*?)@(.*)', url_plus_sha1)
127    url = match.group(1)
128    sha1 = match.group(2)
129    logging.debug('  Got URL %s and SHA1 %s for project %s', url, sha1, path)
130    result[path] = {'url': url, 'sha1': sha1}
131  return result
132
133
134def _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended):
135  """Merges each required Chromium project into the Android repository.
136
137  .DEPS.git is consulted to determine which revision each project must be merged
138  at. Only a whitelist of required projects are merged.
139
140  Args:
141    git_url: The URL of the Chromium repository to merge from.
142    git_branch: The branch in the Chromium repository to merge from.
143    svn_revision: The SVN revision in the Chromium repository to merge from.
144    root_sha1: The git hash corresponding to svn_revision.
145    unattended: Run in unattended mode.
146  Raises:
147    TemporaryMergeError: If incompatibly licensed code is left after pruning.
148  """
149  # The logic for this step lives here, in the Android tree, as it makes no
150  # sense for a Chromium tree to know about this merge.
151
152  if unattended:
153    branch_create_flag = '-B'
154  else:
155    branch_create_flag = '-b'
156  branch_name = 'merge-from-chromium-%s' % svn_revision
157
158  logging.debug('Parsing DEPS ...')
159  deps_vars = _ParseDEPS(git_url, git_branch, root_sha1)
160
161  merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
162                                              deps_vars)
163
164  for path in merge_info:
165    url = merge_info[path]['url']
166    sha1 = merge_info[path]['sha1']
167    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
168    merge_common.GetCommandStdout(['git', 'checkout',
169                                   branch_create_flag, branch_name,
170                                   '-t', 'goog/master-chromium'], cwd=dest_dir)
171    logging.debug('Fetching project %s at %s ...', path, sha1)
172    merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir)
173    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
174                                     cwd=dest_dir):
175      logging.debug('Merging project %s at %s ...', path, sha1)
176      # Merge conflicts make git merge return 1, so ignore errors
177      merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
178                                    cwd=dest_dir, ignore_errors=True)
179      merge_common.CheckNoConflictsAndCommitMerge(
180          'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
181          cwd=dest_dir, unattended=unattended)
182    else:
183      logging.debug('No new commits to merge in project %s', path)
184
185  # Handle root repository separately.
186  merge_common.GetCommandStdout(['git', 'checkout',
187                                 branch_create_flag, branch_name,
188                                 '-t', 'goog/master-chromium'])
189  logging.debug('Fetching Chromium at %s ...', root_sha1)
190  merge_common.GetCommandStdout(['git', 'fetch', git_url, git_branch])
191  logging.debug('Merging Chromium at %s ...', root_sha1)
192  # Merge conflicts make git merge return 1, so ignore errors
193  merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1],
194                                ignore_errors=True)
195  merge_common.CheckNoConflictsAndCommitMerge(
196      'Merge Chromium from %s branch %s at r%s (%s)\n\n%s'
197      % (git_url, git_branch, svn_revision, root_sha1, AUTOGEN_MESSAGE),
198      unattended=unattended)
199
200  logging.debug('Getting directories to exclude ...')
201
202  # We import this now that we have merged the latest version.
203  # It imports to a global in order that it can be used to generate NOTICE
204  # later. We also disable writing bytecode to keep the source tree clean.
205  sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
206                               'tools'))
207  sys.dont_write_bytecode = True
208  global webview_licenses
209  import webview_licenses
210  import known_issues
211
212  for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
213    logging.debug('  %s', '\n  '.join(os.path.join(path, x) for x in
214                                      exclude_list))
215    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
216    merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
217                                  exclude_list, cwd=dest_dir)
218    if _ModifiedFilesInIndex(dest_dir):
219      merge_common.GetCommandStdout(['git', 'commit', '-m',
220                                     'Exclude incompatible directories'],
221                                    cwd=dest_dir)
222
223  directories_left_over = webview_licenses.GetIncompatibleDirectories()
224  if directories_left_over:
225    raise merge_common.TemporaryMergeError(
226        'Incompatibly licensed directories remain: ' +
227        '\n'.join(directories_left_over))
228
229
230def _GenerateMakefiles(svn_revision, unattended):
231  """Run gyp to generate the Android build system makefiles.
232
233  Args:
234    svn_revision: The SVN revision to mention in generated commit messages.
235    unattended: Run in unattended mode.
236  """
237  logging.debug('Generating makefiles ...')
238
239  # TODO(torne): The .tmp files are generated by
240  # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
241  # tree. We should avoid this, or at least use a more specific name to avoid
242  # accidentally removing or adding other files.
243  for path in merge_common.ALL_PROJECTS:
244    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
245    merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
246                                   'GypAndroid.*.mk', '*.target.*.mk',
247                                   '*.host.*.mk', '*.tmp'], cwd=dest_dir)
248
249  try:
250    merge_common.GetCommandStdout(['android_webview/tools/gyp_webview'])
251  except merge_common.MergeError as e:
252    if not unattended:
253      raise
254    else:
255      for path in merge_common.ALL_PROJECTS:
256        merge_common.GetCommandStdout(
257            ['git', 'reset', '--hard'],
258            cwd=os.path.join(merge_common.REPOSITORY_ROOT, path))
259      raise merge_common.TemporaryMergeError('Makefile generation failed: ' +
260                                             str(e))
261
262  for path in merge_common.ALL_PROJECTS:
263    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
264    # git add doesn't have an --ignore-unmatch so we have to do this instead:
265    merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'],
266                                  ignore_errors=True, cwd=dest_dir)
267    merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'],
268                                  ignore_errors=True, cwd=dest_dir)
269    merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'],
270                                  ignore_errors=True, cwd=dest_dir)
271    merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
272                                  ignore_errors=True, cwd=dest_dir)
273    # Only try to commit the makefiles if something has actually changed.
274    if _ModifiedFilesInIndex(dest_dir):
275      merge_common.GetCommandStdout(
276          ['git', 'commit', '-m',
277           'Update makefiles after merge of Chromium at r%s\n\n%s' %
278           (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
279
280
281def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
282  """Returns true if git's index contains any changes."""
283  status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
284                                         cwd=cwd)
285  return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None
286
287
288def _GenerateNoticeFile(svn_revision):
289  """Generates and commits a NOTICE file containing code licenses.
290
291  This covers all third-party code (from Android's perspective) that lives in
292  the Chromium tree.
293
294  Args:
295    svn_revision: The SVN revision for the main Chromium repository.
296  """
297  logging.debug('Regenerating NOTICE file ...')
298
299  contents = webview_licenses.GenerateNoticeFile()
300
301  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
302    f.write(contents)
303  merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
304  # Only try to commit the NOTICE update if the file has actually changed.
305  if _ModifiedFilesInIndex():
306    merge_common.GetCommandStdout([
307        'git', 'commit', '-m',
308        'Update NOTICE file after merge of Chromium at r%s\n\n%s'
309        % (svn_revision, AUTOGEN_MESSAGE)])
310
311
312def _GenerateLastChange(svn_revision):
313  """Write a build/util/LASTCHANGE file containing the current revision.
314
315  The revision number is compiled into the binary at build time from this file.
316
317  Args:
318    svn_revision: The SVN revision for the main Chromium repository.
319  """
320  logging.debug('Updating LASTCHANGE ...')
321  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
322            'w') as f:
323    f.write('LASTCHANGE=%s\n' % svn_revision)
324  merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
325  if _ModifiedFilesInIndex():
326    merge_common.GetCommandStdout([
327        'git', 'commit', '-m',
328        'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s'
329        % (svn_revision, AUTOGEN_MESSAGE)])
330
331
332def GetLKGR():
333  """Fetch the last known good release from Chromium's dashboard.
334
335  Returns:
336    The last known good SVN revision.
337  """
338  with contextlib.closing(
339      urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr:
340    return int(lkgr.read())
341
342
343def GetHEAD():
344  """Fetch the latest HEAD revision from the git mirror of the Chromium svn
345  repo.
346
347  Returns:
348    The latest HEAD SVN revision.
349  """
350  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(SRC_GIT_URL,
351                                                     SRC_GIT_BRANCH,
352                                                     'HEAD')
353  return int(svn_revision)
354
355
356def _GetSVNRevisionAndSHA1(git_url, git_branch, svn_revision):
357  logging.debug('Getting SVN revision and SHA1 ...')
358  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
359                                 git_branch + ':cached_upstream'])
360  if svn_revision == 'HEAD':
361    # Just use the latest commit.
362    commit = merge_common.GetCommandStdout([
363        'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b',
364        'cached_upstream'])
365    sha1 = commit.split()[0]
366    svn_revision = re.search(r'^git-svn-id: .*@([0-9]+)', commit,
367                             flags=re.MULTILINE).group(1)
368    return (svn_revision, sha1)
369
370  if svn_revision is None:
371    # Fetch LKGR from upstream.
372    svn_revision = GetLKGR()
373  output = merge_common.GetCommandStdout([
374      'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision,
375      '--format=%H', 'cached_upstream'])
376  if not output:
377    raise merge_common.TemporaryMergeError('Revision %s not found in git repo.'
378                                           % svn_revision)
379  # The log grep will sometimes match reverts/reapplies of commits. We take the
380  # oldest (last) match because the first time it appears in history is
381  # overwhelmingly likely to be the correct commit.
382  sha1 = output.split()[-1]
383  return (svn_revision, sha1)
384
385
386def Snapshot(svn_revision, unattended):
387  """Takes a snapshot of the Chromium tree and merges it into Android.
388
389  Android makefiles and a top-level NOTICE file are generated and committed
390  after the merge.
391
392  Args:
393    svn_revision: The SVN revision in the Chromium repository to merge from.
394    unattended: Run in unattended mode.
395
396  Returns:
397    True if new commits were merged; False if no new commits were present.
398  """
399  git_url = SRC_GIT_URL
400  git_branch = SRC_GIT_BRANCH
401  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_url, git_branch,
402                                                     svn_revision)
403  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
404                                        'HEAD..' + root_sha1]):
405    logging.info('No new commits to merge from %s branch %s at r%s (%s)',
406                 git_url, git_branch, svn_revision, root_sha1)
407    return False
408
409  logging.info('Snapshotting Chromium from %s branch %s at r%s (%s)',
410               git_url, git_branch, svn_revision, root_sha1)
411
412  # 1. Merge, accounting for excluded directories
413  _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended)
414
415  # 2. Generate Android NOTICE file
416  _GenerateNoticeFile(svn_revision)
417
418  # 3. Generate LASTCHANGE file
419  _GenerateLastChange(svn_revision)
420
421  # 4. Generate Android makefiles
422  _GenerateMakefiles(svn_revision, unattended)
423
424  return True
425
426
427def Push(svn_revision):
428  """Push the finished snapshot to the Android repository."""
429  merge_common.PushToServer('merge-from-chromium-%s' % svn_revision,
430                            'master-chromium', 'master-chromium-merge')
431
432
433def main():
434  parser = optparse.OptionParser(usage='%prog [options]')
435  parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
436                   'Chromium SVN revision and merges it into this repository. '
437                   'Paths marked as excluded for license reasons are removed '
438                   'as part of the merge. Also generates Android makefiles and '
439                   'generates a top-level NOTICE file suitable for use in the '
440                   'Android build.')
441  parser.add_option(
442      '', '--svn_revision',
443      default=None,
444      help=('Merge to the specified chromium SVN revision, rather than using '
445            'the current LKGR. Can also pass HEAD to merge from tip of tree.'))
446  parser.add_option(
447      '', '--push',
448      default=False, action='store_true',
449      help=('Push the result of a previous merge to the server.'))
450  parser.add_option(
451      '', '--get_lkgr',
452      default=False, action='store_true',
453      help=('Just print the current LKGR on stdout and exit.'))
454  parser.add_option(
455      '', '--get_head',
456      default=False, action='store_true',
457      help=('Just print the current HEAD revision on stdout and exit.'))
458  parser.add_option(
459      '', '--unattended',
460      default=False, action='store_true',
461      help=('Run in unattended mode.'))
462  parser.add_option(
463      '', '--no_changes_exit',
464      default=0, type='int',
465      help=('Exit code to use if there are no changes to merge, for scripts.'))
466  (options, args) = parser.parse_args()
467  if args:
468    parser.print_help()
469    return 1
470
471  if 'ANDROID_BUILD_TOP' not in os.environ:
472    print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.'
473    return 1
474
475  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
476                      stream=sys.stdout)
477
478  if options.get_lkgr:
479    print GetLKGR()
480  elif options.get_head:
481    logging.disable(logging.CRITICAL)  # Prevent log messages
482    print GetHEAD()
483  elif options.push:
484    if options.svn_revision is None:
485      print >>sys.stderr, 'You need to pass the SVN revision to push.'
486      return 1
487    else:
488      Push(options.svn_revision)
489  else:
490    if not Snapshot(options.svn_revision, options.unattended):
491      return options.no_changes_exit
492
493  return 0
494
495if __name__ == '__main__':
496  sys.exit(main())
497