merge_from_chromium.py revision 1c263f2b522bcb9159547d14f9f57abac46967c7 
1#!/usr/bin/env python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Merge Chromium into the Android tree. See the output of --help for details.
19
20"""
21import optparse
22import os
23import re
24import sys
25
26import merge_common
27
28
29# We need to import this *after* merging from upstream to get the latest
30# version. Set it to none here to catch uses before it's imported.
31webview_licenses = None
32
33
34AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.'
35
36
37def _ReadGitFile(git_url, git_branch, sha1, path):
38  """Reads a file from a remote git project at a specific revision.
39  Args:
40    git_url: The URL of the git server.
41    git_branch: The branch to read.
42    sha1: The SHA1 at which to read.
43    path: The relative path of the file to read.
44  Returns:
45    The contents of the specified file.
46  """
47
48  # We fetch the branch to a temporary head so that we don't download the same
49  # commits multiple times.
50  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
51                                 git_branch + ':cached_upstream'])
52
53  args = ['git', 'show', '%s:%s' % (sha1, path)]
54  return merge_common.GetCommandStdout(args)
55
56
57def _ParseDEPS(git_url, git_branch, sha1):
58  """Parses the .DEPS.git file from Chromium and returns its contents.
59  Args:
60    git_url: The URL of the git server.
61    git_branch: The branch to read.
62    sha1: The SHA1 at which to read.
63  Returns:
64    A dictionary of the contents of .DEPS.git at the specified revision
65  """
66
67  class FromImpl(object):
68    """Used to implement the From syntax."""
69
70    def __init__(self, module_name):
71      self.module_name = module_name
72
73    def __str__(self):
74      return 'From("%s")' % self.module_name
75
76  class _VarImpl(object):
77    def __init__(self, custom_vars, local_scope):
78      self._custom_vars = custom_vars
79      self._local_scope = local_scope
80
81    def Lookup(self, var_name):
82      """Implements the Var syntax."""
83      if var_name in self._custom_vars:
84        return self._custom_vars[var_name]
85      elif var_name in self._local_scope.get('vars', {}):
86        return self._local_scope['vars'][var_name]
87      raise Exception('Var is not defined: %s' % var_name)
88
89  tmp_locals = {}
90  var = _VarImpl({}, tmp_locals)
91  tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
92  deps_content = _ReadGitFile(git_url, git_branch, sha1, '.DEPS.git')
93  exec(deps_content) in tmp_globals, tmp_locals
94  return tmp_locals
95
96
97def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars):
98  """Gets the git URL for each project and the SHA1 at which it should be
99  merged.
100  Args:
101    third_party_projects: The list of projects to consider.
102    deps_vars: The dictionary of dependencies from .DEPS.git.
103  Returns:
104    A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
105  """
106
107  deps_fallback_order = [
108      deps_vars['deps'],
109      deps_vars['deps_os']['unix'],
110      deps_vars['deps_os']['android'],
111  ]
112  result = {}
113  for path in third_party_projects:
114    for deps in deps_fallback_order:
115      url_plus_sha1 = deps.get(os.path.join('src', path))
116      if url_plus_sha1:
117        break
118    else:
119      raise RuntimeError(
120          ('Could not find .DEPS.git entry for project %s. This probably '
121           'means that the project list in merge_from_chromium.py needs to be '
122           'updated.') %
123          path)
124    match = re.match('(.*?)@(.*)', url_plus_sha1)
125    url = match.group(1)
126    sha1 = match.group(2)
127    print '  Got URL %s and SHA1 %s for project %s' % (url, sha1, path)
128    result[path] = {'url': url, 'sha1': sha1}
129  return result
130
131
132def _MergeProjects(git_url, git_branch, svn_revision, root_sha1):
133  """Merges into this repository all projects required by the specified branch
134  of Chromium, at the SVN revision. Uses a git subtree merge for each project.
135  Directories in the main Chromium repository which are not needed by Clank are
136  not merged.
137  Args:
138    git_url: The URL of the git server for the Chromium branch to merge to
139    git_branch: The branch name to merge to
140    svn_revision: The SVN revision for the main Chromium repository
141    root_sha1: The git SHA1 for the main Chromium repository
142  """
143
144  # The logic for this step lives here, in the Android tree, as it makes no
145  # sense for a Chromium tree to know about this merge.
146
147  print 'Parsing DEPS ...'
148  deps_vars = _ParseDEPS(git_url, git_branch, root_sha1)
149
150  merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
151                                              deps_vars)
152
153  for path in merge_info:
154    url = merge_info[path]['url']
155    sha1 = merge_info[path]['sha1']
156    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
157    merge_common.GetCommandStdout(['git', 'checkout',
158                                   '-b', 'merge-from-chromium',
159                                   '-t', 'goog/master-chromium'], cwd=dest_dir)
160    print 'Fetching project %s at %s ...' % (path, sha1)
161    merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir)
162    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
163                                     cwd=dest_dir):
164      print 'Merging project %s at %s ...' % (path, sha1)
165      # Merge conflicts make git merge return 1, so ignore errors
166      merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
167                                    cwd=dest_dir, ignore_errors=True)
168      merge_common.CheckNoConflictsAndCommitMerge(
169          'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
170          cwd=dest_dir)
171    else:
172      print 'No new commits to merge in project %s' % path
173
174  # Handle root repository separately.
175  merge_common.GetCommandStdout(['git', 'checkout', '-b', 'merge-from-chromium',
176                                 '-t', 'goog/master-chromium'])
177  print 'Fetching Chromium at %s ...' % root_sha1
178  merge_common.GetCommandStdout(['git', 'fetch', git_url, git_branch])
179  print 'Merging Chromium at %s ...' % root_sha1
180  # Merge conflicts make git merge return 1, so ignore errors
181  merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1],
182                                ignore_errors=True)
183  merge_common.CheckNoConflictsAndCommitMerge(
184      'Merge Chromium from %s branch %s at r%s (%s)\n\n%s'
185      % (git_url, git_branch, svn_revision, root_sha1, AUTOGEN_MESSAGE))
186
187  print 'Getting directories to exclude ...'
188
189  # We import this now that we have merged the latest version.
190  # It imports to a global in order that it can be used to generate NOTICE
191  # later. We also disable writing bytecode to keep the source tree clean.
192  sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
193                               'tools'))
194  sys.dont_write_bytecode = True
195  global webview_licenses
196  import webview_licenses
197  import known_incompatible
198
199  for path, exclude_list in known_incompatible.KNOWN_INCOMPATIBLE.iteritems():
200    print '  %s' % '\n  '.join(os.path.join(path, x) for x in exclude_list)
201    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
202    merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
203                                  exclude_list, cwd=dest_dir)
204    if _ModifiedFilesInIndex(dest_dir):
205      merge_common.GetCommandStdout(['git', 'commit', '-m',
206                                     'Exclude incompatible directories'],
207                                    cwd=dest_dir)
208
209  directories_left_over = webview_licenses.GetIncompatibleDirectories()
210  if directories_left_over:
211    raise RuntimeError('Incompatibly licensed directories remain: ' +
212                       '\n'.join(directories_left_over))
213  return True
214
215
216def _GenerateMakefiles(svn_revision):
217  """Run gyp to generate the makefiles required to build Chromium in the
218  Android build system.
219  """
220
221  print 'Regenerating makefiles ...'
222  # TODO(torne): The .tmp files are generated by
223  # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
224  # tree. We should avoid this, or at least use a more specific name to avoid
225  # accidentally removing or adding other files.
226  for path in merge_common.ALL_PROJECTS:
227    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
228    merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
229                                   'GypAndroid.mk', '*.target.mk', '*.host.mk',
230                                   '*.tmp'], cwd=dest_dir)
231
232  merge_common.GetCommandStdout(['bash', '-c',
233                                 'export CHROME_ANDROID_BUILD_WEBVIEW=1 && '
234                                 'export CHROME_SRC=`pwd` && '
235                                 'export PYTHONDONTWRITEBYTECODE=1 && '
236                                 '. build/android/envsetup.sh && '
237                                 'android_gyp'])
238
239  for path in merge_common.ALL_PROJECTS:
240    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
241    # git add doesn't have an --ignore-unmatch so we have to do this instead:
242    merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.mk'],
243                                  ignore_errors=True, cwd=dest_dir)
244    merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.mk'],
245                                  ignore_errors=True, cwd=dest_dir)
246    merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.mk'],
247                                  ignore_errors=True, cwd=dest_dir)
248    merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
249                                  ignore_errors=True, cwd=dest_dir)
250    # Only try to commit the makefiles if something has actually changed.
251    if _ModifiedFilesInIndex(dest_dir):
252      merge_common.GetCommandStdout(['git', 'commit', '-m',
253          'Update makefiles after merge of Chromium at r%s\n\n%s' %
254          (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
255
256
257def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
258  """Returns whether git's index includes modified files, ie 'added' changes.
259  """
260  status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
261                                         cwd=cwd)
262  return re.search(r'^[MADRC]', status, flags=re.MULTILINE) != None
263
264
265def _GenerateNoticeFile(svn_revision):
266  """Generates a NOTICE file for all third-party code (from Android's
267  perspective) that lives in the Chromium tree and commits it to the root of
268  the repository.
269  Args:
270    svn_revision: The SVN revision for the main Chromium repository
271  """
272
273  print 'Regenerating NOTICE file ...'
274
275  contents = webview_licenses.GenerateNoticeFile()
276
277  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
278    f.write(contents)
279  merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
280  # Only try to commit the NOTICE update if the file has actually changed.
281  if _ModifiedFilesInIndex():
282    merge_common.GetCommandStdout([
283        'git', 'commit', '-m',
284        'Update NOTICE file after merge of Chromium at r%s\n\n%s'
285        % (svn_revision, AUTOGEN_MESSAGE)])
286
287
288def _GenerateLastChange(svn_revision):
289  """Write a build/util/LASTCHANGE file containing the current revision. This is
290  used in the Chromium build to include the version number.
291  Args:
292    svn_revision: The SVN revision for the main Chromium repository
293  """
294
295  print 'Updating LASTCHANGE ...'
296  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
297            'w') as f:
298    f.write("LASTCHANGE=%s\n" % svn_revision)
299  merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
300  if _ModifiedFilesInIndex():
301    merge_common.GetCommandStdout([
302        'git', 'commit', '-m',
303        'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s'
304        % (svn_revision, AUTOGEN_MESSAGE)])
305
306
307def _GetSVNRevisionAndSHA1(git_url, git_branch, svn_revision):
308  print 'Getting SVN revision and SHA1 ...'
309  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
310                                 git_branch + ':cached_upstream'])
311  if svn_revision:
312    # Sometimes, we see multiple commits with the same git SVN ID. No idea why.
313    # We take the most recent.
314    sha1 = merge_common.GetCommandStdout([
315        'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision,
316        '--format=%H', 'cached_upstream']).split()[0]
317  else:
318    # Just use the latest commit.
319    # TODO: We may be able to use a LKGR?
320    commit = merge_common.GetCommandStdout([
321        'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b',
322        'cached_upstream'])
323    sha1 = commit.split()[0]
324    svn_revision = re.search(r'^git-svn-id: .*@([0-9]+)', commit,
325                             flags=re.MULTILINE).group(1)
326  return (svn_revision, sha1)
327
328
329def _Snapshot(git_url, git_branch, svn_revision, autopush):
330  """Takes a snapshot of the specified Chromium tree at the specified SVN
331  revision and merges it into this repository. Also generates Android makefiles
332  and generates a top-level NOTICE file suitable for use in the Android build.
333  Args:
334    git_url: The URL of the git server for the Chromium branch to merge to
335    svn_revision: The SVN revision for the main Chromium repository
336  """
337
338  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_url, git_branch,
339                                                     svn_revision)
340  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
341                                        'HEAD..' + root_sha1]):
342    print ('No new commits to merge from %s branch %s at r%s (%s)' %
343        (git_url, git_branch, svn_revision, root_sha1))
344    return
345
346  print ('Snapshotting Chromium from %s branch %s at r%s (%s)' %
347         (git_url, git_branch, svn_revision, root_sha1))
348
349  # 1. Merge, accounting for excluded directories
350  _MergeProjects(git_url, git_branch, svn_revision, root_sha1)
351
352  # 2. Generate Android NOTICE file
353  _GenerateNoticeFile(svn_revision)
354
355  # 3. Generate LASTCHANGE file
356  _GenerateLastChange(svn_revision)
357
358  # 4. Generate Android makefiles
359  _GenerateMakefiles(svn_revision)
360
361  # 5. Push result to server
362  merge_common.PushToServer(autopush, 'merge-from-chromium', 'master-chromium')
363
364  return True
365
366
367def main():
368  parser = optparse.OptionParser(usage='%prog [options]')
369  parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
370                   'Chromium SVN revision and merges it into this repository. '
371                   'Paths marked as excluded for license reasons are removed '
372                   'as part of the merge. Also generates Android makefiles and '
373                   'generates a top-level NOTICE file suitable for use in the '
374                   'Android build.')
375  parser.add_option(
376      '', '--git_url',
377      default='http://git.chromium.org/chromium/src.git',
378      help=('The URL of the git server for the Chromium branch to merge. '
379            'Defaults to upstream.'))
380  parser.add_option(
381      '', '--git_branch',
382      default='git-svn',
383      help=('The name of the upstream branch to merge. Defaults to git-svn.'))
384  parser.add_option(
385      '', '--svn_revision',
386      default=None,
387      help=('Merge to the specified chromium SVN revision, rather than using '
388            'the current latest revision.'))
389  parser.add_option(
390      '', '--autopush',
391      default=False, action='store_true',
392      help=('Automatically push the result to the server without prompting if'
393            'the merge was successful.'))
394  (options, args) = parser.parse_args()
395  if args:
396    parser.print_help()
397    return 1
398
399  if 'ANDROID_BUILD_TOP' not in os.environ:
400    print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.'
401    return 1
402
403  if not _Snapshot(options.git_url, options.git_branch, options.svn_revision,
404                   options.autopush):
405    return 1
406
407  return 0
408
409if __name__ == '__main__':
410  sys.exit(main())
411