merge_from_chromium.py revision 9fb672be51066b51a372885377cc0eff78e1a34c 
1#!/usr/bin/python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Merge Chromium into the Android tree."""
18
19import contextlib
20import logging
21import optparse
22import os
23import re
24import sys
25import urllib2
26
27import merge_common
28
29
30# We need to import this *after* merging from upstream to get the latest
31# version. Set it to none here to catch uses before it's imported.
32webview_licenses = None
33
34
35AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.'
36
37
38def _ReadGitFile(git_url, git_branch, sha1, path):
39  """Reads a file from a remote git project at a specific revision.
40
41  Args:
42    git_url: The URL of the git server.
43    git_branch: The branch to fetch.
44    sha1: The SHA1 at which to read.
45    path: The relative path of the file to read.
46  Returns:
47    The contents of the specified file.
48  """
49  # We fetch the branch to a temporary head so that we don't download the same
50  # commits multiple times.
51  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
52                                 git_branch + ':cached_upstream'])
53  return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)])
54
55
56def _ParseDEPS(git_url, git_branch, sha1):
57  """Parses the .DEPS.git file from Chromium and returns its contents.
58
59  Args:
60    git_url: The URL of the git server.
61    git_branch: The branch to read.
62    sha1: The SHA1 at which to read.
63  Returns:
64    A dictionary of the contents of .DEPS.git at the specified revision
65  """
66
67  class FromImpl(object):
68    """Used to implement the From syntax."""
69
70    def __init__(self, module_name):
71      self.module_name = module_name
72
73    def __str__(self):
74      return 'From("%s")' % self.module_name
75
76  class _VarImpl(object):
77    def __init__(self, custom_vars, local_scope):
78      self._custom_vars = custom_vars
79      self._local_scope = local_scope
80
81    def Lookup(self, var_name):
82      """Implements the Var syntax."""
83      if var_name in self._custom_vars:
84        return self._custom_vars[var_name]
85      elif var_name in self._local_scope.get('vars', {}):
86        return self._local_scope['vars'][var_name]
87      raise Exception('Var is not defined: %s' % var_name)
88
89  tmp_locals = {}
90  var = _VarImpl({}, tmp_locals)
91  tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
92  deps_content = _ReadGitFile(git_url, git_branch, sha1, '.DEPS.git')
93  exec(deps_content) in tmp_globals, tmp_locals
94  return tmp_locals
95
96
97def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars):
98  """Gets the git URL and SHA1 for each project based on .DEPS.git.
99
100  Args:
101    third_party_projects: The list of projects to consider.
102    deps_vars: The dictionary of dependencies from .DEPS.git.
103  Returns:
104    A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
105  Raises:
106    TemporaryMergeError: if a project to be merged is not found in .DEPS.git.
107  """
108  deps_fallback_order = [
109      deps_vars['deps'],
110      deps_vars['deps_os']['unix'],
111      deps_vars['deps_os']['android'],
112  ]
113  result = {}
114  for path in third_party_projects:
115    for deps in deps_fallback_order:
116      url_plus_sha1 = deps.get(os.path.join('src', path))
117      if url_plus_sha1:
118        break
119    else:
120      raise merge_common.TemporaryMergeError(
121          'Could not find .DEPS.git entry for project %s. This probably '
122          'means that the project list in merge_from_chromium.py needs to be '
123          'updated.' % path)
124    match = re.match('(.*?)@(.*)', url_plus_sha1)
125    url = match.group(1)
126    sha1 = match.group(2)
127    logging.debug('  Got URL %s and SHA1 %s for project %s', url, sha1, path)
128    result[path] = {'url': url, 'sha1': sha1}
129  return result
130
131
132def _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended):
133  """Merges each required Chromium project into the Android repository.
134
135  .DEPS.git is consulted to determine which revision each project must be merged
136  at. Only a whitelist of required projects are merged.
137
138  Args:
139    git_url: The URL of the Chromium repository to merge from.
140    git_branch: The branch in the Chromium repository to merge from.
141    svn_revision: The SVN revision in the Chromium repository to merge from.
142    root_sha1: The git hash corresponding to svn_revision.
143    unattended: Run in unattended mode.
144  Raises:
145    TemporaryMergeError: If incompatibly licensed code is left after pruning.
146  """
147  # The logic for this step lives here, in the Android tree, as it makes no
148  # sense for a Chromium tree to know about this merge.
149
150  if unattended:
151    branch_create_flag = '-B'
152  else:
153    branch_create_flag = '-b'
154  branch_name = 'merge-from-chromium-%s' % svn_revision
155
156  logging.debug('Parsing DEPS ...')
157  deps_vars = _ParseDEPS(git_url, git_branch, root_sha1)
158
159  merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
160                                              deps_vars)
161
162  for path in merge_info:
163    url = merge_info[path]['url']
164    sha1 = merge_info[path]['sha1']
165    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
166    merge_common.GetCommandStdout(['git', 'checkout',
167                                   branch_create_flag, branch_name,
168                                   '-t', 'goog/master-chromium'], cwd=dest_dir)
169    logging.debug('Fetching project %s at %s ...', path, sha1)
170    merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir)
171    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
172                                     cwd=dest_dir):
173      logging.debug('Merging project %s at %s ...', path, sha1)
174      # Merge conflicts make git merge return 1, so ignore errors
175      merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
176                                    cwd=dest_dir, ignore_errors=True)
177      merge_common.CheckNoConflictsAndCommitMerge(
178          'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
179          cwd=dest_dir, unattended=unattended)
180    else:
181      logging.debug('No new commits to merge in project %s', path)
182
183  # Handle root repository separately.
184  merge_common.GetCommandStdout(['git', 'checkout',
185                                 branch_create_flag, branch_name,
186                                 '-t', 'goog/master-chromium'])
187  logging.debug('Fetching Chromium at %s ...', root_sha1)
188  merge_common.GetCommandStdout(['git', 'fetch', git_url, git_branch])
189  logging.debug('Merging Chromium at %s ...', root_sha1)
190  # Merge conflicts make git merge return 1, so ignore errors
191  merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1],
192                                ignore_errors=True)
193  merge_common.CheckNoConflictsAndCommitMerge(
194      'Merge Chromium from %s branch %s at r%s (%s)\n\n%s'
195      % (git_url, git_branch, svn_revision, root_sha1, AUTOGEN_MESSAGE),
196      unattended=unattended)
197
198  logging.debug('Getting directories to exclude ...')
199
200  # We import this now that we have merged the latest version.
201  # It imports to a global in order that it can be used to generate NOTICE
202  # later. We also disable writing bytecode to keep the source tree clean.
203  sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
204                               'tools'))
205  sys.dont_write_bytecode = True
206  global webview_licenses
207  import webview_licenses
208  import known_issues
209
210  for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
211    logging.debug('  %s', '\n  '.join(os.path.join(path, x) for x in
212                                      exclude_list))
213    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
214    merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
215                                  exclude_list, cwd=dest_dir)
216    if _ModifiedFilesInIndex(dest_dir):
217      merge_common.GetCommandStdout(['git', 'commit', '-m',
218                                     'Exclude incompatible directories'],
219                                    cwd=dest_dir)
220
221  directories_left_over = webview_licenses.GetIncompatibleDirectories()
222  if directories_left_over:
223    raise merge_common.TemporaryMergeError(
224        'Incompatibly licensed directories remain: ' +
225        '\n'.join(directories_left_over))
226
227
228def _GenerateMakefiles(svn_revision, unattended):
229  """Run gyp to generate the Android build system makefiles.
230
231  Args:
232    svn_revision: The SVN revision to mention in generated commit messages.
233    unattended: Run in unattended mode.
234  """
235  # TODO(torne): The .tmp files are generated by
236  # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
237  # tree. We should avoid this, or at least use a more specific name to avoid
238  # accidentally removing or adding other files.
239  for path in merge_common.ALL_PROJECTS:
240    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
241    merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
242                                   'GypAndroid.*.mk', '*.target.*.mk',
243                                   '*.host.*.mk', '*.tmp'], cwd=dest_dir)
244
245  try:
246    merge_common.GetCommandStdout(['android_webview/tools/gyp_webview'])
247  except merge_common.MergeError as e:
248    if not unattended:
249      raise
250    else:
251      for path in merge_common.ALL_PROJECTS:
252        merge_common.GetCommandStdout(
253            ['git', 'reset', '--hard'],
254            cwd=os.path.join(merge_common.REPOSITORY_ROOT, path))
255      raise merge_common.TemporaryMergeError('Makefile generation failed: ' +
256                                             str(e))
257
258  for path in merge_common.ALL_PROJECTS:
259    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
260    # git add doesn't have an --ignore-unmatch so we have to do this instead:
261    merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'],
262                                  ignore_errors=True, cwd=dest_dir)
263    merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'],
264                                  ignore_errors=True, cwd=dest_dir)
265    merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'],
266                                  ignore_errors=True, cwd=dest_dir)
267    merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
268                                  ignore_errors=True, cwd=dest_dir)
269    # Only try to commit the makefiles if something has actually changed.
270    if _ModifiedFilesInIndex(dest_dir):
271      merge_common.GetCommandStdout(
272          ['git', 'commit', '-m',
273           'Update makefiles after merge of Chromium at r%s\n\n%s' %
274           (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
275
276
277def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
278  """Returns true if git's index contains any changes."""
279  status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
280                                         cwd=cwd)
281  return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None
282
283
284def _GenerateNoticeFile(svn_revision):
285  """Generates and commits a NOTICE file containing code licenses.
286
287  This covers all third-party code (from Android's perspective) that lives in
288  the Chromium tree.
289
290  Args:
291    svn_revision: The SVN revision for the main Chromium repository.
292  """
293  logging.debug('Regenerating NOTICE file ...')
294
295  contents = webview_licenses.GenerateNoticeFile()
296
297  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
298    f.write(contents)
299  merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
300  # Only try to commit the NOTICE update if the file has actually changed.
301  if _ModifiedFilesInIndex():
302    merge_common.GetCommandStdout([
303        'git', 'commit', '-m',
304        'Update NOTICE file after merge of Chromium at r%s\n\n%s'
305        % (svn_revision, AUTOGEN_MESSAGE)])
306
307
308def _GenerateLastChange(svn_revision):
309  """Write a build/util/LASTCHANGE file containing the current revision.
310
311  The revision number is compiled into the binary at build time from this file.
312
313  Args:
314    svn_revision: The SVN revision for the main Chromium repository.
315  """
316  logging.debug('Updating LASTCHANGE ...')
317  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
318            'w') as f:
319    f.write('LASTCHANGE=%s\n' % svn_revision)
320  merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
321  if _ModifiedFilesInIndex():
322    merge_common.GetCommandStdout([
323        'git', 'commit', '-m',
324        'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s'
325        % (svn_revision, AUTOGEN_MESSAGE)])
326
327
328def GetLKGR():
329  """Fetch the last known good release from Chromium's dashboard.
330
331  Returns:
332    The last known good SVN revision.
333  """
334  with contextlib.closing(
335      urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr:
336    return int(lkgr.read())
337
338
339def _GetSVNRevisionAndSHA1(git_url, git_branch, svn_revision):
340  logging.debug('Getting SVN revision and SHA1 ...')
341  merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
342                                 git_branch + ':cached_upstream'])
343  if svn_revision == 'HEAD':
344    # Just use the latest commit.
345    commit = merge_common.GetCommandStdout([
346        'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b',
347        'cached_upstream'])
348    sha1 = commit.split()[0]
349    svn_revision = re.search(r'^git-svn-id: .*@([0-9]+)', commit,
350                             flags=re.MULTILINE).group(1)
351    return (svn_revision, sha1)
352
353  if svn_revision is None:
354    # Fetch LKGR from upstream.
355    svn_revision = GetLKGR()
356  output = merge_common.GetCommandStdout([
357      'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision,
358      '--format=%H', 'cached_upstream'])
359  if not output:
360    raise merge_common.TemporaryMergeError('Revision %s not found in git repo.'
361                                           % svn_revision)
362  # The log grep will sometimes match reverts/reapplies of commits. We take the
363  # oldest (last) match because the first time it appears in history is
364  # overwhelmingly likely to be the correct commit.
365  sha1 = output.split()[-1]
366  return (svn_revision, sha1)
367
368
369def Snapshot(svn_revision, unattended):
370  """Takes a snapshot of the Chromium tree and merges it into Android.
371
372  Android makefiles and a top-level NOTICE file are generated and committed
373  after the merge.
374
375  Args:
376    svn_revision: The SVN revision in the Chromium repository to merge from.
377    unattended: Run in unattended mode.
378
379  Returns:
380    True if new commits were merged; False if no new commits were present.
381  """
382  git_url = 'http://chromium.googlesource.com/chromium/src.git'
383  git_branch = 'git-svn'
384  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_url, git_branch,
385                                                     svn_revision)
386  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
387                                        'HEAD..' + root_sha1]):
388    logging.info('No new commits to merge from %s branch %s at r%s (%s)',
389                 git_url, git_branch, svn_revision, root_sha1)
390    return False
391
392  logging.info('Snapshotting Chromium from %s branch %s at r%s (%s)',
393               git_url, git_branch, svn_revision, root_sha1)
394
395  # 1. Merge, accounting for excluded directories
396  _MergeProjects(git_url, git_branch, svn_revision, root_sha1, unattended)
397
398  # 2. Generate Android NOTICE file
399  _GenerateNoticeFile(svn_revision)
400
401  # 3. Generate LASTCHANGE file
402  _GenerateLastChange(svn_revision)
403
404  # 4. Generate Android makefiles
405  _GenerateMakefiles(svn_revision, unattended)
406
407  return True
408
409
410def Push(svn_revision):
411  """Push the finished snapshot to the Android repository."""
412  merge_common.PushToServer('merge-from-chromium-%s' % svn_revision,
413                            'master-chromium', 'master-chromium-merge')
414
415
416def main():
417  parser = optparse.OptionParser(usage='%prog [options]')
418  parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
419                   'Chromium SVN revision and merges it into this repository. '
420                   'Paths marked as excluded for license reasons are removed '
421                   'as part of the merge. Also generates Android makefiles and '
422                   'generates a top-level NOTICE file suitable for use in the '
423                   'Android build.')
424  parser.add_option(
425      '', '--svn_revision',
426      default=None,
427      help=('Merge to the specified chromium SVN revision, rather than using '
428            'the current LKGR. Can also pass HEAD to merge from tip of tree.'))
429  parser.add_option(
430      '', '--push',
431      default=False, action='store_true',
432      help=('Push the result of a previous merge to the server.'))
433  parser.add_option(
434      '', '--get_lkgr',
435      default=False, action='store_true',
436      help=('Just print the current LKGR on stdout and exit.'))
437  parser.add_option(
438      '', '--unattended',
439      default=False, action='store_true',
440      help=('Run in unattended mode.'))
441  (options, args) = parser.parse_args()
442  if args:
443    parser.print_help()
444    return 1
445
446  if 'ANDROID_BUILD_TOP' not in os.environ:
447    print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.'
448    return 1
449
450  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
451                      stream=sys.stdout)
452
453  if options.get_lkgr:
454    print GetLKGR()
455  elif options.push:
456    if options.svn_revision is None:
457      print >>sys.stderr, 'You need to pass the SVN revision to push.'
458      return 1
459    else:
460      Push(options.svn_revision)
461  else:
462    Snapshot(options.svn_revision, options.unattended)
463
464  return 0
465
466if __name__ == '__main__':
467  sys.exit(main())
468