merge_from_chromium.py revision 1f39fd6f96da9ebacdf3462306f3aef6dec2ec0b 
1#!/usr/bin/python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Merge Chromium into the Android tree."""
18
19import contextlib
20import logging
21import optparse
22import os
23import re
24import sys
25import urllib2
26
27import merge_common
28
29
30# We need to import this *after* merging from upstream to get the latest
31# version. Set it to none here to catch uses before it's imported.
32webview_licenses = None
33
34
35AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.'
36SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master'
37
38
39def _ReadGitFile(sha1, path, git_url=None, git_branch=None):
40  """Reads a file from a (possibly remote) git project at a specific revision.
41
42  Args:
43    sha1: The SHA1 at which to read.
44    path: The relative path of the file to read.
45    git_url: The URL of the git server, if reading a remote project.
46    git_branch: The branch to fetch, if reading a remote project.
47  Returns:
48    The contents of the specified file.
49  """
50  if git_url:
51    # We fetch the branch to a temporary head so that we don't download the same
52    # commits multiple times.
53    merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url,
54                                   git_branch + ':cached_upstream'])
55  return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)])
56
57
58def _ParseDEPS(sha1):
59  """Parses the .DEPS.git file from Chromium and returns its contents.
60
61  Args:
62    sha1: The SHA1 at which to read.
63  Returns:
64    A dictionary of the contents of .DEPS.git at the specified revision
65  """
66
67  class FromImpl(object):
68    """Used to implement the From syntax."""
69
70    def __init__(self, module_name):
71      self.module_name = module_name
72
73    def __str__(self):
74      return 'From("%s")' % self.module_name
75
76  class _VarImpl(object):
77    def __init__(self, custom_vars, local_scope):
78      self._custom_vars = custom_vars
79      self._local_scope = local_scope
80
81    def Lookup(self, var_name):
82      """Implements the Var syntax."""
83      if var_name in self._custom_vars:
84        return self._custom_vars[var_name]
85      elif var_name in self._local_scope.get('vars', {}):
86        return self._local_scope['vars'][var_name]
87      raise Exception('Var is not defined: %s' % var_name)
88
89  tmp_locals = {}
90  var = _VarImpl({}, tmp_locals)
91  tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
92  deps_content = _ReadGitFile(sha1, '.DEPS.git')
93  exec(deps_content) in tmp_globals, tmp_locals
94  return tmp_locals
95
96
97def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars):
98  """Gets the git URL and SHA1 for each project based on .DEPS.git.
99
100  Args:
101    third_party_projects: The list of projects to consider.
102    deps_vars: The dictionary of dependencies from .DEPS.git.
103  Returns:
104    A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
105  Raises:
106    TemporaryMergeError: if a project to be merged is not found in .DEPS.git.
107  """
108  deps_fallback_order = [
109      deps_vars['deps'],
110      deps_vars['deps_os']['unix'],
111      deps_vars['deps_os']['android'],
112  ]
113  result = {}
114  for path in third_party_projects:
115    for deps in deps_fallback_order:
116      url_plus_sha1 = deps.get(os.path.join('src', path))
117      if url_plus_sha1:
118        break
119    else:
120      raise merge_common.TemporaryMergeError(
121          'Could not find .DEPS.git entry for project %s. This probably '
122          'means that the project list in merge_from_chromium.py needs to be '
123          'updated.' % path)
124    match = re.match('(.*?)@(.*)', url_plus_sha1)
125    url = match.group(1)
126    sha1 = match.group(2)
127    logging.debug('  Got URL %s and SHA1 %s for project %s', url, sha1, path)
128    result[path] = {'url': url, 'sha1': sha1}
129  return result
130
131
132def _MergeProjects(git_branch, svn_revision, root_sha1, unattended):
133  """Merges each required Chromium project into the Android repository.
134
135  .DEPS.git is consulted to determine which revision each project must be merged
136  at. Only a whitelist of required projects are merged.
137
138  Args:
139    git_branch: The branch in the Chromium repository to merge from.
140    svn_revision: The SVN revision in the Chromium repository to merge from.
141    root_sha1: The git hash corresponding to svn_revision.
142    unattended: Run in unattended mode.
143  Raises:
144    TemporaryMergeError: If incompatibly licensed code is left after pruning.
145  """
146  # The logic for this step lives here, in the Android tree, as it makes no
147  # sense for a Chromium tree to know about this merge.
148
149  if unattended:
150    branch_create_flag = '-B'
151  else:
152    branch_create_flag = '-b'
153  branch_name = 'merge-from-chromium-%s' % svn_revision
154
155  logging.debug('Parsing DEPS ...')
156  deps_vars = _ParseDEPS(root_sha1)
157
158  merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
159                                              deps_vars)
160
161  for path in merge_info:
162    # webkit needs special handling as we have a local mirror
163    local_mirrored = path == 'third_party/WebKit'
164    url = merge_info[path]['url']
165    sha1 = merge_info[path]['sha1']
166    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
167    if local_mirrored:
168      remote = 'history'
169    else:
170      remote = 'goog'
171    merge_common.GetCommandStdout(['git', 'checkout',
172                                   branch_create_flag, branch_name,
173                                   '-t', remote + '/master-chromium'],
174                                  cwd=dest_dir)
175    if not local_mirrored:
176      logging.debug('Fetching project %s at %s ...', path, sha1)
177      merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir)
178    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
179                                     cwd=dest_dir):
180      logging.debug('Merging project %s at %s ...', path, sha1)
181      # Merge conflicts make git merge return 1, so ignore errors
182      merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
183                                    cwd=dest_dir, ignore_errors=True)
184      merge_common.CheckNoConflictsAndCommitMerge(
185          'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
186          cwd=dest_dir, unattended=unattended)
187    else:
188      logging.debug('No new commits to merge in project %s', path)
189
190  # Handle root repository separately.
191  merge_common.GetCommandStdout(['git', 'checkout',
192                                 branch_create_flag, branch_name,
193                                 '-t', 'history/master-chromium'])
194  logging.debug('Merging Chromium at %s ...', root_sha1)
195  # Merge conflicts make git merge return 1, so ignore errors
196  merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1],
197                                ignore_errors=True)
198  merge_common.CheckNoConflictsAndCommitMerge(
199      'Merge Chromium  at r%s (%s)\n\n%s'
200      % (svn_revision, root_sha1, AUTOGEN_MESSAGE), unattended=unattended)
201
202  logging.debug('Getting directories to exclude ...')
203
204  # We import this now that we have merged the latest version.
205  # It imports to a global in order that it can be used to generate NOTICE
206  # later. We also disable writing bytecode to keep the source tree clean.
207  sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
208                               'tools'))
209  sys.dont_write_bytecode = True
210  global webview_licenses
211  import webview_licenses
212  import known_issues
213
214  for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
215    logging.debug('  %s', '\n  '.join(os.path.join(path, x) for x in
216                                      exclude_list))
217    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
218    merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
219                                  exclude_list, cwd=dest_dir)
220    if _ModifiedFilesInIndex(dest_dir):
221      merge_common.GetCommandStdout(['git', 'commit', '-m',
222                                     'Exclude unwanted directories'],
223                                    cwd=dest_dir)
224
225  directories_left_over = webview_licenses.GetIncompatibleDirectories()
226  if directories_left_over:
227    raise merge_common.TemporaryMergeError(
228        'Incompatibly licensed directories remain: ' +
229        '\n'.join(directories_left_over))
230
231
232def _GenerateMakefiles(svn_revision, unattended):
233  """Run gyp to generate the Android build system makefiles.
234
235  Args:
236    svn_revision: The SVN revision to mention in generated commit messages.
237    unattended: Run in unattended mode.
238  """
239  logging.debug('Generating makefiles ...')
240
241  # TODO(torne): The .tmp files are generated by
242  # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
243  # tree. We should avoid this, or at least use a more specific name to avoid
244  # accidentally removing or adding other files.
245  for path in merge_common.ALL_PROJECTS:
246    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
247    merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
248                                   'GypAndroid.*.mk', '*.target.*.mk',
249                                   '*.host.*.mk', '*.tmp'], cwd=dest_dir)
250
251  try:
252    merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all'])
253  except merge_common.MergeError as e:
254    if not unattended:
255      raise
256    else:
257      for path in merge_common.ALL_PROJECTS:
258        merge_common.GetCommandStdout(
259            ['git', 'reset', '--hard'],
260            cwd=os.path.join(merge_common.REPOSITORY_ROOT, path))
261      raise merge_common.TemporaryMergeError('Makefile generation failed: ' +
262                                             str(e))
263
264  for path in merge_common.ALL_PROJECTS:
265    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
266    # git add doesn't have an --ignore-unmatch so we have to do this instead:
267    merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'],
268                                  ignore_errors=True, cwd=dest_dir)
269    merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'],
270                                  ignore_errors=True, cwd=dest_dir)
271    merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'],
272                                  ignore_errors=True, cwd=dest_dir)
273    merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
274                                  ignore_errors=True, cwd=dest_dir)
275    # Only try to commit the makefiles if something has actually changed.
276    if _ModifiedFilesInIndex(dest_dir):
277      merge_common.GetCommandStdout(
278          ['git', 'commit', '-m',
279           'Update makefiles after merge of Chromium at r%s\n\n%s' %
280           (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
281
282
283def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
284  """Returns true if git's index contains any changes."""
285  status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
286                                         cwd=cwd)
287  return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None
288
289
290def _GenerateNoticeFile(svn_revision):
291  """Generates and commits a NOTICE file containing code licenses.
292
293  This covers all third-party code (from Android's perspective) that lives in
294  the Chromium tree.
295
296  Args:
297    svn_revision: The SVN revision for the main Chromium repository.
298  """
299  logging.debug('Regenerating NOTICE file ...')
300
301  contents = webview_licenses.GenerateNoticeFile()
302
303  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
304    f.write(contents)
305  merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
306  # Only try to commit the NOTICE update if the file has actually changed.
307  if _ModifiedFilesInIndex():
308    merge_common.GetCommandStdout([
309        'git', 'commit', '-m',
310        'Update NOTICE file after merge of Chromium at r%s\n\n%s'
311        % (svn_revision, AUTOGEN_MESSAGE)])
312
313
314def _GenerateLastChange(svn_revision):
315  """Write a build/util/LASTCHANGE file containing the current revision.
316
317  The revision number is compiled into the binary at build time from this file.
318
319  Args:
320    svn_revision: The SVN revision for the main Chromium repository.
321  """
322  logging.debug('Updating LASTCHANGE ...')
323  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
324            'w') as f:
325    f.write('LASTCHANGE=%s\n' % svn_revision)
326  merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
327  if _ModifiedFilesInIndex():
328    merge_common.GetCommandStdout([
329        'git', 'commit', '-m',
330        'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s'
331        % (svn_revision, AUTOGEN_MESSAGE)])
332
333
334def GetLKGR():
335  """Fetch the last known good release from Chromium's dashboard.
336
337  Returns:
338    The last known good SVN revision.
339  """
340  with contextlib.closing(
341      urllib2.urlopen('https://chromium-status.appspot.com/lkgr')) as lkgr:
342    return int(lkgr.read())
343
344
345def GetHEAD():
346  """Fetch the latest HEAD revision from the git mirror of the Chromium svn
347  repo.
348
349  Returns:
350    The latest HEAD SVN revision.
351  """
352  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(SRC_GIT_BRANCH,
353                                                     'HEAD')
354  return int(svn_revision)
355
356
357def _ParseSvnRevisionFromGitCommitMessage(commit_message):
358  return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message,
359                   flags=re.MULTILINE).group(1)
360
361
362def _GetSVNRevisionFromSha(sha1):
363  commit = merge_common.GetCommandStdout([
364      'git', 'show', '--format=%H%n%b', sha1])
365  return _ParseSvnRevisionFromGitCommitMessage(commit)
366
367
368def _GetSVNRevisionAndSHA1(git_branch, svn_revision, sha1=None):
369  logging.debug('Getting SVN revision and SHA1 ...')
370
371  if sha1:
372    return (_GetSVNRevisionFromSha(sha1), sha1)
373
374  if svn_revision == 'HEAD':
375    # Just use the latest commit.
376    commit = merge_common.GetCommandStdout([
377        'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b',
378        git_branch])
379    sha1 = commit.split()[0]
380    svn_revision = _ParseSvnRevisionFromGitCommitMessage(commit)
381    return (svn_revision, sha1)
382
383  if svn_revision is None:
384    # Fetch LKGR from upstream.
385    svn_revision = GetLKGR()
386  output = merge_common.GetCommandStdout([
387      'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision,
388      '--format=%H', git_branch])
389  if not output:
390    raise merge_common.TemporaryMergeError('Revision %s not found in git repo.'
391                                           % svn_revision)
392  # The log grep will sometimes match reverts/reapplies of commits. We take the
393  # oldest (last) match because the first time it appears in history is
394  # overwhelmingly likely to be the correct commit.
395  sha1 = output.split()[-1]
396  return (svn_revision, sha1)
397
398
399def Snapshot(svn_revision, root_sha1, unattended):
400  """Takes a snapshot of the Chromium tree and merges it into Android.
401
402  Android makefiles and a top-level NOTICE file are generated and committed
403  after the merge.
404
405  Args:
406    svn_revision: The SVN revision in the Chromium repository to merge from.
407    root_sha1: The sha1 in the Chromium git mirror to merge from. Only one of
408               svn_revision and root_sha1 should be specified.
409    unattended: Run in unattended mode.
410
411  Returns:
412    True if new commits were merged; False if no new commits were present.
413  """
414  git_branch = SRC_GIT_BRANCH
415  (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_branch, svn_revision,
416                                                     root_sha1)
417  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
418                                        'HEAD..' + root_sha1]):
419    logging.info('No new commits to merge from %s at r%s (%s)',
420                 git_branch, svn_revision, root_sha1)
421    return False
422
423  logging.info('Snapshotting Chromium from %s at r%s (%s)',
424               git_branch, svn_revision, root_sha1)
425
426  # 1. Merge, accounting for excluded directories
427  _MergeProjects(git_branch, svn_revision, root_sha1, unattended)
428
429  # 2. Generate Android NOTICE file
430  _GenerateNoticeFile(svn_revision)
431
432  # 3. Generate LASTCHANGE file
433  _GenerateLastChange(svn_revision)
434
435  # 4. Generate Android makefiles
436  _GenerateMakefiles(svn_revision, unattended)
437
438  return True
439
440
441def Push(svn_revision):
442  """Push the finished snapshot to the Android repository."""
443  src = 'merge-from-chromium-%s' % svn_revision
444  for branch in ['master-chromium-merge', 'master-chromium']:
445    logging.debug('Pushing to server (%s) ...' % branch)
446    for path in merge_common.ALL_PROJECTS:
447      if path in merge_common.PROJECTS_WITH_FLAT_HISTORY:
448        remote = 'history'
449      else:
450        remote = 'goog'
451      logging.debug('Pushing %s', path)
452      dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
453      merge_common.GetCommandStdout(['git', 'push', '-f', remote,
454                                    src + ':' + branch], cwd=dest_dir)
455
456
457def main():
458  parser = optparse.OptionParser(usage='%prog [options]')
459  parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
460                   'Chromium SVN revision and merges it into this repository. '
461                   'Paths marked as excluded for license reasons are removed '
462                   'as part of the merge. Also generates Android makefiles and '
463                   'generates a top-level NOTICE file suitable for use in the '
464                   'Android build.')
465  parser.add_option(
466      '', '--svn_revision',
467      default=None,
468      help=('Merge to the specified chromium SVN revision, rather than using '
469            'the current LKGR. Can also pass HEAD to merge from tip of tree. '
470            'Only one of svn_revision and sha1 should be specified'))
471  parser.add_option(
472      '', '--sha1',
473      default=None,
474      help=('Merge to the specified chromium sha1 revision from ' + SRC_GIT_BRANCH
475            + ' branch, rather than using the current LKGR. Only one of'
476             'svn_revision and sha1 should be specified.'))
477  parser.add_option(
478      '', '--push',
479      default=False, action='store_true',
480      help=('Push the result of a previous merge to the server. Note '
481            'svn_revision must be given.'))
482  parser.add_option(
483      '', '--get_lkgr',
484      default=False, action='store_true',
485      help=('Just print the current LKGR on stdout and exit.'))
486  parser.add_option(
487      '', '--get_head',
488      default=False, action='store_true',
489      help=('Just print the current HEAD revision on stdout and exit.'))
490  parser.add_option(
491      '', '--unattended',
492      default=False, action='store_true',
493      help=('Run in unattended mode.'))
494  parser.add_option(
495      '', '--no_changes_exit',
496      default=0, type='int',
497      help=('Exit code to use if there are no changes to merge, for scripts.'))
498  (options, args) = parser.parse_args()
499  if args:
500    parser.print_help()
501    return 1
502
503  if 'ANDROID_BUILD_TOP' not in os.environ:
504    print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.'
505    return 1
506
507  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
508                      stream=sys.stdout)
509
510  if options.get_lkgr:
511    print GetLKGR()
512  elif options.get_head:
513    logging.disable(logging.CRITICAL)  # Prevent log messages
514    print GetHEAD()
515  elif options.push:
516    if options.svn_revision is None:
517      print >>sys.stderr, 'You need to pass the SVN revision to push.'
518      return 1
519    else:
520      Push(options.svn_revision)
521  else:
522    if not Snapshot(options.svn_revision, options.sha1, options.unattended):
523      return options.no_changes_exit
524
525  return 0
526
527if __name__ == '__main__':
528  sys.exit(main())
529