1#!/usr/bin/python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Merge Chromium into the Android tree."""
18
19import logging
20import optparse
21import os
22import re
23import sys
24
25import merge_common
26
27
28# We need to import this *after* merging from upstream to get the latest
29# version. Set it to none here to catch uses before it's imported.
30webview_licenses = None
31
32
33AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.'
34SRC_GIT_BRANCH = 'refs/remotes/history/upstream-master'
35
36
37def _ReadGitFile(sha1, path, git_url=None, git_branch=None):
38  """Reads a file from a (possibly remote) git project at a specific revision.
39
40  Args:
41    sha1: The SHA1 at which to read.
42    path: The relative path of the file to read.
43    git_url: The URL of the git server, if reading a remote project.
44    git_branch: The branch to fetch, if reading a remote project.
45  Returns:
46    The contents of the specified file.
47  """
48  if git_url:
49    merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, git_branch])
50  return merge_common.GetCommandStdout(['git', 'show', '%s:%s' % (sha1, path)])
51
52
53def _ParseDEPS(deps_content):
54  """Parses the DEPS file from Chromium and returns its contents.
55
56  Args:
57    deps_content: The contents of the DEPS file as text.
58  Returns:
59    A dictionary of the contents of DEPS at the specified revision
60  """
61
62  class FromImpl(object):
63    """Used to implement the From syntax."""
64
65    def __init__(self, module_name):
66      self.module_name = module_name
67
68    def __str__(self):
69      return 'From("%s")' % self.module_name
70
71  class _VarImpl(object):
72
73    def __init__(self, custom_vars, local_scope):
74      self._custom_vars = custom_vars
75      self._local_scope = local_scope
76
77    def Lookup(self, var_name):
78      """Implements the Var syntax."""
79      if var_name in self._custom_vars:
80        return self._custom_vars[var_name]
81      elif var_name in self._local_scope.get('vars', {}):
82        return self._local_scope['vars'][var_name]
83      raise Exception('Var is not defined: %s' % var_name)
84
85  tmp_locals = {}
86  var = _VarImpl({}, tmp_locals)
87  tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}}
88  exec(deps_content) in tmp_globals, tmp_locals  # pylint: disable=W0122
89  return tmp_locals
90
91
92def _GetProjectMergeInfo(projects, deps_vars):
93  """Gets the git URL and SHA1 for each project based on DEPS.
94
95  Args:
96    projects: The list of projects to consider.
97    deps_vars: The dictionary of dependencies from DEPS.
98  Returns:
99    A dictionary from project to git URL and SHA1 - 'path: (url, sha1)'
100  Raises:
101    TemporaryMergeError: if a project to be merged is not found in DEPS.
102  """
103  deps_fallback_order = [
104      deps_vars['deps'],
105      deps_vars['deps_os']['unix'],
106      deps_vars['deps_os']['android'],
107  ]
108  result = {}
109  for path in projects:
110    for deps in deps_fallback_order:
111      if path:
112        upstream_path = os.path.join('src', path)
113      else:
114        upstream_path = 'src'
115      url_plus_sha1 = deps.get(upstream_path)
116      if url_plus_sha1:
117        break
118    else:
119      raise merge_common.TemporaryMergeError(
120          'Could not find DEPS entry for project %s. This probably '
121          'means that the project list in merge_from_chromium.py needs to be '
122          'updated.' % path)
123    match = re.match('(.*?)@(.*)', url_plus_sha1)
124    url = match.group(1)
125    sha1 = match.group(2)
126    logging.debug('  Got URL %s and SHA1 %s for project %s', url, sha1, path)
127    result[path] = {'url': url, 'sha1': sha1}
128  return result
129
130
131def _MergeProjects(version, root_sha1, target, unattended, buildspec_url):
132  """Merges each required Chromium project into the Android repository.
133
134  DEPS is consulted to determine which revision each project must be merged
135  at. Only a whitelist of required projects are merged.
136
137  Args:
138    version: The version to mention in generated commit messages.
139    root_sha1: The git hash to merge in the root repository.
140    target: The target branch to merge to.
141    unattended: Run in unattended mode.
142    buildspec_url: URL for buildspec repository, when merging a branch.
143  Returns:
144    The abbrev sha1 merged. It will be either |root_sha1| itself (when merging
145    chromium trunk) or the upstream sha1 of the release.
146  Raises:
147    TemporaryMergeError: If incompatibly licensed code is left after pruning.
148  """
149  # The logic for this step lives here, in the Android tree, as it makes no
150  # sense for a Chromium tree to know about this merge.
151
152  if unattended:
153    branch_create_flag = '-B'
154  else:
155    branch_create_flag = '-b'
156  branch_name = 'merge-from-chromium-%s' % version
157
158  logging.debug('Parsing DEPS ...')
159  if root_sha1:
160    deps_content = _ReadGitFile(root_sha1, 'DEPS')
161  else:
162    # TODO(primiano): At some point the release branches will use DEPS as well,
163    # instead of .DEPS.git. Rename below when that day will come.
164    deps_content = _ReadGitFile('FETCH_HEAD',
165                                'releases/' + version + '/.DEPS.git',
166                                buildspec_url,
167                                'master')
168
169  deps_vars = _ParseDEPS(deps_content)
170
171  merge_info = _GetProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS,
172                                    deps_vars)
173
174  for path in merge_info:
175    # webkit needs special handling as we have a local mirror
176    local_mirrored = path == 'third_party/WebKit'
177    url = merge_info[path]['url']
178    sha1 = merge_info[path]['sha1']
179    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
180    if local_mirrored:
181      remote = 'history'
182    else:
183      remote = 'goog'
184    merge_common.GetCommandStdout(['git', 'checkout',
185                                   branch_create_flag, branch_name,
186                                   '-t', remote + '/' + target],
187                                  cwd=dest_dir)
188    if not local_mirrored or not root_sha1:
189      logging.debug('Fetching project %s at %s ...', path, sha1)
190      fetch_args = ['git', 'fetch', url, sha1]
191      merge_common.GetCommandStdout(fetch_args, cwd=dest_dir)
192    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1],
193                                     cwd=dest_dir):
194      logging.debug('Merging project %s at %s ...', path, sha1)
195      # Merge conflicts make git merge return 1, so ignore errors
196      merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1],
197                                    cwd=dest_dir, ignore_errors=True)
198      merge_common.CheckNoConflictsAndCommitMerge(
199          'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE),
200          cwd=dest_dir, unattended=unattended)
201    else:
202      logging.debug('No new commits to merge in project %s', path)
203
204  # Handle root repository separately.
205  merge_common.GetCommandStdout(['git', 'checkout',
206                                 branch_create_flag, branch_name,
207                                 '-t', 'history/' + target])
208  if not root_sha1:
209    merge_info = _GetProjectMergeInfo([''], deps_vars)
210    url = merge_info['']['url']
211    merged_sha1 = merge_info['']['sha1']
212    merge_common.GetCommandStdout(['git', 'fetch', url, merged_sha1])
213    merged_sha1 = merge_common.Abbrev(merged_sha1)
214    merge_msg_version = '%s (%s)' % (version, merged_sha1)
215  else:
216    merge_msg_version = root_sha1
217    merged_sha1 = root_sha1
218
219  logging.debug('Merging Chromium at %s ...', merged_sha1)
220  # Merge conflicts make git merge return 1, so ignore errors
221  merge_common.GetCommandStdout(['git', 'merge', '--no-commit', merged_sha1],
222                                ignore_errors=True)
223  merge_common.CheckNoConflictsAndCommitMerge(
224      'Merge Chromium at %s\n\n%s'
225      % (merge_msg_version, AUTOGEN_MESSAGE), unattended=unattended)
226
227  logging.debug('Getting directories to exclude ...')
228
229  # We import this now that we have merged the latest version.
230  # It imports to a global in order that it can be used to generate NOTICE
231  # later. We also disable writing bytecode to keep the source tree clean.
232  sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview',
233                               'tools'))
234  sys.dont_write_bytecode = True
235  global webview_licenses  # pylint: disable=W0602
236  import webview_licenses  # pylint: disable=W0621,W0612,C6204
237  import known_issues  # pylint: disable=C6204
238
239  for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems():
240    logging.debug('  %s', '\n  '.join(os.path.join(path, x) for x in
241                                      exclude_list))
242    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
243    merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
244                                  exclude_list, cwd=dest_dir)
245    if _ModifiedFilesInIndex(dest_dir):
246      merge_common.GetCommandStdout(['git', 'commit', '-m',
247                                     'Exclude unwanted directories'],
248                                    cwd=dest_dir)
249  assert(root_sha1 is None or root_sha1 == merged_sha1)
250  return merged_sha1
251
252
253def _CheckLicenses():
254  """Check that no incompatibly licensed directories exist."""
255  directories_left_over = webview_licenses.GetIncompatibleDirectories()
256  if directories_left_over:
257    raise merge_common.TemporaryMergeError(
258        'Incompatibly licensed directories remain: ' +
259        '\n'.join(directories_left_over))
260
261
262def _GenerateMakefiles(version, unattended):
263  """Run gyp to generate the Android build system makefiles.
264
265  Args:
266    version: The version to mention in generated commit messages.
267    unattended: Run in unattended mode.
268  """
269  logging.debug('Generating makefiles ...')
270
271  # TODO(torne): come up with a way to deal with hooks from DEPS properly
272
273  # TODO(torne): The .tmp files are generated by
274  # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source
275  # tree. We should avoid this, or at least use a more specific name to avoid
276  # accidentally removing or adding other files.
277  for path in merge_common.ALL_PROJECTS:
278    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
279    merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch',
280                                   'GypAndroid.*.mk', '*.target.*.mk',
281                                   '*.host.*.mk', '*.tmp'], cwd=dest_dir)
282
283  try:
284    merge_common.GetCommandStdout(['android_webview/tools/gyp_webview', 'all'])
285  except merge_common.MergeError as e:
286    if not unattended:
287      raise
288    else:
289      for path in merge_common.ALL_PROJECTS:
290        merge_common.GetCommandStdout(
291            ['git', 'reset', '--hard'],
292            cwd=os.path.join(merge_common.REPOSITORY_ROOT, path))
293      raise merge_common.TemporaryMergeError('Makefile generation failed: ' +
294                                             str(e))
295
296  for path in merge_common.ALL_PROJECTS:
297    dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
298    # git add doesn't have an --ignore-unmatch so we have to do this instead:
299    merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'],
300                                  ignore_errors=True, cwd=dest_dir)
301    merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'],
302                                  ignore_errors=True, cwd=dest_dir)
303    merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'],
304                                  ignore_errors=True, cwd=dest_dir)
305    merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'],
306                                  ignore_errors=True, cwd=dest_dir)
307    # Only try to commit the makefiles if something has actually changed.
308    if _ModifiedFilesInIndex(dest_dir):
309      merge_common.GetCommandStdout(
310          ['git', 'commit', '-m',
311           'Update makefiles after merge of Chromium at %s\n\n%s' %
312           (version, AUTOGEN_MESSAGE)], cwd=dest_dir)
313
314
315def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT):
316  """Returns true if git's index contains any changes."""
317  status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'],
318                                         cwd=cwd)
319  return re.search(r'^[MADRC]', status, flags=re.MULTILINE) is not None
320
321
322def _GenerateNoticeFile(version):
323  """Generates and commits a NOTICE file containing code licenses.
324
325  This covers all third-party code (from Android's perspective) that lives in
326  the Chromium tree.
327
328  Args:
329    version: The version to mention in generated commit messages.
330  """
331  logging.debug('Regenerating NOTICE file ...')
332
333  contents = webview_licenses.GenerateNoticeFile()
334
335  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f:
336    f.write(contents)
337  merge_common.GetCommandStdout(['git', 'add', 'NOTICE'])
338  # Only try to commit the NOTICE update if the file has actually changed.
339  if _ModifiedFilesInIndex():
340    merge_common.GetCommandStdout([
341        'git', 'commit', '-m',
342        'Update NOTICE file after merge of Chromium at %s\n\n%s'
343        % (version, AUTOGEN_MESSAGE)])
344
345
346def _GenerateLastChange(version, root_sha1):
347  """Write a build/util/LASTCHANGE file containing the current revision.
348
349  The revision number is compiled into the binary at build time from this file.
350
351  Args:
352    version: The version to mention in generated commit messages.
353    root_sha1: The SHA1 of the main project (before the merge).
354  """
355  logging.debug('Updating LASTCHANGE ...')
356  with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'),
357            'w') as f:
358    f.write('LASTCHANGE=%s\n' % merge_common.Abbrev(root_sha1))
359  merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE'])
360  logging.debug('Updating LASTCHANGE.blink ...')
361  with open(os.path.join(merge_common.REPOSITORY_ROOT,
362                         'build/util/LASTCHANGE.blink'), 'w') as f:
363    f.write('LASTCHANGE=%s\n' % _GetBlinkRevision())
364  merge_common.GetCommandStdout(['git', 'add', '-f',
365                                 'build/util/LASTCHANGE.blink'])
366  if _ModifiedFilesInIndex():
367    merge_common.GetCommandStdout([
368        'git', 'commit', '-m',
369        'Update LASTCHANGE file after merge of Chromium at %s\n\n%s'
370        % (version, AUTOGEN_MESSAGE)])
371
372
373def GetHEAD():
374  """Fetch the latest HEAD revision from the Chromium Git mirror.
375
376  Returns:
377    The latest HEAD revision (A Git abbrev SHA1).
378  """
379  return _GetGitAbbrevSHA1(SRC_GIT_BRANCH, 'HEAD')
380
381
382def _ParseSvnRevisionFromGitCommitMessage(commit_message):
383  return re.search(r'^git-svn-id: .*@([0-9]+)', commit_message,
384                   flags=re.MULTILINE).group(1)
385
386
387def _GetGitAbbrevSHA1(git_branch, revision):
388  """Returns an abbrev. SHA for the given revision (or branch, if HEAD)."""
389  assert revision
390  logging.debug('Getting Git revision for %s ...', revision)
391
392  upstream = git_branch if revision == 'HEAD' else revision
393
394  # Make sure the remote and the branch exist locally.
395  try:
396    merge_common.GetCommandStdout([
397        'git', 'show-ref', '--verify', '--quiet', git_branch])
398  except merge_common.CommandError:
399    raise merge_common.TemporaryMergeError(
400        'Cannot find the branch %s. Have you sync\'d master-chromium in this '
401        'checkout?' % git_branch)
402
403  # Make sure the |upstream| Git object has been mirrored.
404  try:
405    merge_common.GetCommandStdout([
406        'git', 'merge-base', '--is-ancestor', upstream, git_branch])
407  except merge_common.CommandError:
408    raise merge_common.TemporaryMergeError(
409        'Upstream object (%s) not reachable from %s' % (upstream, git_branch))
410
411  abbrev_sha = merge_common.Abbrev(merge_common.GetCommandStdout(
412      ['git', 'rev-list', '--max-count=1', upstream]).split()[0])
413  return abbrev_sha
414
415
416def _GetBlinkRevision():
417  # TODO(primiano): Switch to Git as soon as Blink gets migrated as well.
418  commit = merge_common.GetCommandStdout(
419      ['git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b'],
420      cwd=os.path.join(merge_common.REPOSITORY_ROOT, 'third_party', 'WebKit'))
421  return _ParseSvnRevisionFromGitCommitMessage(commit)
422
423
424def Snapshot(root_sha1, release, target, unattended, buildspec_url):
425  """Takes a snapshot of the Chromium tree and merges it into Android.
426
427  Android makefiles and a top-level NOTICE file are generated and committed
428  after the merge.
429
430  Args:
431    root_sha1: The abbrev sha1 in the Chromium git mirror to merge from.
432    release: The Chromium release version to merge from (e.g. "30.0.1599.20").
433             Only one of root_sha1 and release should be specified.
434    target: The target branch to merge to.
435    unattended: Run in unattended mode.
436    buildspec_url: URL for buildspec repository, used when merging a release.
437
438  Returns:
439    True if new commits were merged; False if no new commits were present.
440  """
441  if release:
442    root_sha1 = None
443    version = release
444  else:
445    root_sha1 = _GetGitAbbrevSHA1(SRC_GIT_BRANCH, root_sha1)
446    version = root_sha1
447
448  assert (root_sha1 is not None and len(root_sha1) > 6) or version == release
449
450  if root_sha1 and not merge_common.GetCommandStdout(
451      ['git', 'rev-list', '-1', 'HEAD..' + root_sha1]):
452    logging.info('No new commits to merge at %s (%s)', version, root_sha1)
453    return False
454
455  logging.info('Snapshotting Chromium at %s (%s)', version, root_sha1)
456
457  # 1. Merge, accounting for excluded directories
458  merged_sha1 = _MergeProjects(version, root_sha1, target, unattended,
459                               buildspec_url)
460
461  # 2. Generate Android makefiles
462  _GenerateMakefiles(version, unattended)
463
464  # 3. Check for incompatible licenses
465  _CheckLicenses()
466
467  # 4. Generate Android NOTICE file
468  _GenerateNoticeFile(version)
469
470  # 5. Generate LASTCHANGE file
471  _GenerateLastChange(version, merged_sha1)
472
473  return True
474
475
476def Push(version, target):
477  """Push the finished snapshot to the Android repository."""
478  src = 'merge-from-chromium-%s' % version
479  # Use forced pushes ('+' prefix) for the temporary and archive branches in
480  # case they already got updated by a previous (possibly failed?) merge, but
481  # do not force push to the real master-chromium branch as this could erase
482  # downstream changes.
483  refspecs = ['%s:%s' % (src, target),
484              '+%s:refs/archive/chromium-%s' % (src, version)]
485  if target == 'master-chromium':
486    refspecs.insert(0, '+%s:master-chromium-merge' % src)
487  for refspec in refspecs:
488    logging.debug('Pushing to server (%s) ...', refspec)
489    for path in merge_common.ALL_PROJECTS:
490      if path in merge_common.PROJECTS_WITH_FLAT_HISTORY:
491        remote = 'history'
492      else:
493        remote = 'goog'
494      logging.debug('Pushing %s', path)
495      dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path)
496      merge_common.GetCommandStdout(['git', 'push', remote, refspec],
497                                    cwd=dest_dir)
498
499
500def main():
501  parser = optparse.OptionParser(usage='%prog [options]')
502  parser.epilog = ('Takes a snapshot of the Chromium tree at the specified '
503                   'Chromium Git revision and merges it into this repository. '
504                   'Paths marked as excluded for license reasons are removed '
505                   'as part of the merge. Also generates Android makefiles and '
506                   'generates a top-level NOTICE file suitable for use in the '
507                   'Android build.')
508  parser.add_option(
509      '', '--sha1',
510      default='HEAD',
511      help=('Merge to the specified chromium sha1 revision from ' +
512            SRC_GIT_BRANCH + ' branch. Default is HEAD, to merge from ToT.'))
513  parser.add_option(
514      '', '--release',
515      default=None,
516      help=('Merge to the specified chromium release buildspec (e.g., "30.0.'
517            '1599.20"). Only one of --sha1 and --release should be specified'))
518  parser.add_option(
519      '', '--buildspec_url',
520      default=None,
521      help=('Git URL for buildspec repository.'))
522  parser.add_option(
523      '', '--target',
524      default='master-chromium', metavar='BRANCH',
525      help=('Target branch to push to. Defaults to master-chromium.'))
526  parser.add_option(
527      '', '--push',
528      default=False, action='store_true',
529      help=('Push the result of a previous merge to the server. Note '
530            '--sha1 must be given.'))
531  parser.add_option(
532      '', '--get_head',
533      default=False, action='store_true',
534      help=('Just print the current HEAD revision on stdout and exit.'))
535  parser.add_option(
536      '', '--unattended',
537      default=False, action='store_true',
538      help=('Run in unattended mode.'))
539  parser.add_option(
540      '', '--no_changes_exit',
541      default=0, type='int',
542      help=('Exit code to use if there are no changes to merge, for scripts.'))
543  (options, args) = parser.parse_args()
544  if args:
545    parser.print_help()
546    return 1
547
548  if 'ANDROID_BUILD_TOP' not in os.environ:
549    print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.'
550    return 1
551
552  if os.environ.get('GYP_DEFINES'):
553    print >>sys.stderr, (
554        'The environment is defining GYP_DEFINES (=%s). It will affect the '
555        ' generated makefiles.' % os.environ['GYP_DEFINES'])
556    if not options.unattended and raw_input('Continue? [y/N]') != 'y':
557      return 1
558
559  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
560                      stream=sys.stdout)
561
562  if options.get_head:
563    logging.disable(logging.CRITICAL)  # Prevent log messages
564    print GetHEAD()
565  elif options.push:
566    if options.release:
567      Push(options.release, options.target)
568    elif options.sha1:
569      Push(options.sha1, options.target)
570    else:
571      print >>sys.stderr, 'You need to pass the version to push.'
572      return 1
573  else:
574    if not Snapshot(options.sha1, options.release, options.target,
575                    options.unattended, options.buildspec_url):
576      return options.no_changes_exit
577
578  return 0
579
580if __name__ == '__main__':
581  sys.exit(main())
582