1#!/usr/bin/python
2#
3# Copyright (C) 2012 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""Merge master-chromium to master within the Android tree."""
18
19import logging
20import optparse
21import os
22import re
23import shutil
24import subprocess
25import sys
26
27import merge_common
28
29
30AUTOGEN_MESSAGE = 'This commit was generated by merge_to_master.py.'
31WEBVIEW_PROJECT = 'frameworks/webview'
32
33
34def _GetAbsPath(project):
35  """Returns the full path to a given project (either Chromium or Android)."""
36  if project in merge_common.ALL_PROJECTS:
37    abs_path = os.path.join(merge_common.REPOSITORY_ROOT, project)
38  else:
39    abs_path = os.path.join(os.environ['ANDROID_BUILD_TOP'], project)
40  if not os.path.exists(abs_path):
41    raise merge_common.MergeError('Cannot find path ' + abs_path)
42  return abs_path
43
44
45def _CheckoutSingleProject(project, target_branch):
46  """Checks out the tip of the target_branch into a local branch (merge-to-XXX).
47
48  Args:
49    project: a Chromium project (., third_party/foo) or frameworks/webview.
50    target_branch: name of the target branch (in the goog remote).
51  """
52  dest_dir = _GetAbsPath(project)
53  tracking_branch = 'goog/' + target_branch
54  logging.debug('Check out %-45s at %-16s', project, tracking_branch)
55  merge_common.GetCommandStdout(['git', 'remote', 'update', 'goog'],
56                                cwd=dest_dir)
57  merge_common.GetCommandStdout(['git', 'checkout',
58                                 '-b', 'merge-to-' + target_branch,
59                                 '-t', tracking_branch], cwd=dest_dir)
60
61
62def _FetchSingleProject(project, remote, remote_ref):
63  """Fetches a remote ref for the given project and returns the fetched SHA.
64
65  Args:
66    project: a Chromium project (., third_party/foo) or frameworks/webview.
67    remote: Git remote name (goog for most projects, history for squashed ones).
68    remote_ref: the remote ref to fetch (e.g., refs/archive/chromium-XXX).
69
70  Returns:
71    The SHA1 of the FETCH_HEAD.
72  """
73  dest_dir = _GetAbsPath(project)
74  logging.debug('Fetch     %-45s %s:%s', project, remote, remote_ref)
75  merge_common.GetCommandStdout(['git', 'fetch', remote, remote_ref],
76                                cwd=dest_dir)
77  return merge_common.GetCommandStdout(['git', 'rev-parse', 'FETCH_HEAD'],
78                                       cwd=dest_dir).strip()
79
80
81def _MergeSingleProject(project, merge_sha, revision, target_branch, flatten):
82  """Merges a single project at a given SHA.
83
84  Args:
85    project: a Chromium project (., third_party/foo) or frameworks/webview.
86    merge_sha: the SHA to merge.
87    revision: Abbrev. commitish in the main Chromium repository.
88    target_branch: name of the target branch.
89    flatten: True: squash history while merging; False: perform a normal merge.
90  """
91  dest_dir = _GetAbsPath(project)
92  if flatten:
93    # Make the previous merges into grafts so we can do a correct merge.
94    old_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
95                                            cwd=dest_dir).strip()
96    merge_log = os.path.join(dest_dir, '.merged-revisions')
97    if os.path.exists(merge_log):
98      shutil.copyfile(merge_log,
99                      os.path.join(dest_dir, '.git', 'info', 'grafts'))
100
101  # Early out if there is nothing to merge.
102  if not merge_common.GetCommandStdout(['git', 'rev-list', '-1',
103                                        'HEAD..' + merge_sha], cwd=dest_dir):
104    logging.debug('No new commits to merge in project %s', project)
105    return
106
107  logging.debug('Merging project %s (flatten: %s)...', project, flatten)
108  merge_cmd = ['git', 'merge', '--no-commit']
109  merge_cmd += ['--squash'] if flatten else ['--no-ff']
110  merge_cmd += [merge_sha]
111  # Merge conflicts cause 'git merge' to return 1, so ignore errors
112  merge_common.GetCommandStdout(merge_cmd, cwd=dest_dir, ignore_errors=True)
113
114  if flatten:
115    dirs_to_prune = merge_common.PRUNE_WHEN_FLATTENING.get(project, [])
116    if dirs_to_prune:
117      merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', '-rf'] +
118                                    dirs_to_prune, cwd=dest_dir)
119
120  if project in merge_common.ALL_PROJECTS:
121    commit_msg = 'Merge from Chromium at DEPS revision %s' % revision
122  else:
123    commit_msg = 'Merge master-chromium into %s at %s' % (target_branch,
124                                                          revision)
125  commit_msg += '\n\n' + AUTOGEN_MESSAGE
126  merge_common.CheckNoConflictsAndCommitMerge(commit_msg, cwd=dest_dir)
127
128  if flatten:
129    # Generate the new grafts file and commit it on top of the merge.
130    new_sha = merge_common.GetCommandStdout(['git', 'rev-parse', 'HEAD'],
131                                            cwd=dest_dir).strip()
132    with open(merge_log, 'a+') as f:
133      f.write('%s %s %s\n' % (new_sha, old_sha, merge_sha))
134    merge_common.GetCommandStdout(['git', 'add', '.merged-revisions'],
135                                  cwd=dest_dir)
136    merge_common.GetCommandStdout(
137        ['git', 'commit', '-m',
138         'Record Chromium merge at DEPS revision %s\n\n%s' %
139         (revision, AUTOGEN_MESSAGE)], cwd=dest_dir)
140
141
142def _IsAncestor(ref1, ref2, cwd):
143  """Checks whether ref1 is a ancestor of ref2 in the given Git repo."""
144  cmd = ['git', 'merge-base', '--is-ancestor', ref1, ref2]
145  ret = subprocess.call(cmd, cwd=cwd)
146  if ret == 0:
147    return True
148  elif ret == 1:
149    return False
150  else:
151    raise merge_common.CommandError(ret, ' '.join(cmd), cwd, 'N/A', 'N/A')
152
153
154def _MergeChromiumProjects(revision, target_branch, repo_shas=None,
155                           force=False):
156  """Merges the Chromium projects from master-chromium to target_branch.
157
158  The larger projects' histories are flattened in the process.
159  When repo_shas != None, it checks that the SHAs of the projects in the
160  archive match exactly the SHAs of the projects in repo.prop.
161
162  Args:
163    revision: Abbrev. commitish in the main Chromium repository.
164    target_branch: target branch name to merge and push to.
165    repo_shas: optional dict. of expected revisions (only for --repo-prop).
166    force: True: merge anyways using the SHAs from repo.prop; False: bail out if
167                 projects mismatch (archive vs repo.prop).
168  """
169  # Sync and checkout ToT for all projects (creating the merge-to-XXX branch)
170  # and fetch the archive snapshot.
171  fetched_shas = {}
172  remote_ref = 'refs/archive/chromium-%s' % revision
173  for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
174    _CheckoutSingleProject(project, target_branch)
175    fetched_shas[project] = _FetchSingleProject(project, 'history', remote_ref)
176  for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
177    _CheckoutSingleProject(project, target_branch)
178    fetched_shas[project] = _FetchSingleProject(project, 'goog', remote_ref)
179
180  if repo_shas:
181    project_shas_mismatch = False
182    for project, merge_sha in fetched_shas.items():  # the dict can be modified.
183      expected_sha = repo_shas.get(project)
184      if expected_sha != merge_sha:
185        logging.warn('The SHA for project %s specified in the repo.prop (%s) '
186                     'and the one in the archive (%s) differ.',
187                     project, expected_sha, merge_sha)
188        dest_dir = _GetAbsPath(project)
189        if expected_sha is None:
190          reason = 'cannot find a SHA in the repo.pro for %s' % project
191        elif _IsAncestor(merge_sha, expected_sha, cwd=dest_dir):
192          reason = 'the SHA in repo.prop is ahead of the SHA in the archive. '
193          log_cmd = ['git', 'log', '--oneline', '--graph', '--max-count=10',
194                     '%s..%s' % (merge_sha, expected_sha)]
195          log_cmd_output = merge_common.GetCommandStdout(log_cmd, cwd=dest_dir)
196          reason += 'showing partial log (%s): \n %s' % (' '.join(log_cmd),
197                                                         log_cmd_output)
198        elif _IsAncestor(expected_sha, merge_sha, cwd=dest_dir):
199          reason = 'The SHA is already merged in the archive'
200        else:
201          reason = 'The project history diverged. Consult your Git historian.'
202
203        project_shas_mismatch = True
204        if force:
205          logging.debug('Merging the SHA in repo.prop anyways (due to --force)')
206          fetched_shas[project] = expected_sha
207        else:
208          logging.debug('Reason: %s', reason)
209    if not force and project_shas_mismatch:
210      raise merge_common.MergeError(
211          'The revision of some projects in the archive is different from the '
212          'one provided in build.prop. See the log for more details. Re-run '
213          'with --force to continue.')
214
215  for project in merge_common.PROJECTS_WITH_FLAT_HISTORY:
216    _MergeSingleProject(project, fetched_shas[project], revision, target_branch,
217                        flatten=True)
218  for project in merge_common.PROJECTS_WITH_FULL_HISTORY:
219    _MergeSingleProject(project, fetched_shas[project], revision, target_branch,
220                        flatten=False)
221
222
223def _GetNearestUpstreamAbbrevSHA(reference='history/master-chromium'):
224  """Returns the abbrev. upstream SHA which closest to the given reference."""
225  logging.debug('Getting upstream SHA for %s...', reference)
226  merge_common.GetCommandStdout(['git', 'remote', 'update', 'history'])
227  upstream_commit = merge_common.Abbrev(merge_common.GetCommandStdout([
228      'git', 'merge-base', 'history/upstream-master', reference]))
229
230  # Pedantic check: look for the existence of a merge commit which contains the
231  # |upstream_commit| in its message and is its children.
232  merge_parents = merge_common.GetCommandStdout([
233      'git', 'rev-list', reference, '--grep', upstream_commit, '--merges',
234      '--parents', '-1'])
235  if upstream_commit not in merge_parents:
236    raise merge_common.MergeError(
237        'Found upstream commit %s, but the merge child (%s) could not be found '
238        'or is not a parent of the upstream SHA')
239  logging.debug('Found nearest Chromium revision %s', upstream_commit)
240  return upstream_commit
241
242
243def _MergeWithRepoProp(repo_prop_file, target_branch, force):
244  """Performs a merge using a repo.prop file (from Android build waterfall).
245
246  This does NOT merge (unless forced with force=True) the pinned
247  revisions in repo.prop, as a repo.prop can snapshot an intermediate state
248  (between two automerger cycles). Instead, this looks up the archived snapshot
249  (generated by the chromium->master-chromium auto-merger) which is closest to
250  the given repo.prop (following the main Chromium project) and merges that one.
251  If the projects revisions don't match, it fails with detailed error messages.
252
253  Args:
254    repo_prop_file: Path to a downloaded repo.prop file.
255    target_branch: name of the target branch to merget to.
256    force: ignores the aforementioned check and merged anyways.
257  """
258  chromium_sha = None
259  webview_sha = None
260  repo_shas = {}  # 'project/path' -> 'sha'
261  with open(repo_prop_file) as prop:
262    for line in prop:
263      repo, sha = line.split()
264      # Translate the Android repo paths into the relative project paths used in
265      # merge_common (e.g., platform/external/chromium_org/foo -> foo).
266      m = (
267          re.match(r'^platform/(frameworks/.+)$', repo) or
268          re.match(r'^platform/external/chromium_org/?(.*?)(-history)?$', repo))
269      if m:
270        project = m.group(1) if m.group(1) else '.'  # '.' = Main project.
271        repo_shas[project] = sha
272
273  chromium_sha = repo_shas.get('.')
274  webview_sha = repo_shas.get(WEBVIEW_PROJECT)
275  if not chromium_sha or not webview_sha:
276    raise merge_common.MergeError('SHAs for projects not found; '
277                                  'invalid build.prop?')
278
279  # Check that the revisions in repo.prop and the on in the archive match.
280  archived_chromium_revision = _GetNearestUpstreamAbbrevSHA(chromium_sha)
281  logging.info('Merging Chromium at %s and WebView at %s',
282               archived_chromium_revision, webview_sha)
283  _MergeChromiumProjects(archived_chromium_revision, target_branch, repo_shas,
284                         force)
285
286  _CheckoutSingleProject(WEBVIEW_PROJECT, target_branch)
287  _MergeSingleProject(WEBVIEW_PROJECT, webview_sha,
288                      archived_chromium_revision, target_branch, flatten=False)
289
290
291def Push(target_branch):
292  """Push the finished snapshot to the Android repository.
293
294  Creates first a CL for frameworks/webview (if the merge-to-XXX branch exists)
295  then wait for user confirmation and pushes the Chromium merges. This is to
296  give an opportunity to get a +2 for  frameworks/webview and then push both
297  frameworks/webview and the Chromium projects atomically(ish).
298
299  Args:
300    target_branch: name of the target branch (in the goog remote).
301  """
302  merge_branch = 'merge-to-%s' % target_branch
303
304  # Create a Gerrit CL for the frameworks/webview project (if needed).
305  dest_dir = _GetAbsPath(WEBVIEW_PROJECT)
306  did_upload_webview_cl = False
307  if merge_common.GetCommandStdout(['git', 'branch', '--list', merge_branch],
308                                   cwd=dest_dir):
309    # Check that there was actually something to merge.
310    merge_range = 'goog/%s..%s' % (target_branch, merge_branch)
311    if merge_common.GetCommandStdout(['git', 'rev-list', '-1', merge_range],
312                                     cwd=dest_dir):
313      logging.info('Uploading a merge CL for %s...', WEBVIEW_PROJECT)
314      refspec = '%s:refs/for/%s' % (merge_branch, target_branch)
315      upload = merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
316                                             cwd=dest_dir)
317      logging.info(upload)
318      did_upload_webview_cl = True
319
320  prompt_msg = 'About push the Chromium projects merge. '
321  if not did_upload_webview_cl:
322    logging.info('No merge CL needed for %s.', WEBVIEW_PROJECT)
323  else:
324    prompt_msg += ('At this point you should have the CL +2-ed and merge it '
325                   'together with this push.')
326  prompt_msg += '\nPress "y" to continue: '
327  if raw_input(prompt_msg) != 'y':
328    logging.warn('Push aborted by the user!')
329    return
330
331  logging.debug('Pushing Chromium projects to %s ...', target_branch)
332  refspec = '%s:%s' % (merge_branch, target_branch)
333  for path in merge_common.ALL_PROJECTS:
334    logging.debug('Pushing %s', path)
335    dest_dir = _GetAbsPath(path)
336    # Delete the graft before pushing otherwise git will attempt to push all the
337    # grafted-in objects to the server as well as the ones we want.
338    graftfile = os.path.join(dest_dir, '.git', 'info', 'grafts')
339    if os.path.exists(graftfile):
340      os.remove(graftfile)
341    merge_common.GetCommandStdout(['git', 'push', 'goog', refspec],
342                                  cwd=dest_dir)
343
344
345def main():
346  parser = optparse.OptionParser(usage='%prog [options]')
347  parser.epilog = ('Takes the current master-chromium branch of the Chromium '
348                   'projects in Android and merges them into master to publish '
349                   'them.')
350  parser.add_option(
351      '', '--revision',
352      default=None,
353      help=('Merge to the specified archived master-chromium revision (abbrev. '
354            'SHA or release version) rather than using HEAD. e.g., '
355            '--revision=a1b2c3d4e5f6 or --revision=38.0.2125.24'))
356  parser.add_option(
357      '', '--repo-prop',
358      default=None, metavar='FILE',
359      help=('Merge to the revisions specified in this repo.prop file.'))
360  parser.add_option(
361      '', '--force',
362      default=False, action='store_true',
363      help=('Skip history checks and merged anyways (only for --repo-prop).'))
364  parser.add_option(
365      '', '--push',
366      default=False, action='store_true',
367      help=('Push the result of a previous merge to the server.'))
368  parser.add_option(
369      '', '--target',
370      default='master', metavar='BRANCH',
371      help=('Target branch to push to. Defaults to master.'))
372  (options, args) = parser.parse_args()
373  if args:
374    parser.print_help()
375    return 1
376
377  logging.basicConfig(format='%(message)s', level=logging.DEBUG,
378                      stream=sys.stdout)
379
380  if options.push:
381    Push(options.target)
382  elif options.repo_prop:
383    _MergeWithRepoProp(os.path.expanduser(options.repo_prop),
384                       options.target, options.force)
385  elif options.revision:
386    _MergeChromiumProjects(options.revision, options.target)
387  else:
388    first_upstream_sha = _GetNearestUpstreamAbbrevSHA()
389    _MergeChromiumProjects(first_upstream_sha, options.target)
390
391  return 0
392
393if __name__ == '__main__':
394  sys.exit(main())
395