merge_common.py revision 1bcf220e8ca3cfeb937c8e6dcf8138ab5611e8fc 
1# Copyright (C) 2012 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
15"""Common data/functions for the Chromium merging scripts."""
16
17import logging
18import os
19import re
20import subprocess
21
22
23REPOSITORY_ROOT = os.path.join(os.environ['ANDROID_BUILD_TOP'],
24                               'external/chromium_org')
25
26
27# Whitelist of projects that need to be merged to build WebView. We don't need
28# the other upstream repositories used to build the actual Chrome app.
29# Different stages of the merge process need different ways of looking at the
30# list, so we construct different combinations below.
31
32THIRD_PARTY_PROJECTS_WITH_FLAT_HISTORY = [
33    'third_party/WebKit',
34]
35
36THIRD_PARTY_PROJECTS_WITH_FULL_HISTORY = [
37    'sdch/open-vcdiff',
38    'testing/gtest',
39    'third_party/angle',
40    'third_party/brotli/src',
41    'third_party/eyesfree/src/android/java/src/com/googlecode/eyesfree/braille',
42    'third_party/freetype',
43    'third_party/icu',
44    'third_party/leveldatabase/src',
45    'third_party/libaddressinput/src',
46    'third_party/libjingle/source/talk',
47    'third_party/libphonenumber/src/phonenumbers',
48    'third_party/libphonenumber/src/resources',
49    'third_party/libsrtp',
50    'third_party/libvpx',
51    'third_party/libyuv',
52    'third_party/mesa/src',
53    'third_party/openmax_dl',
54    'third_party/openssl',
55    'third_party/opus/src',
56    'third_party/ots',
57    'third_party/sfntly/cpp/src',
58    'third_party/skia',
59    'third_party/smhasher/src',
60    'third_party/usrsctp/usrsctplib',
61    'third_party/webrtc',
62    'third_party/yasm/source/patched-yasm',
63    'tools/grit',
64    'tools/gyp',
65    'v8',
66]
67
68PROJECTS_WITH_FLAT_HISTORY = ['.'] + THIRD_PARTY_PROJECTS_WITH_FLAT_HISTORY
69PROJECTS_WITH_FULL_HISTORY = THIRD_PARTY_PROJECTS_WITH_FULL_HISTORY
70
71THIRD_PARTY_PROJECTS = (THIRD_PARTY_PROJECTS_WITH_FLAT_HISTORY +
72                        THIRD_PARTY_PROJECTS_WITH_FULL_HISTORY)
73
74ALL_PROJECTS = ['.'] + THIRD_PARTY_PROJECTS
75
76
77# Directories to be removed when flattening history.
78PRUNE_WHEN_FLATTENING = {
79    'third_party/WebKit': [
80        'LayoutTests',
81    ],
82}
83
84
85# Only projects that have their history flattened can have directories pruned.
86assert all(p in PROJECTS_WITH_FLAT_HISTORY for p in PRUNE_WHEN_FLATTENING)
87
88
89class MergeError(Exception):
90  """Used to signal an error that prevents the merge from being completed."""
91
92
93class CommandError(MergeError):
94  """This exception is raised when a process run by GetCommandStdout fails."""
95
96  def __init__(self, returncode, cmd, cwd, stdout, stderr):
97    super(CommandError, self).__init__()
98    self.returncode = returncode
99    self.cmd = cmd
100    self.cwd = cwd
101    self.stdout = stdout
102    self.stderr = stderr
103
104  def __str__(self):
105    return ("Command '%s' returned non-zero exit status %d. cwd was '%s'.\n\n"
106            "===STDOUT===\n%s\n===STDERR===\n%s\n" %
107            (self.cmd, self.returncode, self.cwd, self.stdout, self.stderr))
108
109
110class TemporaryMergeError(MergeError):
111  """A merge error that can potentially be resolved by trying again later."""
112
113
114def GetCommandStdout(args, cwd=REPOSITORY_ROOT, ignore_errors=False):
115  """Gets stdout from runnng the specified shell command.
116
117  Similar to subprocess.check_output() except that it can capture stdout and
118  stderr separately for better error reporting.
119
120  Args:
121    args: The command and its arguments as an iterable.
122    cwd: The working directory to use. Defaults to REPOSITORY_ROOT.
123    ignore_errors: Ignore the command's return code and stderr.
124  Returns:
125    stdout from running the command.
126  Raises:
127    CommandError: if the command exited with a nonzero status.
128  """
129  p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
130                       stderr=subprocess.PIPE)
131  stdout, stderr = p.communicate()
132  if p.returncode == 0 or ignore_errors:
133    return stdout
134  else:
135    raise CommandError(p.returncode, ' '.join(args), cwd, stdout, stderr)
136
137
138def CheckNoConflictsAndCommitMerge(commit_message, unattended=False,
139                                   cwd=REPOSITORY_ROOT):
140  """Checks for conflicts and commits once they are resolved.
141
142  Certain conflicts are resolved automatically; if any remain, the user is
143  prompted to resolve them. The user can specify a custom commit message.
144
145  Args:
146    commit_message: The default commit message.
147    unattended: If running unattended, abort on conflicts.
148    cwd: Working directory to use.
149  Raises:
150    TemporaryMergeError: If there are conflicts in unattended mode.
151  """
152  status = GetCommandStdout(['git', 'status', '--porcelain'], cwd=cwd)
153  conflicts_deleted_by_us = re.findall(r'^(?:DD|DU) ([^\n]+)$', status,
154                                       flags=re.MULTILINE)
155  if conflicts_deleted_by_us:
156    logging.info('Keeping ours for the following locally deleted files.\n  %s',
157                 '\n  '.join(conflicts_deleted_by_us))
158    GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] +
159                     conflicts_deleted_by_us, cwd=cwd)
160
161  # If upstream renames a file we have deleted then it will conflict, but
162  # we shouldn't just blindly delete these files as they may have been renamed
163  # into a directory we don't delete. Let them get re-added; they will get
164  # re-deleted if they are still in a directory we delete.
165  conflicts_renamed_by_them = re.findall(r'^UA ([^\n]+)$', status,
166                                         flags=re.MULTILINE)
167  if conflicts_renamed_by_them:
168    logging.info('Adding theirs for the following locally deleted files.\n %s',
169                 '\n  '.join(conflicts_renamed_by_them))
170    GetCommandStdout(['git', 'add', '-f'] + conflicts_renamed_by_them, cwd=cwd)
171
172  while True:
173    status = GetCommandStdout(['git', 'status', '--porcelain'], cwd=cwd)
174    conflicts = re.findall(r'^((DD|AU|UD|UA|DU|AA|UU) [^\n]+)$', status,
175                           flags=re.MULTILINE)
176    if not conflicts:
177      break
178    if unattended:
179      GetCommandStdout(['git', 'reset', '--hard'], cwd=cwd)
180      raise TemporaryMergeError('Cannot resolve merge conflicts.')
181    conflicts_string = '\n'.join([x[0] for x in conflicts])
182    new_commit_message = raw_input(
183        ('The following conflicts exist and must be resolved.\n\n%s\n\nWhen '
184         'done, enter a commit message or press enter to use the default '
185         '(\'%s\').\n\n') % (conflicts_string, commit_message))
186    if new_commit_message:
187      commit_message = new_commit_message
188
189  GetCommandStdout(['git', 'commit', '-m', commit_message], cwd=cwd)
190