merge_from_chromium.py revision 5568eda06be15d573395eaba779d62c09f6f96f1
1#!/usr/bin/env python 2# 3# Copyright (C) 2012 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17""" 18Merge Chromium into the Android tree. See the output of --help for details. 19 20""" 21import optparse 22import os 23import re 24import sys 25 26import merge_common 27 28 29# We need to import this *after* merging from upstream to get the latest 30# version. Set it to none here to catch uses before it's imported. 31webview_licenses = None 32 33 34AUTOGEN_MESSAGE = 'This commit was generated by merge_from_chromium.py.' 35 36 37def _ReadGitFile(git_url, git_branch, sha1, path): 38 """Reads a file from a remote git project at a specific revision. 39 Args: 40 git_url: The URL of the git server. 41 git_branch: The branch to read. 42 sha1: The SHA1 at which to read. 43 path: The relative path of the file to read. 44 Returns: 45 The contents of the specified file. 46 """ 47 48 # We fetch the branch to a temporary head so that we don't download the same 49 # commits multiple times. 50 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, 51 git_branch + ':cached_upstream']) 52 53 args = ['git', 'show', '%s:%s' % (sha1, path)] 54 return merge_common.GetCommandStdout(args) 55 56 57def _ParseDEPS(git_url, git_branch, sha1): 58 """Parses the .DEPS.git file from Chromium and returns its contents. 59 Args: 60 git_url: The URL of the git server. 61 git_branch: The branch to read. 62 sha1: The SHA1 at which to read. 63 Returns: 64 A dictionary of the contents of .DEPS.git at the specified revision 65 """ 66 67 class FromImpl(object): 68 """Used to implement the From syntax.""" 69 70 def __init__(self, module_name): 71 self.module_name = module_name 72 73 def __str__(self): 74 return 'From("%s")' % self.module_name 75 76 class _VarImpl(object): 77 def __init__(self, custom_vars, local_scope): 78 self._custom_vars = custom_vars 79 self._local_scope = local_scope 80 81 def Lookup(self, var_name): 82 """Implements the Var syntax.""" 83 if var_name in self._custom_vars: 84 return self._custom_vars[var_name] 85 elif var_name in self._local_scope.get('vars', {}): 86 return self._local_scope['vars'][var_name] 87 raise Exception('Var is not defined: %s' % var_name) 88 89 tmp_locals = {} 90 var = _VarImpl({}, tmp_locals) 91 tmp_globals = {'From': FromImpl, 'Var': var.Lookup, 'deps_os': {}} 92 deps_content = _ReadGitFile(git_url, git_branch, sha1, '.DEPS.git') 93 exec(deps_content) in tmp_globals, tmp_locals 94 return tmp_locals 95 96 97def _GetThirdPartyProjectMergeInfo(third_party_projects, deps_vars): 98 """Gets the git URL for each project and the SHA1 at which it should be 99 merged. 100 Args: 101 third_party_projects: The list of projects to consider. 102 deps_vars: The dictionary of dependencies from .DEPS.git. 103 Returns: 104 A dictionary from project to git URL and SHA1 - 'path: (url, sha1)' 105 """ 106 107 deps_fallback_order = [ 108 deps_vars['deps'], 109 deps_vars['deps_os']['unix'], 110 deps_vars['deps_os']['android'], 111 ] 112 result = {} 113 for path in third_party_projects: 114 for deps in deps_fallback_order: 115 url_plus_sha1 = deps.get(os.path.join('src', path)) 116 if url_plus_sha1: 117 break 118 else: 119 raise RuntimeError( 120 ('Could not find .DEPS.git entry for project %s. This probably ' 121 'means that the project list in merge_from_chromium.py needs to be ' 122 'updated.') % 123 path) 124 match = re.match('(.*?)@(.*)', url_plus_sha1) 125 url = match.group(1) 126 sha1 = match.group(2) 127 print ' Got URL %s and SHA1 %s for project %s' % (url, sha1, path) 128 result[path] = {'url': url, 'sha1': sha1} 129 return result 130 131 132def _MergeProjects(git_url, git_branch, svn_revision, root_sha1): 133 """Merges into this repository all projects required by the specified branch 134 of Chromium, at the SVN revision. Uses a git subtree merge for each project. 135 Directories in the main Chromium repository which are not needed by Clank are 136 not merged. 137 Args: 138 git_url: The URL of the git server for the Chromium branch to merge to 139 git_branch: The branch name to merge to 140 svn_revision: The SVN revision for the main Chromium repository 141 root_sha1: The git SHA1 for the main Chromium repository 142 """ 143 144 # The logic for this step lives here, in the Android tree, as it makes no 145 # sense for a Chromium tree to know about this merge. 146 147 print 'Parsing DEPS ...' 148 deps_vars = _ParseDEPS(git_url, git_branch, root_sha1) 149 150 merge_info = _GetThirdPartyProjectMergeInfo(merge_common.THIRD_PARTY_PROJECTS, 151 deps_vars) 152 153 for path in merge_info: 154 url = merge_info[path]['url'] 155 sha1 = merge_info[path]['sha1'] 156 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 157 merge_common.GetCommandStdout(['git', 'checkout', 158 '-b', 'merge-from-chromium', 159 '-t', 'goog/master-chromium'], cwd=dest_dir) 160 print 'Fetching project %s at %s ...' % (path, sha1) 161 merge_common.GetCommandStdout(['git', 'fetch', url], cwd=dest_dir) 162 if merge_common.GetCommandStdout(['git', 'rev-list', '-1', 'HEAD..' + sha1], 163 cwd=dest_dir): 164 print 'Merging project %s at %s ...' % (path, sha1) 165 # Merge conflicts make git merge return 1, so ignore errors 166 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', sha1], 167 cwd=dest_dir, ignore_errors=True) 168 merge_common.CheckNoConflictsAndCommitMerge( 169 'Merge %s from %s at %s\n\n%s' % (path, url, sha1, AUTOGEN_MESSAGE), 170 cwd=dest_dir) 171 else: 172 print 'No new commits to merge in project %s' % path 173 174 # Handle root repository separately. 175 merge_common.GetCommandStdout(['git', 'checkout', '-b', 'merge-from-chromium', 176 '-t', 'goog/master-chromium']) 177 print 'Fetching Chromium at %s ...' % root_sha1 178 merge_common.GetCommandStdout(['git', 'fetch', git_url, git_branch]) 179 print 'Merging Chromium at %s ...' % root_sha1 180 # Merge conflicts make git merge return 1, so ignore errors 181 merge_common.GetCommandStdout(['git', 'merge', '--no-commit', root_sha1], 182 ignore_errors=True) 183 merge_common.CheckNoConflictsAndCommitMerge( 184 'Merge Chromium from %s branch %s at r%s (%s)\n\n%s' 185 % (git_url, git_branch, svn_revision, root_sha1, AUTOGEN_MESSAGE)) 186 187 print 'Getting directories to exclude ...' 188 189 # We import this now that we have merged the latest version. 190 # It imports to a global in order that it can be used to generate NOTICE 191 # later. We also disable writing bytecode to keep the source tree clean. 192 sys.path.append(os.path.join(merge_common.REPOSITORY_ROOT, 'android_webview', 193 'tools')) 194 sys.dont_write_bytecode = True 195 global webview_licenses 196 import webview_licenses 197 import known_issues 198 199 for path, exclude_list in known_issues.KNOWN_INCOMPATIBLE.iteritems(): 200 print ' %s' % '\n '.join(os.path.join(path, x) for x in exclude_list) 201 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 202 merge_common.GetCommandStdout(['git', 'rm', '-rf', '--ignore-unmatch'] + 203 exclude_list, cwd=dest_dir) 204 if _ModifiedFilesInIndex(dest_dir): 205 merge_common.GetCommandStdout(['git', 'commit', '-m', 206 'Exclude incompatible directories'], 207 cwd=dest_dir) 208 209 directories_left_over = webview_licenses.GetIncompatibleDirectories() 210 if directories_left_over: 211 raise RuntimeError('Incompatibly licensed directories remain: ' + 212 '\n'.join(directories_left_over)) 213 return True 214 215 216def _GenerateMakefiles(svn_revision): 217 """Run gyp to generate the makefiles required to build Chromium in the 218 Android build system. 219 """ 220 221 print 'Regenerating makefiles ...' 222 # TODO(torne): The .tmp files are generated by 223 # third_party/WebKit/Source/WebCore/WebCore.gyp/WebCore.gyp into the source 224 # tree. We should avoid this, or at least use a more specific name to avoid 225 # accidentally removing or adding other files. 226 for path in merge_common.ALL_PROJECTS: 227 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 228 merge_common.GetCommandStdout(['git', 'rm', '--ignore-unmatch', 229 'GypAndroid.*.mk', '*.target.*.mk', 230 '*.host.*.mk', '*.tmp'], cwd=dest_dir) 231 232 merge_common.GetCommandStdout(['android_webview/tools/gyp_webview']) 233 234 for path in merge_common.ALL_PROJECTS: 235 dest_dir = os.path.join(merge_common.REPOSITORY_ROOT, path) 236 # git add doesn't have an --ignore-unmatch so we have to do this instead: 237 merge_common.GetCommandStdout(['git', 'add', '-f', 'GypAndroid.*.mk'], 238 ignore_errors=True, cwd=dest_dir) 239 merge_common.GetCommandStdout(['git', 'add', '-f', '*.target.*.mk'], 240 ignore_errors=True, cwd=dest_dir) 241 merge_common.GetCommandStdout(['git', 'add', '-f', '*.host.*.mk'], 242 ignore_errors=True, cwd=dest_dir) 243 merge_common.GetCommandStdout(['git', 'add', '-f', '*.tmp'], 244 ignore_errors=True, cwd=dest_dir) 245 # Only try to commit the makefiles if something has actually changed. 246 if _ModifiedFilesInIndex(dest_dir): 247 merge_common.GetCommandStdout(['git', 'commit', '-m', 248 'Update makefiles after merge of Chromium at r%s\n\n%s' % 249 (svn_revision, AUTOGEN_MESSAGE)], cwd=dest_dir) 250 251 252def _ModifiedFilesInIndex(cwd=merge_common.REPOSITORY_ROOT): 253 """Returns whether git's index includes modified files, ie 'added' changes. 254 """ 255 status = merge_common.GetCommandStdout(['git', 'status', '--porcelain'], 256 cwd=cwd) 257 return re.search(r'^[MADRC]', status, flags=re.MULTILINE) != None 258 259 260def _GenerateNoticeFile(svn_revision): 261 """Generates a NOTICE file for all third-party code (from Android's 262 perspective) that lives in the Chromium tree and commits it to the root of 263 the repository. 264 Args: 265 svn_revision: The SVN revision for the main Chromium repository 266 """ 267 268 print 'Regenerating NOTICE file ...' 269 270 contents = webview_licenses.GenerateNoticeFile() 271 272 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'NOTICE'), 'w') as f: 273 f.write(contents) 274 merge_common.GetCommandStdout(['git', 'add', 'NOTICE']) 275 # Only try to commit the NOTICE update if the file has actually changed. 276 if _ModifiedFilesInIndex(): 277 merge_common.GetCommandStdout([ 278 'git', 'commit', '-m', 279 'Update NOTICE file after merge of Chromium at r%s\n\n%s' 280 % (svn_revision, AUTOGEN_MESSAGE)]) 281 282 283def _GenerateLastChange(svn_revision): 284 """Write a build/util/LASTCHANGE file containing the current revision. This is 285 used in the Chromium build to include the version number. 286 Args: 287 svn_revision: The SVN revision for the main Chromium repository 288 """ 289 290 print 'Updating LASTCHANGE ...' 291 with open(os.path.join(merge_common.REPOSITORY_ROOT, 'build/util/LASTCHANGE'), 292 'w') as f: 293 f.write("LASTCHANGE=%s\n" % svn_revision) 294 merge_common.GetCommandStdout(['git', 'add', '-f', 'build/util/LASTCHANGE']) 295 if _ModifiedFilesInIndex(): 296 merge_common.GetCommandStdout([ 297 'git', 'commit', '-m', 298 'Update LASTCHANGE file after merge of Chromium at r%s\n\n%s' 299 % (svn_revision, AUTOGEN_MESSAGE)]) 300 301 302def _GetSVNRevisionAndSHA1(git_url, git_branch, svn_revision): 303 print 'Getting SVN revision and SHA1 ...' 304 merge_common.GetCommandStdout(['git', 'fetch', '-f', git_url, 305 git_branch + ':cached_upstream']) 306 if svn_revision: 307 # Sometimes, we see multiple commits with the same git SVN ID. No idea why. 308 # We take the most recent. 309 sha1 = merge_common.GetCommandStdout([ 310 'git', 'log', '--grep=git-svn-id: .*@%s' % svn_revision, 311 '--format=%H', 'cached_upstream']).split()[0] 312 else: 313 # Just use the latest commit. 314 # TODO: We may be able to use a LKGR? 315 commit = merge_common.GetCommandStdout([ 316 'git', 'log', '-n1', '--grep=git-svn-id:', '--format=%H%n%b', 317 'cached_upstream']) 318 sha1 = commit.split()[0] 319 svn_revision = re.search(r'^git-svn-id: .*@([0-9]+)', commit, 320 flags=re.MULTILINE).group(1) 321 return (svn_revision, sha1) 322 323 324def _Snapshot(git_url, git_branch, svn_revision, autopush): 325 """Takes a snapshot of the specified Chromium tree at the specified SVN 326 revision and merges it into this repository. Also generates Android makefiles 327 and generates a top-level NOTICE file suitable for use in the Android build. 328 Args: 329 git_url: The URL of the git server for the Chromium branch to merge to 330 svn_revision: The SVN revision for the main Chromium repository 331 """ 332 333 (svn_revision, root_sha1) = _GetSVNRevisionAndSHA1(git_url, git_branch, 334 svn_revision) 335 if not merge_common.GetCommandStdout(['git', 'rev-list', '-1', 336 'HEAD..' + root_sha1]): 337 print ('No new commits to merge from %s branch %s at r%s (%s)' % 338 (git_url, git_branch, svn_revision, root_sha1)) 339 return 340 341 print ('Snapshotting Chromium from %s branch %s at r%s (%s)' % 342 (git_url, git_branch, svn_revision, root_sha1)) 343 344 # 1. Merge, accounting for excluded directories 345 _MergeProjects(git_url, git_branch, svn_revision, root_sha1) 346 347 # 2. Generate Android NOTICE file 348 _GenerateNoticeFile(svn_revision) 349 350 # 3. Generate LASTCHANGE file 351 _GenerateLastChange(svn_revision) 352 353 # 4. Generate Android makefiles 354 _GenerateMakefiles(svn_revision) 355 356 # 5. Push result to server 357 merge_common.PushToServer(autopush, 'merge-from-chromium', 'master-chromium') 358 359 return True 360 361 362def main(): 363 parser = optparse.OptionParser(usage='%prog [options]') 364 parser.epilog = ('Takes a snapshot of the Chromium tree at the specified ' 365 'Chromium SVN revision and merges it into this repository. ' 366 'Paths marked as excluded for license reasons are removed ' 367 'as part of the merge. Also generates Android makefiles and ' 368 'generates a top-level NOTICE file suitable for use in the ' 369 'Android build.') 370 parser.add_option( 371 '', '--git_url', 372 default='http://git.chromium.org/chromium/src.git', 373 help=('The URL of the git server for the Chromium branch to merge. ' 374 'Defaults to upstream.')) 375 parser.add_option( 376 '', '--git_branch', 377 default='git-svn', 378 help=('The name of the upstream branch to merge. Defaults to git-svn.')) 379 parser.add_option( 380 '', '--svn_revision', 381 default=None, 382 help=('Merge to the specified chromium SVN revision, rather than using ' 383 'the current latest revision.')) 384 parser.add_option( 385 '', '--autopush', 386 default=False, action='store_true', 387 help=('Automatically push the result to the server without prompting if' 388 'the merge was successful.')) 389 (options, args) = parser.parse_args() 390 if args: 391 parser.print_help() 392 return 1 393 394 if 'ANDROID_BUILD_TOP' not in os.environ: 395 print >>sys.stderr, 'You need to run the Android envsetup.sh and lunch.' 396 return 1 397 398 if not _Snapshot(options.git_url, options.git_branch, options.svn_revision, 399 options.autopush): 400 return 1 401 402 return 0 403 404if __name__ == '__main__': 405 sys.exit(main()) 406